import unittest
import importlib
import re
import types
from nltk.tokenize import sent_tokenize, word_tokenize
import nltk


# Helper function to require a successful test
def _require_success(test_case, condition, failure_message):
  if not condition:
    test_case._testMethodName = failure_message  # Rename the test with a failure description
    test_case.fail()  # Fail the test with a failure message


# Helper function to check if test is successful
def _check_success(test_case, condition, success_message, failure_message):
  if condition:
    test_case._testMethodName = success_message  # Rename the test for better readability in test reports
  else:
    test_case._testMethodName = failure_message  # Rename the test with a failure description
    test_case.fail()  # Fail the test with a failure message


class TestUserCode(unittest.TestCase):
  # Test that everything is imported correctly
  def test_imports(self):
    import user_code
    _require_success(
      self,
      hasattr(user_code, "sent_tokenize"),
      "`sent_tokenize` is not declared"
    )
    _require_success(
      self,
      user_code.sent_tokenize == sent_tokenize,
      "`sent_tokenize` is declared, but it's not `nltk.tokenize.sent_tokenize`"
    )
    _require_success(
      self,
      hasattr(user_code, "word_tokenize"),
      "`word_tokenize` is not declared"
    )
    _require_success(
      self,
      user_code.word_tokenize == word_tokenize,
      "`word_tokenize` is declared, but it's not `nltk.tokenize.word_tokenize`"
    )
    _require_success(
      self,
      hasattr(user_code, "nltk"),
      "`nltk` is not declared"
    )
    _require_success(
      self,
      user_code.nltk == nltk,
      "`nltk` is declared, but it's not `nltk` library"
    )
    self._testMethodName = "imports are correct"

  # Test that 'punk_tab' is downloaded
  def test_nltk_download_punkt(self):
    import user_code
    _require_success(
      self,
      hasattr(user_code, "nltk"),
      "`nltk` is not declared"
    )
    _require_success(
      self,
      user_code.nltk == nltk,
      "`nltk` is declared, but it's not `nltk` library"
    )
    with open("user_code.py", "r") as f:
      user_code_text = f.read()
      _require_success(
        self,
        re.search(r"nltk *\. *download *\( *'punkt_tab' *\)", user_code_text),
        "Code for downloading `'punkt_tab'` tokenizer shouldn't be modified"
      )
    self._testMethodName = "`'punkt_tab'` tokenizer is downloaded"

  # Test that text was not modified
  @unittest.skip("doesn't work properly now")
  def test_text(self):
    import user_code
    _require_success(
      self,
      hasattr(user_code, "text"),
      "`text` is not declared"
    )
    expected = 'Sherlock Holmes picked them up one by one, and laid them along the edge of the table. Then he reseated himself in the chair, and looked over with a gleam of satisfaction in his eyes. "These," said he, "are all that I have left to remind me of âThe Adventure of the Musgrave Ritualâ."'
    _check_success(
      self,
      user_code.text == expected,
      "`text` contains correct value",
      "`text` shouldn't be modified"
    )

  # Test that text was correctly tokenized into sentences
  def test_sent_tokenize(self):
    import user_code
    _require_success(
      self,
      hasattr(user_code, "text"),
      "`text` is not declared"
    )
    _require_success(
      self,
      isinstance(user_code.text, str),
      "`text` is not a string"
    )
    _require_success(
      self,
      hasattr(user_code, "sentences"),
      "`sentences` is not declared"
    )
    _check_success(
      self,
      sent_tokenize(user_code.text) == user_code.sentences,
      "`sentences` is computed correctly",
      "`text` is incorrectly tokenized into sentences"
    )

  # Test that last sentence was saved and transformed into lowercase
  def test_last_sentence(self):
    import user_code
    _require_success(
      self,
      hasattr(user_code, "sentences"),
      "`sentences` is not declared"
    )
    _require_success(
      self,
      isinstance(user_code.sentences, list),
      "`sentences` is not a list"
    )
    _require_success(
      self,
      isinstance(user_code.sentences[-1], str),
      "last element of `sentences` is not a string"
    )
    _require_success(
      self,
      hasattr(user_code, "last_sentence"),
      "`last_sentence` is not declared"
    )
    _require_success(
      self,
      user_code.last_sentence.lower() == user_code.sentences[-1].lower(),
      "`last_sentence` does not contain the last sentence"
    )
    _require_success(
      self,
      user_code.last_sentence == user_code.sentences[-1].lower(),
      "`last_sentence` is not in lowercase"
    )
    _testMethodName = "`last_sentence` is computed correctly"

  # Test that last sentence was tokenized into words
  def test_word_tokenize(self):
    import user_code
    _require_success(
      self,
      hasattr(user_code, "last_sentence"),
      "`last_sentence` is not declared"
    )
    _require_success(
      self,
      isinstance(user_code.last_sentence, str),
      "`last_sentence` is not a string"
    )
    _require_success(
      self,
      hasattr(user_code, "tokens"),
      "`tokens` is not declared"
    )
    _check_success(
      self,
      word_tokenize(user_code.last_sentence) == user_code.tokens,
      "`tokens` is computed correctly",
      "`last_sentence` is incorrectly tokenized into words"
    )

  # Test that print is not modified
  def test_print(self):
    with open("user_code.py", "r") as f:
      user_code_text = f.read()
      _check_success(
        self,
        re.search(r"print *\( *tokens *\)", user_code_text),
        "print statement is correct",
        "print statement shouldn't be modified"
      )


# Run the tests when executing the script
if __name__ == '__main__':
  nltk.download('punkt_tab')
  unittest.main()

test_main.py

Utforsk grunnleggende prinsipper innen Natural Language Processing (NLP) ved å lære essensielle teknikker for tekstforbehandling og metoder for å representere tekstdata. Få praktisk erfaring med verktøyene som brukes til å rense, analysere og tolke tekstinformasjon. Utvikle ferdigheter som kreves for å omforme råspråk til strukturerte innsikter, og legg et solid grunnlag for avanserte anvendelser innen kunstig intelligens og maskinlæring.

Utforsk det grunnleggende innen tekstforbehandling for å klargjøre råtekst for analyse. Lær hvordan man deler opp tekst i token, filtrerer ut stoppord og tilpasser tokenisering med regulære uttrykk.

Utforsk hvordan ord kan reduseres til sine grunnformer ved hjelp av stamming og lemmatisering. Behersk ordklassemerking for å berike tekst med grammatisk kontekst og anvend ordklassebevisst lemmatisering.

Lær hvordan tekst kan representeres med tall ved hjelp av vektorrommodeller. Få praktisk erfaring ved å implementere og tilpasse to populære vektorrommodeller: bag of words og TF-IDF.

Få en grundig forståelse av ordinnbygginger og hvordan de fanger opp semantisk betydning. Utforsk CBoW- og Skip-gram-arkitekturer brukt i Word2Vec, og implementer dem selv.

Utfordring: Tokenisering

Løsning

Utfordring: Tokenisering

Løsning