import unittest
import re
import nltk
from nltk.tokenize import word_tokenize
from nltk import pos_tag
import pandas as pd


# Helper functions
def _require_success(test_case, condition, failure_message):
    if not condition:
        test_case._testMethodName = failure_message
        test_case.fail()

def _check_success(test_case, condition, success_message, failure_message):
    if condition:
        test_case._testMethodName = success_message
    else:
        test_case._testMethodName = failure_message
        test_case.fail()


class TestUserCode(unittest.TestCase):
    # Test that everything is imported correctly
    def test_imports(self):
        import user_code
        _require_success(
            self, 
            hasattr(user_code, "word_tokenize"), 
            "`word_tokenize` is not declared"
        )
        _require_success(
            self,
            user_code.word_tokenize == word_tokenize,
            "`word_tokenize` is declared, but it's not `nltk.tokenize.word_tokenize`"
        )
        _require_success(
            self, 
            hasattr(user_code, "pos_tag"), 
            "`pos_tag` is not declared"
        )
        _require_success(
            self,
            user_code.pos_tag == pos_tag,
            "`pos_tag` is declared, but it's not `nltk.pos_tag`"
        )
        _require_success(
            self,
            hasattr(user_code, "nltk"),
            "`nltk` is not declared"
        )
        _require_success(
            self,
            user_code.nltk == nltk,
            "`nltk` is declared, but it's not `nltk` library"
        )
        self._testMethodName = "imports are correct"

    # Test that required NLTK resources are downloaded
    def test_nltk_downloads(self):
        import user_code
        _require_success(
            self,
            hasattr(user_code, "nltk"),
            "`nltk` is not declared"
        )
        _require_success(
            self,
            user_code.nltk == nltk,
            "`nltk` is declared, but it's not `nltk` library"
        )
        with open("user_code.py", "r") as f:
            user_code_text = f.read()
            _require_success(
                self,
                re.search(r"nltk *\. *download *\( *'punkt_tab' *\)", user_code_text),
                "Code for downloading `'punkt_tab'` tokenizer shouldn't be modified"
            )
            _require_success(
                self,
                re.search(r"nltk *\. *download *\( *'averaged_perceptron_tagger_eng' *\)", user_code_text),
                "Code for downloading `'averaged_perceptron_tagger_eng'` is not provided"
            )
        self._testMethodName = "NLTK resources are downloaded"

    # Test original text
    def test_text(self):
        import user_code
        expected = "The central bank raised interest rates by 0.5 percentage points yesterday."
        _require_success(
            self, 
            hasattr(user_code, "text"), 
            "`text` is not declared"
        )
        _check_success(
            self,
            user_code.text == expected,
            "`text` contains correct value",
            "`text` shouldn't be modified"
        )

    # Test lowercase conversion
    def test_text_lower(self):
        import user_code
        _require_success(
            self, 
            hasattr(user_code, "text"), 
            "`text` is not declared"
        )
        _require_success(
            self, 
            isinstance(user_code.text, str), 
            "`text` is not a string"
        )
        _require_success(
            self, 
            hasattr(user_code, "text_lower"), 
            "`text_lower` is not declared"
        )
        _check_success(
            self,
            user_code.text_lower == user_code.text.lower(),
            "`text_lower` is computed correctly",
            "`text_lower` is not the lowercase of `text`"
        )

    # Test tokenization of text_lower
    def test_tokens(self):
        import user_code
        _require_success(
            self, 
            hasattr(user_code, "text_lower"), 
            "`text_lower` is not declared"
        )
        _require_success(
            self, 
            isinstance(user_code.text_lower, str), 
            "`text_lower` is not a string"
        )
        _require_success(
            self, 
            hasattr(user_code, "tokens"), 
            "`tokens` is not declared"
        )
        _check_success(
            self,
            user_code.tokens == word_tokenize(user_code.text_lower),
            "`tokens` is computed correctly",
            "`text_lower` is not correctly tokenized into words"
        )

    # Test pos tagging results
    def test_tagged_tokens(self):
        import user_code
        _require_success(
            self, 
            hasattr(user_code, "tokens"), 
            "`tokens` is not declared"
        )
        _require_success(
            self, 
            isinstance(user_code.tokens, list), 
            "`tokens` is not a list"
        )
        _require_success(
            self, 
            hasattr(user_code, "tagged_tokens"), 
            "`tagged_tokens` is not declared"
        )
        _check_success(
            self,
            user_code.tagged_tokens == pos_tag(user_code.tokens),
            "`tagged_tokens` is computed correctly",
            "`tokens` are not correctly pos tagged"
        )

    # Test print statement
    def test_dataframe_print(self):
        import user_code
        _require_success(
            self,
            hasattr(user_code, "pd"),
            "`pd` is not declared"
        )
        _require_success(
            self,
            user_code.pd == pd,
            "`pd` is declared, but it's not `pandas` library"
        )
        with open("user_code.py", "r") as f:
            user_code_text = f.read()
        _check_success(
            self,
            re.search(r"print *\( *pd *. *DataFrame *\( *tagged_tokens *, *columns *= *\[ *'Token', *'POS tag' *\] *\) *\)", user_code_text),
            "print statement is correct",
            "print statement is either incorrect, or original parts were modified"
        )


if __name__ == '__main__':
    nltk.download('punkt_tab')
    nltk.download('averaged_perceptron_tagger_eng')
    unittest.main()

test_main.py

Udforsk grundlæggende principper inden for Natural Language Processing (NLP) ved at lære essentielle teknikker til tekstforbehandling og metoder til repræsentation af tekstdata. Opnå praktisk erfaring med værktøjer til at rense, analysere og fortolke tekstuelle informationer. Udvikl de nødvendige færdigheder til at omdanne råt sprog til strukturerede indsigter og skab et solidt fundament for avancerede anvendelser inden for kunstig intelligens og maskinlæring.

Dyk ned i de grundlæggende principper for tekstforbehandling for at klargøre rå tekst til analyse. Lær hvordan man opdeler tekst i tokens, filtrerer stopord fra og tilpasser tokenisering med regulære udtryk.

Opdag, hvordan ord kan reduceres til deres grundformer ved hjælp af stamme- og lemmatisering. Behersk ordklassemærkning for at berige tekst med grammatisk kontekst og anvend ordklassebevidst lemmatisering.

Lær, hvordan tekst kan repræsenteres med tal ved hjælp af vektorrumsmodeller. Få praktisk erfaring med at implementere og tilpasse to populære vektorrumsmodeller: bag of words og TF-IDF.

Opnå en solid forståelse af ordindlejringer og hvordan de indfanger semantisk betydning. Udforsk CBoW- og Skip-gram-arkitekturer anvendt i Word2Vec, og implementer dem selv.

Udfordring: Ordklassemærkning

Løsning

Udfordring: Ordklassemærkning

Løsning