import unittest
import re
import nltk
from sklearn.feature_extraction.text import CountVectorizer
import pandas as pd
from scipy.sparse import csr_matrix


# Helper functions
def _require_success(test_case, condition, failure_message):
    if not condition:
        test_case._testMethodName = failure_message
        test_case.fail()

def _check_success(test_case, condition, success_message, failure_message):
    if condition:
        test_case._testMethodName = success_message
    else:
        test_case._testMethodName = failure_message
        test_case.fail()

class TestUserCode(unittest.TestCase):
    # Test that everything is imported correctly
    def test_imports(self):
        import user_code
        _require_success(
            self, 
            hasattr(user_code, "pd"), 
            "`pd` is not declared"
        )
        _require_success(
            self,
            user_code.pd == pd,
            "`pd` is declared, but it's not `pandas` library"
        )
        _require_success(
            self, 
            hasattr(user_code, "CountVectorizer"), 
            "`CountVectorizer` is not declared"
        )
        _require_success(
            self,
            user_code.CountVectorizer == CountVectorizer,
            "`CountVectorizer` is declared, but it's not `sklearn.feature_extraction.text.CountVectorizer`"
        )
        self._testMethodName = "imports are correct"

    # Test original corpus
    def test_corpus(self):
        import user_code
        expected = pd.read_csv(
    'https://staging-content-media-cdn.codefinity.com/courses/c68c1f2e-2c90-4d5d-8db9-1e97ca89d15e/section_3/chapter_4/example_corpus.csv')
        _require_success(
            self, 
            hasattr(user_code, "corpus"), 
            "`corpus` is not declared"
        )
        _check_success(
            self,
            expected.equals(user_code.corpus),
            "`corpus` contains correct value",
            "`corpus` shouldn't be modified"
        )

    # Test that count_vectorizer is declared correctly
    def test_count_vectorizer(self):
        import user_code
        _require_success(
            self, 
            hasattr(user_code, "count_vectorizer"), 
            "`count_vectorizer` is not declared"
        )
        _require_success(
            self, 
            isinstance(user_code.count_vectorizer, CountVectorizer), 
            "`count_vectorizer` is not an instance of `CountVectorizer`"
        )
        _require_success(
            self, 
            user_code.count_vectorizer.binary == False, 
            "`count_vectorizer` should be a frequency-based BoW"
        )
        _require_success(
            self, 
            user_code.count_vectorizer.ngram_range == (1, 2), 
            "`count_vectorizer` should work with and only with unigrams and bigrams"
        )
        self._testMethodName = "`count_vectorizer` is defined correctly"

    # Test bow_matrix computation
    def test_bow_matrix(self):
        import user_code
        _require_success(
            self, 
            hasattr(user_code, "corpus"), 
            "`corpus` is not declared"
        )
        _require_success(
            self, 
            isinstance(user_code.corpus, pd.DataFrame), 
            "`corpus` is not a DataFrame"
        )
        _require_success(
            self, 
            "Document" in user_code.corpus.columns, 
            "`corpus` doesn't contain `'Document'` column"
        )
        _require_success(
            self, 
            hasattr(user_code, "bow_matrix"), 
            "`bow_matrix` is not declared"
        )
        count_vectorizer = CountVectorizer(ngram_range=(1, 2))
        bow_matrix = count_vectorizer.fit_transform(user_code.corpus['Document'])
        _check_success(
            self,
            user_code.bow_matrix.shape == bow_matrix.shape and (user_code.bow_matrix - bow_matrix).nnz == 0,
            "`bow_matrix` is computed correctly",
            "`bow_matrix` is not computed correctly"
        )

    # Test bow_df conversion
    def test_bow_df(self):
        import user_code
        _require_success(
            self, 
            hasattr(user_code, "count_vectorizer"), 
            "`count_vectorizer` is not declared"
        )
        _require_success(
            self, 
            isinstance(user_code.count_vectorizer, CountVectorizer), 
            "`count_vectorizer` is not an instance of `CountVectorizer`"
        )
        _require_success(
            self, 
            hasattr(user_code, "bow_matrix"), 
            "`bow_matrix` is not declared"
        )
        _require_success(
            self, 
            isinstance(user_code.bow_matrix, csr_matrix), 
            "`bow_matrix` is not a `csr_matrix`"
        )
        _require_success(
            self, 
            hasattr(user_code, "bow_df"), 
            "`bow_df` is not declared"
        )
        bow_df = pd.DataFrame(user_code.bow_matrix.toarray(), columns=user_code.count_vectorizer.get_feature_names_out())
        _check_success(
            self,
            bow_df.equals(user_code.bow_df),
            "`bow_df` is computed correctly",
            "`bow_df` is not computed correctly"
        )

    # Test print statement
    def test_print(self):
        with open("user_code.py", "r") as f:
            user_code_text = f.read()
        _check_success(
            self,
            re.search(r"print *\( *bow_df *\[ *(?P<quote>[\\"'])graphic design(?P=quote) *\] *\. *values *\)", user_code_text),
            "print statement is correct",
            "print statement is not correct"
        )


if __name__ == '__main__':
    unittest.main()

test_main.py

Utforsk grunnleggende prinsipper innen Natural Language Processing (NLP) ved å lære essensielle teknikker for tekstforbehandling og metoder for å representere tekstdata. Få praktisk erfaring med verktøyene som brukes til å rense, analysere og tolke tekstinformasjon. Utvikle ferdigheter som trengs for å omforme råspråk til strukturerte innsikter, og legg et solid grunnlag for avanserte anvendelser innen kunstig intelligens og maskinlæring.

Utforsk det grunnleggende innen tekstforbehandling for å klargjøre råtekst for analyse. Lær hvordan man deler opp tekst i token, filtrerer ut stoppord og tilpasser tokenisering med regulære uttrykk.

Utforsk hvordan ord kan reduseres til sine grunnformer ved hjelp av stamming og lemmatisering. Behersk ordklassemerking for å berike tekst med grammatisk kontekst og anvend ordklassebevisst lemmatisering.

Lær hvordan tekst kan representeres med tall ved hjelp av vektorrommodeller. Få praktisk erfaring ved å implementere og tilpasse to populære vektorrommodeller: bag of words og TF-IDF.

Få en grundig forståelse av ordinnbygginger og hvordan de fanger opp semantisk betydning. Utforsk CBoW- og Skip-gram-arkitekturer brukt i Word2Vec, og implementer dem selv.

Utfordring: Bag of Words

Løsning

Awesome!

Utfordring: Bag of Words

Løsning