import unittest
import re
import nltk
from gensim.models import Word2Vec
import pandas as pd
from scipy.sparse import csr_matrix


# Helper functions
def _require_success(test_case, condition, failure_message):
    if not condition:
        test_case._testMethodName = failure_message
        test_case.fail()

def _check_success(test_case, condition, success_message, failure_message):
    if condition:
        test_case._testMethodName = success_message
    else:
        test_case._testMethodName = failure_message
        test_case.fail()

class TestUserCode(unittest.TestCase):
    # Test that everything is imported correctly
    def test_imports(self):
        import user_code
        _require_success(
            self, 
            hasattr(user_code, "pd"), 
            "`pd` is not declared"
        )
        _require_success(
            self,
            user_code.pd == pd,
            "`pd` is declared, but it's not `pandas` library"
        )
        _require_success(
            self, 
            hasattr(user_code, "Word2Vec"), 
            "`Word2Vec` is not declared"
        )
        _require_success(
            self,
            user_code.Word2Vec == Word2Vec,
            "`Word2Vec` is declared, but it's not `gensim.models.Word2Vec`"
        )
        self._testMethodName = "imports are correct"

    # Test original corpus
    def test_corpus(self):
        import user_code
        expected = pd.read_csv(
    'https://staging-content-media-cdn.codefinity.com/courses/c68c1f2e-2c90-4d5d-8db9-1e97ca89d15e/section_3/chapter_4/example_corpus.csv')
        _require_success(
            self, 
            hasattr(user_code, "corpus"), 
            "`corpus` is not declared"
        )
        _check_success(
            self,
            expected.equals(user_code.corpus),
            "`corpus` contains correct value",
            "`corpus` shouldn't be modified"
        )

    # Test that sentences are correctly computed
    def test_sentences(self):
        import user_code
        _require_success(
            self, 
            hasattr(user_code, "corpus"), 
            "`corpus` is not declared"
        )
        _require_success(
            self, 
            isinstance(user_code.corpus, pd.DataFrame), 
            "`corpus` is not a DataFrame"
        )
        _require_success(
            self, 
            "Document" in user_code.corpus.columns, 
            "`corpus` doesn't contain `'Document'` column"
        )
        _require_success(
            self, 
            hasattr(user_code, "sentences"), 
            "`sentences` is not declared"
        )
        sentences = user_code.corpus['Document'].str.split()
        _check_success(
            self,
            sentences.equals(user_code.sentences),
            "`sentences` is computed correctly",
            "`sentences` is not computed correctly"
        )
  
    # Test that model is declared correctly
    def test_model(self):
        import user_code
        _require_success(
            self, 
            hasattr(user_code, "sentences"), 
            "`sentences` is not declared"
        )
        _require_success(
            self, 
            isinstance(user_code.sentences, pd.Series), 
            "`sentences` is not a `Series` object"
        )
        model = Word2Vec(user_code.sentences, vector_size=50, window=2, min_count=1, sg=1)
        _require_success(
            self, 
            hasattr(user_code, "model"), 
            "`model` is not declared"
        )
        _require_success(
            self, 
            isinstance(user_code.model, Word2Vec), 
            "`model` is not an instance of `Word2Vec`"
        )
        _require_success(
            self, 
            user_code.model.corpus_count == model.corpus_count and user_code.model.corpus_total_words == model.corpus_total_words, 
            "`model` should be initialized with `sentences`"
        )
        _require_success(
            self, 
            user_code.model.vector_size == model.vector_size, 
            "`model` should be initialized with `vector_size=50`"
        )
        _require_success(
            self, 
            user_code.model.window == model.window, 
            "`model` should be initialized with `window=2`"
        )
        _require_success(
            self, 
            user_code.model.min_count == model.min_count, 
            "`model` should be initialized with `min_count=1`"
        )
        _require_success(
            self, 
            user_code.model.sg == model.sg, 
            "`model` should be initialized with `sg=1`"
        )
        self._testMethodName = "`model` is defined correctly"

    # Test print statement
    def test_print(self):
        with open("user_code.py", "r") as f:
            user_code_text = f.read()
        _check_success(
            self,
            re.search(r"print *\( *model *\. *wv *\. *most_similar *\( *(?P<quote>[\\"'])bowl(?P=quote) *, *topn *= *3 *\) *\)", user_code_text),
            "print statement is correct",
            "print statement is not correct"
        )


if __name__ == '__main__':
    unittest.main()

test_main.py

Tutustu luonnollisen kielen käsittelyn (NLP) perusteisiin oppimalla keskeiset tekstin esikäsittelytekniikat ja menetelmät tekstiaineiston esittämiseen. Saat käytännön kokemusta työkaluista, joilla puhdistetaan, analysoidaan ja tulkitaan tekstimuotoista tietoa. Kehitä taidot, joilla muutat raakakielen jäsennellyiksi havainnoiksi ja luot vahvan pohjan tekoälyn ja koneoppimisen edistyneille sovelluksille.

Perehdy tekstin esikäsittelyn perusteisiin raakatekstin valmistelussa analyysia varten. Opiskele tekstin tokenisointia, stop-sanojen suodatusta sekä tokenisoinnin mukauttamista säännöllisten lausekkeiden avulla.

Opi, miten sanoja voidaan palauttaa perusmuotoonsa stemmauksen ja lemmatisaation avulla. Hallitse sanaluokkien tunnistus tekstin kieliopillisen kontekstin rikastamiseksi ja sovella sanaluokkatietoista lemmatisaatiota.

Opi, miten teksti voidaan esittää numeroina vektoriavaruusmallien avulla. Saat käytännön kokemusta toteuttamalla ja mukauttamalla kahta suosittua vektoriavaruusmallia: sanapussimalli ja TF-IDF.

Hanki vankka ymmärrys sanaesityksistä ja siitä, miten ne tallentavat semanttista merkitystä. Tutustu Word2Vecin CBoW- ja Skip-gram-arkkitehtuureihin ja toteuta ne itse.

Haaste: Sanaupotusten Luominen

Ratkaisu

Haaste: Sanaupotusten Luominen

Ratkaisu