import unittest
import importlib
import re
from nltk.tokenize import word_tokenize
from nltk.corpus import stopwords
import nltk


# Helper function to require a successful test
def _require_success(test_case, condition, failure_message):
    if not condition:
        test_case._testMethodName = failure_message  # Rename the test with a failure description
        test_case.fail()  # Fail the test with a failure message


# Helper function to check if test is successful
def _check_success(test_case, condition, success_message, failure_message):
    if condition:
        test_case._testMethodName = success_message  # Rename the test for better readability in test reports
    else:
      test_case._testMethodName = failure_message  # Rename the test with a failure description
      test_case.fail()  # Fail the test with a failure message


class TestUserCode(unittest.TestCase):
    # Test that everything imported correctly
    def test_imports(self):
        import user_code
        _require_success(
            self,
            hasattr(user_code, "stopwords"),
            "`stopwords` is not declared"
        )
        _require_success(
            self,
            user_code.stopwords == stopwords,
            "`stopwords` is declared, but it's not `nltk.corpus.stopwords`"
        )
        _require_success(
            self,
            hasattr(user_code, "word_tokenize"),
            "`word_tokenize` is not declared"
        )
        _require_success(
            self,
            user_code.word_tokenize == word_tokenize,
            "`word_tokenize` is declared, but it's not `nltk.tokenize.word_tokenize`"
        )
        _require_success(
            self,
            hasattr(user_code, "nltk"),
            "`nltk` is not declared"
        )
        _require_success(
            self,
            user_code.nltk == nltk,
            "`nltk` is declared, but it's not `nltk` library"
        )
        self._testMethodName = "imports are correct"

    # Test that required NLTK resources are downloaded
    def test_nltk_downloads(self):
        import user_code
        _require_success(
            self,
            hasattr(user_code, "nltk"),
            "`nltk` is not declared"
        )
        _require_success(
            self,
            user_code.nltk == nltk,
            "`nltk` is declared, but it's not `nltk` library"
        )
        with open("user_code.py", "r") as f:
            user_code_text = f.read()
            _require_success(
                self,
                re.search(r"nltk *\. *download *\( *'punkt_tab' *\)", user_code_text),
                "Code for downloading `'punkt_tab'` tokenizer shouldn't be modified"
            )
            _require_success(
                self,
                re.search(r"nltk *\. *download *\( *'stopwords' *\)", user_code_text),
                "Code for downloading `'stopwords'` shouldn't be modified"
            )
        self._testMethodName = "NLTK resources are downloaded"

    # Test that text is declared and unmodified
    def test_text(self):
        import user_code
        expected = """
The Natural Language Toolkit, or NLTK for short, provides a suite of libraries and programs for symbolic and statistical natural language processing in Python. It includes  a diverse set of linguistic data and tools to work with, making it easier for developers and researchers to build 
NLP applications without having to worry about data gathering and preprocessing.
"""
        _require_success(
            self,
            hasattr(user_code, "text"),
            "`text` is not declared"
        )
        _check_success(
            self,
            user_code.text == expected,
            "`text` contains correct value",
            "`text` shouldn't be modified"
        )

    # Test that text_lower is declared and lowercase
    def test_text_lower(self):
        import user_code
        _require_success(
            self,
            hasattr(user_code, "text"),
            "`text` is not declared"
        )
        _require_success(
            self,
            isinstance(user_code.text, str),
            "`text` is not a string"
        )
        _require_success(
            self,
            hasattr(user_code, "text_lower"),
            "`text_lower` is not declared"
        )
        _check_success(
            self,
            user_code.text_lower == user_code.text.lower(),
            "`text_lower` is computed correctly",
            "`text_lower` is not the lowercase of `text`"
        )

    # Test that stop_words variable is declared and correct
    def test_stop_words(self):
        import user_code
        _require_success(
            self,
            hasattr(user_code, "stop_words"),
            "`stop_words` is not declared"
        )
        _require_success(
            self,
            isinstance(user_code.stop_words, set),
            "`stop_words` is not a set"
        )
        _check_success(
            self,
            user_code.stop_words == set(stopwords.words('english')),
            "`stop_words` contains correct English stopwords set",
            "`stop_words` doesn't contain English stopwords"
        )

    # Test that tokens is declared and tokenized correctly
    def test_tokens(self):
        import user_code
        _require_success(
            self,
            hasattr(user_code, "text_lower"),
            "`text_lower` is not declared"
        )
        _require_success(
            self,
            isinstance(user_code.text_lower, str),
            "`text_lower` is not a string"
        )
        _require_success(
            self,
            hasattr(user_code, "tokens"),
            "`tokens` is not declared"
        )
        _check_success(
            self,
            user_code.tokens == word_tokenize(user_code.text_lower),
            "`tokens` is computed correctly",
            "`text_lower` is not correctly tokenized into words"
        )

    # Test that tokens_clean is declared and filtered correctly
    def test_tokens_clean(self):
        import user_code
        _require_success(
            self,
            hasattr(user_code, "tokens"),
            "`tokens` is not declared"
        )
        _require_success(
            self,
            isinstance(user_code.tokens, list),
            "`tokens` is not a list"
        )
        _require_success(
            self,
            hasattr(user_code, "stop_words"),
            "`stop_words` is not declared"
        )
        _require_success(
            self,
            isinstance(user_code.stop_words, set),
            "`stop_words` is not a set"
        )
        _require_success(
            self,
            hasattr(user_code, "tokens_clean"),
            "`tokens_clean` is not declared"
        )
        expected = [tok for tok in user_code.tokens if tok not in user_code.stop_words]
        _check_success(
            self,
            user_code.tokens_clean == expected,
            "`tokens_clean` is computed correctly",
            "stop words in `tokens` are not correctly filtered out"
        )

    # Test that the print statement is present and unmodified
    def test_print(self):
        with open("user_code.py", "r") as f:
            user_code_text = f.read()
            _check_success(
                self,
                re.search(r"print *\( *\\"Filtered Tokens:\\" *, *tokens_clean *\)", user_code_text) is not None,
                "print statement is correct",
                "print statement shouldn't be modified"
            )


if __name__ == '__main__':
    nltk.download('punkt_tab')
    nltk.download('stopwords')
    unittest.main()

test_main.py

Explorons les fondamentaux du Traitement Automatique du Langage Naturel (NLP) en vous plongeant dans les techniques de prétraitement de texte et divers modèles de texte utilisés pour représenter les données textuelles. Vous acquerrez des connaissances pratiques et une expérience pratique avec les outils et méthodes essentiels pour analyser et interpréter efficacement les données textuelles. Ce cours vous dote des compétences nécessaires pour transformer le texte brut en informations significatives, ouvrant la voie à des applications avancées en IA et apprentissage automatique.

Nous commencerons notre voyage en apprenant et en mettant en œuvre les techniques de prétraitement de texte les plus courantes utilisées en NLP pour convertir le texte brut initial en une forme propre et standardisée.

Sans plus tarder, explorons la racine et la lemmatisation. Ces techniques peuvent améliorer l'efficacité et l'efficience de certaines tâches de NLP, surtout lorsqu'on travaille avec de grands corpus de texte et qu'on traite différentes formes de mots comme le même mot.

Le texte prétraité doit ensuite être transformé en une représentation numérique pour être utilisé dans des modèles d'apprentissage automatique ou d'apprentissage profond pour diverses tâches telles que la prédiction, la classification ou le regroupement. Ici, nous apprendrons à implémenter les modèles de texte les plus basiques mais populaires qui transforment les données textuelles en nombres.

Il est temps de libérer la puissance des incorporations de mots et de plonger dans des techniques avancées pour capturer les relations sémantiques entre les mots. Nous explorerons divers modèles d'incorporation tels que Word2Vec, GloVe et FastText, avec un accent particulier sur le modèle Word2Vec et son implémentation.

Challenge: Stop Words

Solution

Awesome!

Challenge: Stop Words

Solution

Awesome!