import unittest
import user_code as uc


def _dynamic_test(test_case, condition, success_message, failure_message):
    if condition:
        test_case._testMethodName = success_message
        test_case.assertTrue(True)
    else:
        test_case._testMethodName = failure_message
        test_case.fail(failure_message)


class TestMLPrep(unittest.TestCase):

    def test_ml_count(self):
        _dynamic_test(self,
            isinstance(uc.ml_count, int) and uc.ml_count > 0,
            "The `ml_count` is a positive integer.",
            "Expected `ml_count` to be a positive integer.")

    def test_label_column_exists(self):
        _dynamic_test(self,
            "LABEL" in uc.ml_df.columns,
            "The `ml_df` contains the `LABEL` column.",
            "Expected `ml_df` to contain the `LABEL` column.")

    def test_features_column_exists(self):
        _dynamic_test(self,
            "FEATURES" in uc.ml_df.columns,
            "The `ml_df` contains the `FEATURES` column.",
            "Expected `ml_df` to contain the `FEATURES` column.")

    def test_label_is_binary(self):
        from pyspark.sql.functions import col
        invalid = uc.ml_df.filter(~col("LABEL").isin(0, 1)).count()
        _dynamic_test(self,
            invalid == 0,
            "The `LABEL` column contains only 0 and 1.",
            f"Expected `LABEL` to be binary, but found {invalid} invalid values.")

    def test_is_weekend_column_exists(self):
        _dynamic_test(self,
            "IS_WEEKEND" in uc.ml_df.columns,
            "The `ml_df` contains the `IS_WEEKEND` column.",
            "Expected `ml_df` to contain the `IS_WEEKEND` column.")


if __name__ == "__main__":
    unittest.main()

test_main.py

Master the essential techniques of feature engineering using PySpark, from handling categorical and numerical data to building robust machine learning pipelines. This course guides you through practical methods for transforming raw data into meaningful features, preparing datasets for advanced analytics and machine learning workflows.

Explore the core concepts and practical techniques of feature engineering in PySpark, including handling different data types, building feature pipelines, and preparing datasets for machine learning.

Challenge: Preparing a Dataset for Machine Learning

Solution