import unittest
import importlib
import numpy as np
from sklearn.datasets import load_breast_cancer
from sklearn.model_selection import train_test_split, cross_val_score
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import (
    accuracy_score, precision_score, recall_score, f1_score,
    roc_auc_score, confusion_matrix
)

def _dynamic_test(test_case, condition, success_msg, failure_msg):
    if condition:
        test_case._testMethodName = success_msg
        test_case.assertTrue(True, success_msg)
    else:
        test_case._testMethodName = failure_msg
        test_case.fail(failure_msg)

class TestClassificationMetricsExtended(unittest.TestCase):
    @classmethod
    def setUpClass(cls):
        cls.user = importlib.import_module("user_code")

        data = load_breast_cancer()
        X_train, X_test, y_train, y_test = train_test_split(
            data.data, data.target, test_size=0.3, random_state=42
        )

        model = LogisticRegression(max_iter=1000)
        model.fit(X_train, y_train)
        y_pred = model.predict(X_test)
        y_prob = model.predict_proba(X_test)[:, 1]

        cls.true_metrics = {
            "accuracy": accuracy_score(y_test, y_pred),
            "precision": precision_score(y_test, y_pred),
            "recall": recall_score(y_test, y_pred),
            "f1": f1_score(y_test, y_pred),
            "roc_auc": roc_auc_score(y_test, y_prob),
            "cm": confusion_matrix(y_test, y_pred),
            "mean_cv": cross_val_score(model, data.data, data.target, cv=5).mean(),
        }

    def test_accuracy(self):
        cond = abs(self.user.accuracy - self.true_metrics["accuracy"]) < 1e-6
        _dynamic_test(self, cond, "Accuracy computed correctly.", "Accuracy is incorrect.")

    def test_precision(self):
        cond = abs(self.user.precision - self.true_metrics["precision"]) < 1e-6
        _dynamic_test(self, cond, "Precision computed correctly.", "Precision is incorrect.")

    def test_recall(self):
        cond = abs(self.user.recall - self.true_metrics["recall"]) < 1e-6
        _dynamic_test(self, cond, "Recall computed correctly.", "Recall is incorrect.")

    def test_f1(self):
        cond = abs(self.user.f1 - self.true_metrics["f1"]) < 1e-6
        _dynamic_test(self, cond, "F1 score computed correctly.", "F1 score is incorrect.")

    def test_roc_auc(self):
        cond = abs(self.user.roc_auc - self.true_metrics["roc_auc"]) < 1e-6
        _dynamic_test(self, cond, "ROC–AUC computed correctly.", "ROC–AUC is incorrect.")

    def test_confusion_matrix(self):
        cond = np.array_equal(self.user.cm, self.true_metrics["cm"])
        _dynamic_test(self, cond, "Confusion matrix computed correctly.", "Confusion matrix is incorrect.")

    def test_cross_validation(self):
        cond = abs(self.user.mean_cv - self.true_metrics["mean_cv"]) < 1e-6
        _dynamic_test(self, cond, "Cross-validation mean accuracy computed correctly.", "Cross-validation mean accuracy is incorrect.")

if __name__ == "__main__":
    unittest.main(argv=["first-arg-is-ignored"], exit=False)

test_main.py

A comprehensive course for intermediate learners to master the key evaluation metrics used in machine learning, covering both supervised and unsupervised tasks. The course progresses from foundational concepts to practical Python implementations, interpretation of results, and a comparative overview of metric types and their trade-offs.

Explore the fundamental metrics used to evaluate classification models, including their definitions, formulas, Python implementations, and interpretation. Includes advanced metrics and model validation techniques relevant to classification.

Delve into the core metrics for evaluating regression models, including their mathematical foundations, Python implementations, and result interpretation. Includes advanced validation techniques relevant to regression.

Examine the key metrics for evaluating unsupervised learning models, including clustering, dimensionality reduction, and anomaly detection. Learn their mathematical foundations, Python implementations, and interpretation.

Challenge: Classification Metrics

Solution

Awesome!

Challenge: Classification Metrics

Solution