Rather than manually selecting specific values for our model's hyperparameters, **randomized search** (`RandomizedSearchCV`) offers a more efficient way to find an optimal configuration. Unlike **grid search** (`GridSearchCV`), which systematically evaluates all possible combinations of hyperparameters, randomized search selects a **random subset** of these combinations. This approach significantly reduces computational cost while still yielding strong results.

For neural networks, where the number of possible hyperparameter combinations can be immense, exhaustively testing every option is often **impractical**. Randomized search circumvents this issue by **randomly sampling a defined number of hyperparameter sets**, balancing exploration and efficiency.

```python
RandomizedSearchCV(
    estimator=model, 
    param_distributions=randomized_parameters, 
    n_iter=number_of_models_to_test,  # Number of random combinations to evaluate
    scoring='accuracy',  # Evaluation metric
    random_state=42,  # Ensures reproducibility
)
```

- **`estimator`**: the model to optimize (e.g., `MLPClassifier`);
- **`param_distributions`**: a dictionary where keys are hyperparameter names and values are lists which to sample;
- **`n_iter`**: specifies how many random combinations should be tested. A higher value increases the chances of finding an optimal combination but requires more computation;
- **`scoring`**: defines the evaluation metric (e.g., `'accuracy'` for classification).

import unittest
import importlib
import numpy as np


def _dynamic_test(test_case, condition, success_msg, failure_msg):
    if condition:
        test_case._testMethodName = success_msg
        test_case.assertTrue(True, success_msg)
    else:
        test_case._testMethodName = failure_msg
        test_case.fail(failure_msg)


class TestRandomizedSearchCV(unittest.TestCase):

    @classmethod
    def setUpClass(cls):
        cls.user_code = importlib.import_module("user_code")
        from sklearn.neural_network import MLPClassifier
        from sklearn.model_selection import RandomizedSearchCV
        from sklearn.metrics import accuracy_score
        from data import X_train, y_train, X_test, y_test

        cls.MLPClassifier = MLPClassifier
        cls.RandomizedSearchCV = RandomizedSearchCV
        cls.accuracy_score = accuracy_score
        cls.X_train = X_train
        cls.y_train = y_train
        cls.X_test = X_test
        cls.y_test = y_test

    def test_param_distributions_defined(self):
        params = getattr(self.user_code, "param_distributions", None)
        _dynamic_test(
            self,
            isinstance(params, dict),
            "Parameter grid defined as dictionary",
            "param_distributions not found or not a dictionary",
        )

        if params:
            correct_keys = all(k in params for k in ["hidden_layer_sizes", "learning_rate_init", "max_iter"])
            _dynamic_test(
                self,
                correct_keys,
                "Parameter grid contains all required keys",
                f"Missing keys in param_distributions: {list(params.keys())}",
            )

            correct_layers = params.get("hidden_layer_sizes") == [(20, 20), (25, 25), (30, 30)]
            correct_lr = params.get("learning_rate_init") == [0.02, 0.01, 0.005]
            correct_iters = params.get("max_iter") == [10, 30, 50]

            _dynamic_test(
                self,
                correct_layers and correct_lr and correct_iters,
                "Parameter grid values are correct",
                f"Incorrect values in param_distributions: {params}",
            )

    def test_randomized_search_instance(self):
        rs = getattr(self.user_code, "random_search", None)
        _dynamic_test(
            self,
            rs is not None,
            "RandomizedSearchCV instance created",
            "random_search variable not found",
        )

        if rs is not None:
            correct_estimator = isinstance(rs.estimator, self.MLPClassifier)
            correct_n_iter = getattr(rs, "n_iter", None) == 4
            correct_scoring = getattr(rs, "scoring", None) == "accuracy"

            _dynamic_test(
                self,
                correct_estimator and correct_n_iter and correct_scoring,
                "RandomizedSearchCV configured correctly",
                f"Incorrect RandomizedSearchCV configuration: n_iter={getattr(rs, 'n_iter', None)}, "
                f"scoring={getattr(rs, 'scoring', None)}",
            )

    def test_best_params_exist(self):
        rs = getattr(self.user_code, "random_search", None)
        _dynamic_test(
            self,
            hasattr(rs, "best_params_"),
            "Best parameters found after RandomizedSearchCV fit",
            "best_params_ attribute missing — did you call random_search.fit()?",
        )

    def test_accuracy_values(self):
        train_acc = getattr(self.user_code, "train_accuracy", None)
        test_acc = getattr(self.user_code, "test_accuracy", None)

        _dynamic_test(
            self,
            train_acc is not None and test_acc is not None,
            "Train and test accuracies computed",
            "train_accuracy or test_accuracy missing",
        )

        if train_acc is not None and test_acc is not None:
            _dynamic_test(
                self,
                0.5 <= test_acc <= 1.0 and 0.5 <= train_acc <= 1.0,
                "Accuracy values are within realistic range (0.5–1.0)",
                f"Unrealistic accuracy values: train={train_acc}, test={test_acc}",
            )


if __name__ == "__main__":
    unittest.main(argv=["first-arg-is-ignored"], exit=False)


test_code.py

Neural networks are powerful algorithms inspired by the structure of the human brain that are used to solve complex machine learning problems. You will build your own Neural Network from scratch to understand how it works. After this course, you will be able to create neural networks for solving classification and regression problems using the scikit-learn library.

First, we will discuss what a neural network is and how it works. And also consider the scope of its application.

Next, we will try to build our own neural network and see how efficiently it copes with learning. We will also consider a ready-made solution from the scikit-learn library.

Finally, we will give you some additional useful information on how to understand which model to use and what types of neural networks there are. To complete the course, you will be tested on your acquired knowledge.

Challenge: Automatic Hyperparameter Tuning

解答