From c1f66e55c75baec038a4d03df77b2d0c65932d5e Mon Sep 17 00:00:00 2001 From: Simon Breuer <86068340+sibre28@users.noreply.github.com> Date: Mon, 15 Jul 2024 16:18:37 +0200 Subject: [PATCH] feat: hyperparameter optimization for fnn models (#897) Closes #861 ### Summary of Changes Added fit_by_exhaustive_search method to NNClassifier, NNRegressor Added FittingWithChoice and FittingWithoutChoice Error Added Choice options for neuron_count params of all layers Added contains_choices method to all layers Added _get_layers_for_all_choices method to all layers Added ClassifierMetric and RegressorMetric as Enums fit_by_exhaustive_search currently only works for tables as input, images and timeseries will be added in a later pr --------- Co-authored-by: megalinter-bot <129584137+megalinter-bot@users.noreply.github.com> --- src/safeds/exceptions/__init__.py | 8 +- src/safeds/exceptions/_ml.py | 27 + src/safeds/ml/hyperparameters/_choice.py | 14 +- src/safeds/ml/nn/_model.py | 306 ++++- .../ml/nn/layers/_convolutional2d_layer.py | 6 + src/safeds/ml/nn/layers/_dropout_layer.py | 6 + src/safeds/ml/nn/layers/_flatten_layer.py | 6 + src/safeds/ml/nn/layers/_forward_layer.py | 24 +- src/safeds/ml/nn/layers/_gru_layer.py | 26 +- src/safeds/ml/nn/layers/_layer.py | 13 +- src/safeds/ml/nn/layers/_lstm_layer.py | 26 +- src/safeds/ml/nn/layers/_pooling2d_layer.py | 6 + .../safeds/ml/hyperparameters/test_choice.py | 37 +- .../safeds/ml/nn/layers/test_forward_layer.py | 39 +- tests/safeds/ml/nn/layers/test_gru_layer.py | 53 +- tests/safeds/ml/nn/layers/test_lstm_layer.py | 53 +- tests/safeds/ml/nn/test_model.py | 1120 ++++++++++------- 17 files changed, 1237 insertions(+), 533 deletions(-) diff --git a/src/safeds/exceptions/__init__.py b/src/safeds/exceptions/__init__.py index b885a746e..802afdb57 100644 --- a/src/safeds/exceptions/__init__.py +++ b/src/safeds/exceptions/__init__.py @@ -17,7 +17,10 @@ from ._ml import ( DatasetMissesDataError, DatasetMissesFeaturesError, + EmptyChoiceError, FeatureDataMismatchError, + FittingWithChoiceError, + FittingWithoutChoiceError, InputSizeError, InvalidFitDataError, InvalidModelStructureError, @@ -72,8 +75,10 @@ class OutOfBoundsError(SafeDsError): # ML exceptions "DatasetMissesDataError", "DatasetMissesFeaturesError", - "TargetDataMismatchError", + "EmptyChoiceError", "FeatureDataMismatchError", + "FittingWithChoiceError", + "FittingWithoutChoiceError", "InvalidFitDataError", "InputSizeError", "InvalidModelStructureError", @@ -81,4 +86,5 @@ class OutOfBoundsError(SafeDsError): "ModelNotFittedError", "PlainTableError", "PredictionError", + "TargetDataMismatchError", ] diff --git a/src/safeds/exceptions/_ml.py b/src/safeds/exceptions/_ml.py index b7600df34..4f0462188 100644 --- a/src/safeds/exceptions/_ml.py +++ b/src/safeds/exceptions/_ml.py @@ -42,6 +42,33 @@ def __init__(self) -> None: super().__init__("Dataset contains no rows") +class EmptyChoiceError(ValueError): + """Raised when a choice object is created, but no arguments are provided.""" + + def __init__(self) -> None: + super().__init__("Please provide at least one Value in a Choice Parameter") + + +class FittingWithChoiceError(Exception): + """Raised when a model is fitted with a choice object as a parameter.""" + + def __init__(self) -> None: + super().__init__( + "Error occurred while fitting: Trying to fit with a Choice Parameter. Please use " + "fit_by_exhaustive_search() instead.", + ) + + +class FittingWithoutChoiceError(Exception): + """Raised when a model is fitted by exhaustive search without a choice object as a parameter.""" + + def __init__(self) -> None: + super().__init__( + "Error occurred while fitting: Trying to fit by exhaustive search without a Choice " + "Parameter. Please use fit() instead.", + ) + + class InvalidFitDataError(Exception): """Raised when a Neural Network is fitted on invalid data.""" diff --git a/src/safeds/ml/hyperparameters/_choice.py b/src/safeds/ml/hyperparameters/_choice.py index 6d0f59db2..09530ded2 100644 --- a/src/safeds/ml/hyperparameters/_choice.py +++ b/src/safeds/ml/hyperparameters/_choice.py @@ -3,6 +3,8 @@ from collections.abc import Collection from typing import TYPE_CHECKING, TypeVar +from safeds.exceptions import EmptyChoiceError + if TYPE_CHECKING: from collections.abc import Iterator from typing import Any @@ -15,7 +17,7 @@ class Choice(Collection[T]): def __init__(self, *args: T) -> None: """ - Create a new choice. + Create a new choice. Duplicate values will be removed. Parameters ---------- @@ -23,6 +25,9 @@ def __init__(self, *args: T) -> None: The values to choose from. """ self.elements = list(args) + if len(args) < 1: + raise EmptyChoiceError + self.elements = list(dict.fromkeys(args)) def __contains__(self, value: Any) -> bool: """ @@ -61,3 +66,10 @@ def __len__(self) -> int: The number of values in this choice. """ return len(self.elements) + + def __eq__(self, other: object) -> bool: + if not isinstance(other, Choice): + return NotImplemented + if self is other: + return True + return (self is other) or (self.elements == other.elements) diff --git a/src/safeds/ml/nn/_model.py b/src/safeds/ml/nn/_model.py index 3d9086d61..57e9d10a6 100644 --- a/src/safeds/ml/nn/_model.py +++ b/src/safeds/ml/nn/_model.py @@ -1,7 +1,9 @@ from __future__ import annotations import copy -from typing import TYPE_CHECKING, Generic, Self, TypeVar +import multiprocessing as mp +from concurrent.futures import ALL_COMPLETED, ProcessPoolExecutor, wait +from typing import TYPE_CHECKING, Any, Generic, Literal, Self, TypeVar from safeds._config import _init_default_device from safeds._validation import _check_bounds, _ClosedBound @@ -12,9 +14,13 @@ from safeds.data.tabular.transformation import OneHotEncoder from safeds.exceptions import ( FeatureDataMismatchError, + FittingWithChoiceError, + FittingWithoutChoiceError, InvalidModelStructureError, + LearningError, ModelNotFittedError, ) +from safeds.ml.metrics import ClassificationMetrics, RegressionMetrics from safeds.ml.nn.converters import ( InputConversionImageToColumn, InputConversionImageToImage, @@ -210,6 +216,9 @@ def fit( _init_default_device() + if self._contains_choices(): + raise FittingWithChoiceError + if not self._input_conversion._is_fit_data_valid(train_data): raise FeatureDataMismatchError @@ -258,6 +267,149 @@ def fit( copied_model._model.eval() return copied_model + def fit_by_exhaustive_search( + self, + train_data: IFT, + optimization_metric: Literal[ + "mean_squared_error", + "mean_absolute_error", + "median_absolute_deviation", + "coefficient_of_determination", + ], + epoch_size: int = 25, + batch_size: int = 1, + learning_rate: float = 0.001, + ) -> Self: + """ + Use the hyperparameter choices to create multiple models and fit them. + + **Note:** This model is not modified. + + Parameters + ---------- + train_data: + The data the network should be trained on. + optimization_metric: + The metric that should be used for determining the performance of a model. + epoch_size: + The number of times the training cycle should be done. + batch_size: + The size of data batches that should be loaded at one time. + learning_rate: + The learning rate of the neural network. + + Returns + ------- + best_model: + The model that performed the best out of all possible models given the Choices of hyperparameters. + + Raises + ------ + FittingWithoutChoiceError + When calling this method on a model without hyperparameter choices. + LearningError + If the training data contains invalid values or if the training failed. Currently raised, when calling this on RNNs or CNNs as well. + """ + _init_default_device() + + if not self._contains_choices(): + raise FittingWithoutChoiceError + + if isinstance(train_data, TimeSeriesDataset): + raise LearningError("RNN-Hyperparameter optimization is currently not supported.") # pragma: no cover + if isinstance(train_data, ImageDataset): + raise LearningError("CNN-Hyperparameter optimization is currently not supported.") # pragma: no cover + + _check_bounds("epoch_size", epoch_size, lower_bound=_ClosedBound(1)) + _check_bounds("batch_size", batch_size, lower_bound=_ClosedBound(1)) + + list_of_models = self._get_models_for_all_choices() + list_of_fitted_models: list[Self] = [] + + with ProcessPoolExecutor(max_workers=len(list_of_models), mp_context=mp.get_context("spawn")) as executor: + futures = [] + for model in list_of_models: + futures.append(executor.submit(model.fit, train_data, epoch_size, batch_size, learning_rate)) + [done, _] = wait(futures, return_when=ALL_COMPLETED) + for future in done: + list_of_fitted_models.append(future.result()) + executor.shutdown() + + # Cross Validation + [train_split, test_split] = train_data.to_table().split_rows(0.75) + train_data = train_split.to_tabular_dataset( + target_name=train_data.target.name, + extra_names=train_data.extras.column_names, + ) + test_data = test_split.to_tabular_dataset( + target_name=train_data.target.name, + extra_names=train_data.extras.column_names, + ).features + target_col = train_data.target + + best_model = None + best_metric_value = None + for fitted_model in list_of_fitted_models: + if best_model is None: + best_model = fitted_model + match optimization_metric: + case "mean_squared_error": + best_metric_value = RegressionMetrics.mean_squared_error(predicted=fitted_model.predict(test_data), expected=target_col) # type: ignore[arg-type] + case "mean_absolute_error": + best_metric_value = RegressionMetrics.mean_absolute_error(predicted=fitted_model.predict(test_data), expected=target_col) # type: ignore[arg-type] + case "median_absolute_deviation": + best_metric_value = RegressionMetrics.median_absolute_deviation(predicted=fitted_model.predict(test_data), expected=target_col) # type: ignore[arg-type] + case "coefficient_of_determination": + best_metric_value = RegressionMetrics.coefficient_of_determination(predicted=fitted_model.predict(test_data), expected=target_col) # type: ignore[arg-type] + else: + match optimization_metric: + case "mean_squared_error": + error_of_fitted_model = RegressionMetrics.mean_squared_error(predicted=fitted_model.predict(test_data), expected=target_col) # type: ignore[arg-type] + if error_of_fitted_model < best_metric_value: + best_model = fitted_model # pragma: no cover + best_metric_value = error_of_fitted_model # pragma: no cover + case "mean_absolute_error": + error_of_fitted_model = RegressionMetrics.mean_absolute_error(predicted=fitted_model.predict(test_data), expected=target_col) # type: ignore[arg-type] + if error_of_fitted_model < best_metric_value: + best_model = fitted_model # pragma: no cover + best_metric_value = error_of_fitted_model # pragma: no cover + case "median_absolute_deviation": + error_of_fitted_model = RegressionMetrics.median_absolute_deviation(predicted=fitted_model.predict(test_data), expected=target_col) # type: ignore[arg-type] + if error_of_fitted_model < best_metric_value: + best_model = fitted_model # pragma: no cover + best_metric_value = error_of_fitted_model # pragma: no cover + case "coefficient_of_determination": + error_of_fitted_model = RegressionMetrics.coefficient_of_determination(predicted=fitted_model.predict(test_data), expected=target_col) # type: ignore[arg-type] + if error_of_fitted_model > best_metric_value: + best_model = fitted_model # pragma: no cover + best_metric_value = error_of_fitted_model # pragma: no cover + assert best_model is not None # just for linter + best_model._is_fitted = True + return best_model + + def _get_models_for_all_choices(self) -> list[Self]: + + all_possible_layer_combinations: list[list] = [[]] + for layer in self._layers: + if not layer._contains_choices(): + for item in all_possible_layer_combinations: + item.append(layer) + else: + updated_combinations = [] + versions_of_one_layer = layer._get_layers_for_all_choices() + for version in versions_of_one_layer: + copy_of_all_current_possible_combinations = copy.deepcopy(all_possible_layer_combinations) + for combination in copy_of_all_current_possible_combinations: + combination.append(version) + updated_combinations.append(combination) + all_possible_layer_combinations = updated_combinations + + models = [] + for combination in all_possible_layer_combinations: + new_model = NeuralNetworkRegressor(input_conversion=self._input_conversion, layers=combination) + models.append(new_model) + return models # type: ignore[return-value] + def predict(self, test_data: IPT) -> IFT: """ Make a prediction for the given test data. @@ -313,6 +465,10 @@ def input_size(self) -> int | ModelImageSize | None: # TODO: raise if not fitted, don't return None return self._input_size + def _contains_choices(self) -> bool: + """Whether the model contains choices in any layer.""" + return any(layer._contains_choices() for layer in self._layers) + class NeuralNetworkClassifier(Generic[IFT, IPT]): """ @@ -505,6 +661,9 @@ def fit( _init_default_device() + if self._contains_choices(): + raise FittingWithChoiceError + if not self._input_conversion._is_fit_data_valid(train_data): raise FeatureDataMismatchError @@ -562,6 +721,147 @@ def fit( copied_model._model.eval() return copied_model + def fit_by_exhaustive_search( + self, + train_data: IFT, + optimization_metric: Literal["accuracy", "precision", "recall", "f1_score"], + positive_class: Any = None, + epoch_size: int = 25, + batch_size: int = 1, + learning_rate: float = 0.001, + ) -> Self: + """ + Use the hyperparameter choices to create multiple models and fit them. + + **Note:** This model is not modified. + + Parameters + ---------- + train_data: + The data the network should be trained on. + optimization_metric: + The metric that should be used for determining the performance of a model. + positive_class: + The class to be considered positive. Only needs to be provided when choosing precision, recall or f1_score as the optimization metric. + epoch_size: + The number of times the training cycle should be done. + batch_size: + The size of data batches that should be loaded at one time. + learning_rate: + The learning rate of the neural network. + + Returns + ------- + best_model: + The model that performed the best out of all possible models given the Choices of hyperparameters. + + Raises + ------ + FittingWithoutChoiceError + When calling this method on a model without hyperparameter choices. + LearningError + If the training data contains invalid values or if the training failed. Currently raised, when calling this on RNNs or CNNs as well. + """ + _init_default_device() + + if not self._contains_choices(): + raise FittingWithoutChoiceError + + if isinstance(train_data, TimeSeriesDataset): + raise LearningError("RNN-Hyperparameter optimization is currently not supported.") # pragma: no cover + if isinstance(train_data, ImageDataset): + raise LearningError("CNN-Hyperparameter optimization is currently not supported.") # pragma: no cover + + _check_bounds("epoch_size", epoch_size, lower_bound=_ClosedBound(1)) + _check_bounds("batch_size", batch_size, lower_bound=_ClosedBound(1)) + + list_of_models = self._get_models_for_all_choices() + list_of_fitted_models: list[Self] = [] + + with ProcessPoolExecutor(max_workers=len(list_of_models), mp_context=mp.get_context("spawn")) as executor: + futures = [] + for model in list_of_models: + futures.append(executor.submit(model.fit, train_data, epoch_size, batch_size, learning_rate)) + [done, _] = wait(futures, return_when=ALL_COMPLETED) + for future in done: + list_of_fitted_models.append(future.result()) + executor.shutdown() + + # Cross Validation + [train_split, test_split] = train_data.to_table().split_rows(0.75) + train_data = train_split.to_tabular_dataset( + target_name=train_data.target.name, + extra_names=train_data.extras.column_names, + ) + test_data = test_split.to_tabular_dataset( + target_name=train_data.target.name, + extra_names=train_data.extras.column_names, + ).features + target_col = train_data.target + + best_model = None + best_metric_value = None + for fitted_model in list_of_fitted_models: + if best_model is None: + best_model = fitted_model + match optimization_metric: + case "accuracy": + best_metric_value = ClassificationMetrics.accuracy(predicted=fitted_model.predict(test_data), expected=target_col) # type: ignore[arg-type] + case "precision": + best_metric_value = ClassificationMetrics.precision(predicted=fitted_model.predict(test_data), expected=target_col, positive_class=positive_class) # type: ignore[arg-type] + case "recall": + best_metric_value = ClassificationMetrics.recall(predicted=fitted_model.predict(test_data), expected=target_col, positive_class=positive_class) # type: ignore[arg-type] + case "f1_score": + best_metric_value = ClassificationMetrics.f1_score(predicted=fitted_model.predict(test_data), expected=target_col, positive_class=positive_class) # type: ignore[arg-type] + else: + match optimization_metric: + case "accuracy": + error_of_fitted_model = ClassificationMetrics.accuracy(predicted=fitted_model.predict(test_data), expected=target_col) # type: ignore[arg-type] + if error_of_fitted_model > best_metric_value: + best_model = fitted_model # pragma: no cover + best_metric_value = error_of_fitted_model # pragma: no cover + case "precision": + error_of_fitted_model = ClassificationMetrics.precision(predicted=fitted_model.predict(test_data), expected=target_col, positive_class=positive_class) # type: ignore[arg-type] + if error_of_fitted_model > best_metric_value: + best_model = fitted_model # pragma: no cover + best_metric_value = error_of_fitted_model # pragma: no cover + case "recall": + error_of_fitted_model = ClassificationMetrics.recall(predicted=fitted_model.predict(test_data), expected=target_col, positive_class=positive_class) # type: ignore[arg-type] + if error_of_fitted_model > best_metric_value: + best_model = fitted_model # pragma: no cover + best_metric_value = error_of_fitted_model # pragma: no cover + case "f1_score": + error_of_fitted_model = ClassificationMetrics.f1_score(predicted=fitted_model.predict(test_data), expected=target_col, positive_class=positive_class) # type: ignore[arg-type] + if error_of_fitted_model > best_metric_value: + best_model = fitted_model # pragma: no cover + best_metric_value = error_of_fitted_model # pragma: no cover + assert best_model is not None # just for linter + best_model._is_fitted = True + return best_model + + def _get_models_for_all_choices(self) -> list[Self]: + + all_possible_layer_combinations: list[list] = [[]] + for layer in self._layers: + if not layer._contains_choices(): + for item in all_possible_layer_combinations: + item.append(layer) + else: + updated_combinations = [] + versions_of_one_layer = layer._get_layers_for_all_choices() + for version in versions_of_one_layer: + copy_of_all_current_possible_combinations = copy.deepcopy(all_possible_layer_combinations) + for combination in copy_of_all_current_possible_combinations: + combination.append(version) + updated_combinations.append(combination) + all_possible_layer_combinations = updated_combinations + + models = [] + for combination in all_possible_layer_combinations: + new_model = NeuralNetworkClassifier(input_conversion=self._input_conversion, layers=combination) + models.append(new_model) + return models # type: ignore[return-value] + def predict(self, test_data: IPT) -> IFT: """ Make a prediction for the given test data. @@ -619,3 +919,7 @@ def input_size(self) -> int | ModelImageSize | None: """The input size of the model.""" # TODO: raise if not fitted, don't return None return self._input_size + + def _contains_choices(self) -> bool: + """Whether the model contains choices in any layer.""" + return any(layer._contains_choices() for layer in self._layers) diff --git a/src/safeds/ml/nn/layers/_convolutional2d_layer.py b/src/safeds/ml/nn/layers/_convolutional2d_layer.py index dd42f2d97..b67bd5330 100644 --- a/src/safeds/ml/nn/layers/_convolutional2d_layer.py +++ b/src/safeds/ml/nn/layers/_convolutional2d_layer.py @@ -125,6 +125,12 @@ def _set_input_size(self, input_size: int | ModelImageSize) -> None: self._input_size = input_size self._output_size = None + def _contains_choices(self) -> bool: + return False + + def _get_layers_for_all_choices(self) -> list[Convolutional2DLayer]: + raise NotImplementedError # pragma: no cover + def __hash__(self) -> int: return _structural_hash( self._output_channel, diff --git a/src/safeds/ml/nn/layers/_dropout_layer.py b/src/safeds/ml/nn/layers/_dropout_layer.py index 1814e4383..cb8fa548f 100644 --- a/src/safeds/ml/nn/layers/_dropout_layer.py +++ b/src/safeds/ml/nn/layers/_dropout_layer.py @@ -87,6 +87,12 @@ def output_size(self) -> int | ModelImageSize: def _set_input_size(self, input_size: int | ModelImageSize) -> None: self._input_size = input_size + def _contains_choices(self) -> bool: + return False + + def _get_layers_for_all_choices(self) -> list[DropoutLayer]: + raise NotImplementedError # pragma: no cover + def __hash__(self) -> int: return _structural_hash(self._input_size, self._probability) diff --git a/src/safeds/ml/nn/layers/_flatten_layer.py b/src/safeds/ml/nn/layers/_flatten_layer.py index a84551c2b..1b2671712 100644 --- a/src/safeds/ml/nn/layers/_flatten_layer.py +++ b/src/safeds/ml/nn/layers/_flatten_layer.py @@ -76,6 +76,12 @@ def _set_input_size(self, input_size: int | ModelImageSize) -> None: self._input_size = input_size self._output_size = None + def _contains_choices(self) -> bool: + return False + + def _get_layers_for_all_choices(self) -> list[FlattenLayer]: + raise NotImplementedError # pragma: no cover + def __hash__(self) -> int: return _structural_hash(self._input_size, self._output_size) diff --git a/src/safeds/ml/nn/layers/_forward_layer.py b/src/safeds/ml/nn/layers/_forward_layer.py index e420b78ec..9324af01b 100644 --- a/src/safeds/ml/nn/layers/_forward_layer.py +++ b/src/safeds/ml/nn/layers/_forward_layer.py @@ -4,6 +4,7 @@ from safeds._utils import _structural_hash from safeds._validation import _check_bounds, _ClosedBound +from safeds.ml.hyperparameters import Choice from safeds.ml.nn.typing import ModelImageSize from ._layer import Layer @@ -28,13 +29,19 @@ class ForwardLayer(Layer): If output_size < 1 """ - def __init__(self, neuron_count: int): - _check_bounds("neuron_count", neuron_count, lower_bound=_ClosedBound(1)) + def __init__(self, neuron_count: int | Choice[int]) -> None: + if isinstance(neuron_count, Choice): + for val in neuron_count: + _check_bounds("neuron_count", val, lower_bound=_ClosedBound(1)) + else: + _check_bounds("neuron_count", neuron_count, lower_bound=_ClosedBound(1)) self._input_size: int | None = None self._output_size = neuron_count def _get_internal_layer(self, **kwargs: Any) -> nn.Module: + assert not self._contains_choices() + assert not isinstance(self._output_size, Choice) from ._internal_layers import _InternalForwardLayer # Slow import on global level if "activation_function" not in kwargs: @@ -65,7 +72,7 @@ def input_size(self) -> int: return self._input_size @property - def output_size(self) -> int: + def output_size(self) -> int | Choice[int]: """ Get the output_size of this layer. @@ -82,6 +89,17 @@ def _set_input_size(self, input_size: int | ModelImageSize) -> None: self._input_size = input_size + def _contains_choices(self) -> bool: + return isinstance(self._output_size, Choice) + + def _get_layers_for_all_choices(self) -> list[ForwardLayer]: + assert self._contains_choices() + assert isinstance(self._output_size, Choice) # just for linter + layers = [] + for val in self._output_size: + layers.append(ForwardLayer(neuron_count=val)) + return layers + def __hash__(self) -> int: return _structural_hash(self._input_size, self._output_size) diff --git a/src/safeds/ml/nn/layers/_gru_layer.py b/src/safeds/ml/nn/layers/_gru_layer.py index e74fec417..3a0fae379 100644 --- a/src/safeds/ml/nn/layers/_gru_layer.py +++ b/src/safeds/ml/nn/layers/_gru_layer.py @@ -5,6 +5,7 @@ from safeds._utils import _structural_hash from safeds._validation import _check_bounds, _ClosedBound +from safeds.ml.hyperparameters import Choice from safeds.ml.nn.typing import ModelImageSize from ._layer import Layer @@ -29,13 +30,19 @@ class GRULayer(Layer): If output_size < 1 """ - def __init__(self, neuron_count: int): - _check_bounds("neuron_count", neuron_count, lower_bound=_ClosedBound(1)) + def __init__(self, neuron_count: int | Choice[int]): + if isinstance(neuron_count, Choice): + for val in neuron_count: + _check_bounds("neuron_count", val, lower_bound=_ClosedBound(1)) + else: + _check_bounds("neuron_count", neuron_count, lower_bound=_ClosedBound(1)) self._input_size: int | None = None self._output_size = neuron_count def _get_internal_layer(self, **kwargs: Any) -> nn.Module: + assert not self._contains_choices() + assert not isinstance(self._output_size, Choice) # just for linter from ._internal_layers import _InternalGRULayer # Slow import on global level if "activation_function" not in kwargs: @@ -65,7 +72,7 @@ def input_size(self) -> int: return self._input_size @property - def output_size(self) -> int: + def output_size(self) -> int | Choice[int]: """ Get the output_size of this layer. @@ -78,10 +85,21 @@ def output_size(self) -> int: def _set_input_size(self, input_size: int | ModelImageSize) -> None: if isinstance(input_size, ModelImageSize): - raise TypeError("The input_size of a forward layer has to be of type int.") + raise TypeError("The input_size of a gru layer has to be of type int.") self._input_size = input_size + def _contains_choices(self) -> bool: + return isinstance(self._output_size, Choice) + + def _get_layers_for_all_choices(self) -> list[GRULayer]: + assert self._contains_choices() + assert isinstance(self._output_size, Choice) # just for linter + layers = [] + for val in self._output_size: + layers.append(GRULayer(neuron_count=val)) + return layers + def __hash__(self) -> int: return _structural_hash( self._input_size, diff --git a/src/safeds/ml/nn/layers/_layer.py b/src/safeds/ml/nn/layers/_layer.py index 058036688..d469af9aa 100644 --- a/src/safeds/ml/nn/layers/_layer.py +++ b/src/safeds/ml/nn/layers/_layer.py @@ -1,11 +1,12 @@ from __future__ import annotations from abc import ABC, abstractmethod -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, Self if TYPE_CHECKING: from torch import nn + from safeds.ml.hyperparameters import Choice from safeds.ml.nn.typing import ModelImageSize @@ -25,13 +26,21 @@ def input_size(self) -> int | ModelImageSize: @property @abstractmethod - def output_size(self) -> int | ModelImageSize: + def output_size(self) -> int | ModelImageSize | Choice[int]: pass # pragma: no cover @abstractmethod def _set_input_size(self, input_size: int | ModelImageSize) -> None: pass # pragma: no cover + @abstractmethod + def _contains_choices(self) -> bool: + pass # pragma: no cover + + @abstractmethod + def _get_layers_for_all_choices(self) -> list[Self]: + pass # pragma: no cover + @abstractmethod def __hash__(self) -> int: pass # pragma: no cover diff --git a/src/safeds/ml/nn/layers/_lstm_layer.py b/src/safeds/ml/nn/layers/_lstm_layer.py index 330809474..6202d4fd2 100644 --- a/src/safeds/ml/nn/layers/_lstm_layer.py +++ b/src/safeds/ml/nn/layers/_lstm_layer.py @@ -5,6 +5,7 @@ from safeds._utils import _structural_hash from safeds._validation import _check_bounds, _ClosedBound +from safeds.ml.hyperparameters import Choice from safeds.ml.nn.typing import ModelImageSize from ._layer import Layer @@ -29,13 +30,19 @@ class LSTMLayer(Layer): If output_size < 1 """ - def __init__(self, neuron_count: int): - _check_bounds("neuron_count", neuron_count, lower_bound=_ClosedBound(1)) + def __init__(self, neuron_count: int | Choice[int]): + if isinstance(neuron_count, Choice): + for val in neuron_count: + _check_bounds("neuron_count", val, lower_bound=_ClosedBound(1)) + else: + _check_bounds("neuron_count", neuron_count, lower_bound=_ClosedBound(1)) self._input_size: int | None = None self._output_size = neuron_count def _get_internal_layer(self, **kwargs: Any) -> nn.Module: + assert not self._contains_choices() + assert not isinstance(self._output_size, Choice) from ._internal_layers import _InternalLSTMLayer # Slow import on global level if "activation_function" not in kwargs: @@ -65,7 +72,7 @@ def input_size(self) -> int: return self._input_size @property - def output_size(self) -> int: + def output_size(self) -> int | Choice[int]: """ Get the output_size of this layer. @@ -78,10 +85,21 @@ def output_size(self) -> int: def _set_input_size(self, input_size: int | ModelImageSize) -> None: if isinstance(input_size, ModelImageSize): - raise TypeError("The input_size of a forward layer has to be of type int.") + raise TypeError("The input_size of a lstm layer has to be of type int.") self._input_size = input_size + def _contains_choices(self) -> bool: + return isinstance(self._output_size, Choice) + + def _get_layers_for_all_choices(self) -> list[LSTMLayer]: + assert self._contains_choices() + assert isinstance(self._output_size, Choice) # just for linter + layers = [] + for val in self._output_size: + layers.append(LSTMLayer(neuron_count=val)) + return layers + def __hash__(self) -> int: return _structural_hash( self._input_size, diff --git a/src/safeds/ml/nn/layers/_pooling2d_layer.py b/src/safeds/ml/nn/layers/_pooling2d_layer.py index d658ed848..401e95f50 100644 --- a/src/safeds/ml/nn/layers/_pooling2d_layer.py +++ b/src/safeds/ml/nn/layers/_pooling2d_layer.py @@ -102,6 +102,12 @@ def _set_input_size(self, input_size: int | ModelImageSize) -> None: self._input_size = input_size self._output_size = None + def _contains_choices(self) -> bool: + return False + + def _get_layers_for_all_choices(self) -> list[_Pooling2DLayer]: + raise NotImplementedError # pragma: no cover + def __hash__(self) -> int: return _structural_hash( self._strategy, diff --git a/tests/safeds/ml/hyperparameters/test_choice.py b/tests/safeds/ml/hyperparameters/test_choice.py index 8adcd5952..ce4213044 100644 --- a/tests/safeds/ml/hyperparameters/test_choice.py +++ b/tests/safeds/ml/hyperparameters/test_choice.py @@ -3,12 +3,19 @@ from typing import TYPE_CHECKING import pytest +from safeds.exceptions import EmptyChoiceError from safeds.ml.hyperparameters import Choice if TYPE_CHECKING: from typing import Any +class TestInit: + def test_should_iterate_values(self) -> None: + with pytest.raises(EmptyChoiceError): + Choice() + + class TestContains: @pytest.mark.parametrize( ("choice", "value", "expected"), @@ -35,11 +42,9 @@ class TestIter: @pytest.mark.parametrize( ("choice", "expected"), [ - (Choice(), []), (Choice(1, 2, 3), [1, 2, 3]), ], ids=[ - "empty", "non-empty", ], ) @@ -51,13 +56,37 @@ class TestLen: @pytest.mark.parametrize( ("choice", "expected"), [ - (Choice(), 0), (Choice(1, 2, 3), 3), ], ids=[ - "empty", "non-empty", ], ) def test_should_return_number_of_values(self, choice: Choice, expected: int) -> None: assert len(choice) == expected + + +class TestEq: + @pytest.mark.parametrize( + ("choice1", "choice2", "equal"), + [ + ( + Choice(1), + Choice(1), + True, + ), + ( + Choice(1), + Choice(2), + False, + ), + ( + Choice(1, 2, 3), + Choice(1, 2, 3), + True, + ), + ], + ids=["equal", "not_equal", "equal with multiple values"], + ) + def test_should_compare_choices(self, choice1: Choice[int], choice2: Choice[int], equal: bool) -> None: + assert (choice1.__eq__(choice2)) == equal diff --git a/tests/safeds/ml/nn/layers/test_forward_layer.py b/tests/safeds/ml/nn/layers/test_forward_layer.py index 0ecd3bd05..e396e7308 100644 --- a/tests/safeds/ml/nn/layers/test_forward_layer.py +++ b/tests/safeds/ml/nn/layers/test_forward_layer.py @@ -4,6 +4,7 @@ import pytest from safeds.data.image.typing import ImageSize from safeds.exceptions import OutOfBoundsError +from safeds.ml.hyperparameters import Choice from safeds.ml.nn.layers import ForwardLayer from torch import nn @@ -66,23 +67,21 @@ def test_should_raise_if_unknown_activation_function_is_passed(activation_functi "output_size", [ 0, + Choice(0), ], - ids=["output_size_out_of_bounds"], + ids=["invalid_int", "invalid_choice"], ) -def test_should_raise_if_output_size_out_of_bounds(output_size: int) -> None: +def test_should_raise_if_output_size_out_of_bounds(output_size: int | Choice[int]) -> None: with pytest.raises(OutOfBoundsError): ForwardLayer(neuron_count=output_size) @pytest.mark.parametrize( "output_size", - [ - 1, - 20, - ], - ids=["one", "twenty"], + [1, 20, Choice(1, 20)], + ids=["one", "twenty", "choice"], ) -def test_should_return_output_size(output_size: int) -> None: +def test_should_return_output_size(output_size: int | Choice[int]) -> None: assert ForwardLayer(neuron_count=output_size).output_size == output_size @@ -114,8 +113,23 @@ def test_should_raise_if_activation_function_not_set() -> None: ForwardLayer(neuron_count=1), False, ), + ( + ForwardLayer(neuron_count=Choice(2)), + ForwardLayer(neuron_count=Choice(2)), + True, + ), + ( + ForwardLayer(neuron_count=Choice(2)), + ForwardLayer(neuron_count=Choice(1)), + False, + ), + ( + ForwardLayer(neuron_count=Choice(2)), + ForwardLayer(neuron_count=2), + False, + ), ], - ids=["equal", "not equal"], + ids=["equal", "not equal", "equal choices", "not equal choices", "choice and int"], ) def test_should_compare_forward_layers(layer1: ForwardLayer, layer2: ForwardLayer, equal: bool) -> None: assert (layer1.__eq__(layer2)) == equal @@ -177,3 +191,10 @@ def test_should_assert_that_different_forward_layers_have_different_hash( ) def test_should_assert_that_layer_size_is_greater_than_normal_object(layer: ForwardLayer) -> None: assert sys.getsizeof(layer) > sys.getsizeof(object()) + + +def test_should_get_all_possible_combinations_of_forward_layer() -> None: + layer = ForwardLayer(Choice(1, 2)) + possible_layers = layer._get_layers_for_all_choices() + assert possible_layers[0] == ForwardLayer(1) + assert possible_layers[1] == ForwardLayer(2) diff --git a/tests/safeds/ml/nn/layers/test_gru_layer.py b/tests/safeds/ml/nn/layers/test_gru_layer.py index 4a6f366e4..409b5880e 100644 --- a/tests/safeds/ml/nn/layers/test_gru_layer.py +++ b/tests/safeds/ml/nn/layers/test_gru_layer.py @@ -4,6 +4,7 @@ import pytest from safeds.data.image.typing import ImageSize from safeds.exceptions import OutOfBoundsError +from safeds.ml.hyperparameters import Choice from safeds.ml.nn.layers import GRULayer from torch import nn @@ -54,29 +55,27 @@ def test_should_raise_if_unknown_activation_function_is_passed(activation_functi "output_size", [ 0, + Choice(0), ], - ids=["output_size_out_of_bounds"], + ids=["invalid_int", "invalid_choice"], ) -def test_should_raise_if_output_size_out_of_bounds(output_size: int) -> None: +def test_should_raise_if_output_size_out_of_bounds(output_size: int | Choice[int]) -> None: with pytest.raises(OutOfBoundsError): GRULayer(neuron_count=output_size) @pytest.mark.parametrize( "output_size", - [ - 1, - 20, - ], - ids=["one", "twenty"], + [1, 20, Choice(1, 20)], + ids=["one", "twenty", "choice"], ) -def test_should_raise_if_output_size_doesnt_match(output_size: int) -> None: +def test_should_raise_if_output_size_doesnt_match(output_size: int | Choice[int]) -> None: assert GRULayer(neuron_count=output_size).output_size == output_size def test_should_raise_if_input_size_is_set_with_image_size() -> None: layer = GRULayer(1) - with pytest.raises(TypeError, match=r"The input_size of a forward layer has to be of type int."): + with pytest.raises(TypeError, match=r"The input_size of a gru layer has to be of type int."): layer._set_input_size(ImageSize(1, 2, 3)) @@ -102,14 +101,29 @@ def test_should_raise_if_activation_function_not_set() -> None: GRULayer(neuron_count=1), False, ), + ( + GRULayer(neuron_count=Choice(2)), + GRULayer(neuron_count=Choice(2)), + True, + ), + ( + GRULayer(neuron_count=Choice(2)), + GRULayer(neuron_count=Choice(1)), + False, + ), + ( + GRULayer(neuron_count=Choice(2)), + GRULayer(neuron_count=2), + False, + ), ], - ids=["equal", "not equal"], + ids=["equal", "not equal", "equal choices", "not equal choices", "choice and int"], ) -def test_should_compare_forward_layers(layer1: GRULayer, layer2: GRULayer, equal: bool) -> None: +def test_should_compare_gru_layers(layer1: GRULayer, layer2: GRULayer, equal: bool) -> None: assert (layer1.__eq__(layer2)) == equal -def test_should_assert_that_forward_layer_is_equal_to_itself() -> None: +def test_should_assert_that_gru_layer_is_equal_to_itself() -> None: layer = GRULayer(neuron_count=1) assert layer.__eq__(layer) @@ -119,9 +133,9 @@ def test_should_assert_that_forward_layer_is_equal_to_itself() -> None: [ (GRULayer(neuron_count=1), None), ], - ids=["ForwardLayer vs. None"], + ids=["GRULayer vs. None"], ) -def test_should_return_not_implemented_if_other_is_not_forward_layer(layer: GRULayer, other: Any) -> None: +def test_should_return_not_implemented_if_other_is_not_gru_layer(layer: GRULayer, other: Any) -> None: assert (layer.__eq__(other)) is NotImplemented @@ -135,7 +149,7 @@ def test_should_return_not_implemented_if_other_is_not_forward_layer(layer: GRUL ], ids=["equal"], ) -def test_should_assert_that_equal_forward_layers_have_equal_hash(layer1: GRULayer, layer2: GRULayer) -> None: +def test_should_assert_that_equal_gru_layers_have_equal_hash(layer1: GRULayer, layer2: GRULayer) -> None: assert layer1.__hash__() == layer2.__hash__() @@ -149,7 +163,7 @@ def test_should_assert_that_equal_forward_layers_have_equal_hash(layer1: GRULaye ], ids=["not equal"], ) -def test_should_assert_that_different_forward_layers_have_different_hash( +def test_should_assert_that_different_gru_layers_have_different_hash( layer1: GRULayer, layer2: GRULayer, ) -> None: @@ -187,3 +201,10 @@ def test_internal_layer_should_raise_error() -> None: layer = GRULayer(1) with pytest.raises(ValueError, match="The input_size is not yet set."): layer._get_internal_layer(activation_function="relu") + + +def test_should_get_all_possible_combinations_of_gru_layer() -> None: + layer = GRULayer(Choice(1, 2)) + possible_layers = layer._get_layers_for_all_choices() + assert possible_layers[0] == GRULayer(1) + assert possible_layers[1] == GRULayer(2) diff --git a/tests/safeds/ml/nn/layers/test_lstm_layer.py b/tests/safeds/ml/nn/layers/test_lstm_layer.py index 8d58e5dd8..fc04c6eac 100644 --- a/tests/safeds/ml/nn/layers/test_lstm_layer.py +++ b/tests/safeds/ml/nn/layers/test_lstm_layer.py @@ -4,6 +4,7 @@ import pytest from safeds.data.image.typing import ImageSize from safeds.exceptions import OutOfBoundsError +from safeds.ml.hyperparameters import Choice from safeds.ml.nn.layers import LSTMLayer from torch import nn @@ -66,29 +67,27 @@ def test_should_raise_if_unknown_activation_function_is_passed(activation_functi "output_size", [ 0, + Choice(0), ], - ids=["output_size_out_of_bounds"], + ids=["invalid_int", "invalid_choice"], ) -def test_should_raise_if_output_size_out_of_bounds(output_size: int) -> None: +def test_should_raise_if_output_size_out_of_bounds(output_size: int | Choice[int]) -> None: with pytest.raises(OutOfBoundsError): LSTMLayer(neuron_count=output_size) @pytest.mark.parametrize( "output_size", - [ - 1, - 20, - ], - ids=["one", "twenty"], + [1, 20, Choice(1, 20)], + ids=["one", "twenty", "choice"], ) -def test_should_raise_if_output_size_doesnt_match(output_size: int) -> None: +def test_should_raise_if_output_size_doesnt_match(output_size: int | Choice[int]) -> None: assert LSTMLayer(neuron_count=output_size).output_size == output_size def test_should_raise_if_input_size_is_set_with_image_size() -> None: layer = LSTMLayer(1) - with pytest.raises(TypeError, match=r"The input_size of a forward layer has to be of type int."): + with pytest.raises(TypeError, match=r"The input_size of a lstm layer has to be of type int."): layer._set_input_size(ImageSize(1, 2, 3)) @@ -114,14 +113,29 @@ def test_should_raise_if_activation_function_not_set() -> None: LSTMLayer(neuron_count=1), False, ), + ( + LSTMLayer(neuron_count=Choice(2)), + LSTMLayer(neuron_count=Choice(2)), + True, + ), + ( + LSTMLayer(neuron_count=Choice(2)), + LSTMLayer(neuron_count=Choice(1)), + False, + ), + ( + LSTMLayer(neuron_count=Choice(2)), + LSTMLayer(neuron_count=2), + False, + ), ], - ids=["equal", "not equal"], + ids=["equal", "not equal", "equal choices", "not equal choices", "choice and int"], ) -def test_should_compare_forward_layers(layer1: LSTMLayer, layer2: LSTMLayer, equal: bool) -> None: +def test_should_compare_lstm_layers(layer1: LSTMLayer, layer2: LSTMLayer, equal: bool) -> None: assert (layer1.__eq__(layer2)) == equal -def test_should_assert_that_forward_layer_is_equal_to_itself() -> None: +def test_should_assert_that_lstm_layer_is_equal_to_itself() -> None: layer = LSTMLayer(neuron_count=1) assert layer.__eq__(layer) @@ -131,9 +145,9 @@ def test_should_assert_that_forward_layer_is_equal_to_itself() -> None: [ (LSTMLayer(neuron_count=1), None), ], - ids=["ForwardLayer vs. None"], + ids=["LSTMLayer vs. None"], ) -def test_should_return_not_implemented_if_other_is_not_forward_layer(layer: LSTMLayer, other: Any) -> None: +def test_should_return_not_implemented_if_other_is_not_lstm_layer(layer: LSTMLayer, other: Any) -> None: assert (layer.__eq__(other)) is NotImplemented @@ -147,7 +161,7 @@ def test_should_return_not_implemented_if_other_is_not_forward_layer(layer: LSTM ], ids=["equal"], ) -def test_should_assert_that_equal_forward_layers_have_equal_hash(layer1: LSTMLayer, layer2: LSTMLayer) -> None: +def test_should_assert_that_equal_lstm_layers_have_equal_hash(layer1: LSTMLayer, layer2: LSTMLayer) -> None: assert layer1.__hash__() == layer2.__hash__() @@ -161,7 +175,7 @@ def test_should_assert_that_equal_forward_layers_have_equal_hash(layer1: LSTMLay ], ids=["not equal"], ) -def test_should_assert_that_different_forward_layers_have_different_hash( +def test_should_assert_that_different_lstm_layers_have_different_hash( layer1: LSTMLayer, layer2: LSTMLayer, ) -> None: @@ -177,3 +191,10 @@ def test_should_assert_that_different_forward_layers_have_different_hash( ) def test_should_assert_that_layer_size_is_greater_than_normal_object(layer: LSTMLayer) -> None: assert sys.getsizeof(layer) > sys.getsizeof(object()) + + +def test_should_get_all_possible_combinations_of_lstm_layer() -> None: + layer = LSTMLayer(Choice(1, 2)) + possible_layers = layer._get_layers_for_all_choices() + assert possible_layers[0] == LSTMLayer(1) + assert possible_layers[1] == LSTMLayer(2) diff --git a/tests/safeds/ml/nn/test_model.py b/tests/safeds/ml/nn/test_model.py index 43fc67aa6..fc7b87a89 100644 --- a/tests/safeds/ml/nn/test_model.py +++ b/tests/safeds/ml/nn/test_model.py @@ -1,5 +1,6 @@ import pickle import re +from typing import Any, Literal import pytest from safeds.data.image.typing import ImageSize @@ -7,11 +8,14 @@ from safeds.data.tabular.containers import Table from safeds.exceptions import ( FeatureDataMismatchError, + FittingWithChoiceError, + FittingWithoutChoiceError, InvalidFitDataError, InvalidModelStructureError, ModelNotFittedError, OutOfBoundsError, ) +from safeds.ml.hyperparameters import Choice from safeds.ml.nn import ( NeuralNetworkClassifier, NeuralNetworkRegressor, @@ -41,302 +45,388 @@ @pytest.mark.parametrize("device", get_devices(), ids=get_devices_ids()) class TestClassificationModel: - def test_should_return_input_size(self, device: Device) -> None: - configure_test_with_device(device) - model = NeuralNetworkClassifier( - InputConversionTable(), - [ForwardLayer(neuron_count=1)], - ).fit( - Table.from_dict({"a": [1], "b": [2]}).to_tabular_dataset("a"), - ) + class TestFit: + def test_should_return_input_size(self, device: Device) -> None: + configure_test_with_device(device) + model = NeuralNetworkClassifier( + InputConversionTable(), + [ForwardLayer(neuron_count=1)], + ).fit( + Table.from_dict({"a": [1], "b": [2]}).to_tabular_dataset("a"), + ) - assert model.input_size == 1 + assert model.input_size == 1 - @pytest.mark.parametrize( - "epoch_size", - [ - 0, - ], - ids=["epoch_size_out_of_bounds"], - ) - def test_should_raise_if_epoch_size_out_of_bounds(self, epoch_size: int, device: Device) -> None: - configure_test_with_device(device) - with pytest.raises(OutOfBoundsError): - NeuralNetworkClassifier( + def test_should_raise_if_epoch_size_out_of_bounds(self, device: Device) -> None: + invalid_epoch_size = 0 + configure_test_with_device(device) + with pytest.raises(OutOfBoundsError): + NeuralNetworkClassifier( + InputConversionTable(), + [ForwardLayer(1)], + ).fit( + Table.from_dict({"a": [1], "b": [2]}).to_tabular_dataset("a"), + epoch_size=invalid_epoch_size, + ) + + def test_should_raise_if_batch_size_out_of_bounds(self, device: Device) -> None: + invalid_batch_size = 0 + configure_test_with_device(device) + with pytest.raises(OutOfBoundsError): + NeuralNetworkClassifier( + InputConversionTable(), + [ForwardLayer(neuron_count=1)], + ).fit( + Table.from_dict({"a": [1], "b": [2]}).to_tabular_dataset("a"), + batch_size=invalid_batch_size, + ) + + def test_should_raise_if_fit_function_returns_wrong_datatype(self, device: Device) -> None: + configure_test_with_device(device) + fitted_model = NeuralNetworkClassifier( InputConversionTable(), - [ForwardLayer(1)], + [ForwardLayer(neuron_count=2), ForwardLayer(neuron_count=1)], ).fit( - Table.from_dict({"a": [1], "b": [2]}).to_tabular_dataset("a"), - epoch_size=epoch_size, + Table.from_dict({"a": [1], "b": [0]}).to_tabular_dataset("a"), ) + assert isinstance(fitted_model, NeuralNetworkClassifier) - @pytest.mark.parametrize( - "batch_size", - [ - 0, - ], - ids=["batch_size_out_of_bounds"], - ) - def test_should_raise_if_batch_size_out_of_bounds(self, batch_size: int, device: Device) -> None: - configure_test_with_device(device) - with pytest.raises(OutOfBoundsError): - NeuralNetworkClassifier( + def test_should_raise_when_fitting_with_choice(self, device: Device) -> None: + configure_test_with_device(device) + model = NeuralNetworkClassifier(InputConversionTable(), [ForwardLayer(Choice(1, 2))]) + with pytest.raises(FittingWithChoiceError): + model.fit(Table.from_dict({"a": [1], "b": [0]}).to_tabular_dataset("a")) + + def test_should_raise_if_is_fitted_is_set_correctly_for_binary_classification(self, device: Device) -> None: + configure_test_with_device(device) + model = NeuralNetworkClassifier( InputConversionTable(), [ForwardLayer(neuron_count=1)], - ).fit( - Table.from_dict({"a": [1], "b": [2]}).to_tabular_dataset("a"), - batch_size=batch_size, ) + model_2 = NeuralNetworkClassifier( + InputConversionTable(), + [LSTMLayer(neuron_count=1)], + ) + assert not model.is_fitted + assert not model_2.is_fitted + model = model.fit( + Table.from_dict({"a": [1], "b": [0]}).to_tabular_dataset("a"), + ) + model_2 = model_2.fit( + Table.from_dict({"a": [1], "b": [0]}).to_tabular_dataset("a"), + ) + assert model.is_fitted + assert model_2.is_fitted - def test_should_raise_if_fit_function_returns_wrong_datatype(self, device: Device) -> None: - configure_test_with_device(device) - fitted_model = NeuralNetworkClassifier( - InputConversionTable(), - [ForwardLayer(neuron_count=8), ForwardLayer(neuron_count=1)], - ).fit( - Table.from_dict({"a": [1], "b": [0]}).to_tabular_dataset("a"), - ) - assert isinstance(fitted_model, NeuralNetworkClassifier) + def test_should_raise_if_is_fitted_is_set_correctly_for_multiclass_classification(self, device: Device) -> None: + configure_test_with_device(device) + model = NeuralNetworkClassifier( + InputConversionTable(), + [ForwardLayer(neuron_count=1), ForwardLayer(neuron_count=3)], + ) + model_2 = NeuralNetworkClassifier( + InputConversionTable(), + [ForwardLayer(neuron_count=1), LSTMLayer(neuron_count=3)], + ) + assert not model.is_fitted + assert not model_2.is_fitted + model = model.fit( + Table.from_dict({"a": [1, 0, 2], "b": [0, 15, 5]}).to_tabular_dataset("a"), + ) + model_2 = model_2.fit( + Table.from_dict({"a": [1, 0, 2], "b": [0, 15, 5]}).to_tabular_dataset("a"), + ) + assert model.is_fitted + assert model_2.is_fitted - @pytest.mark.parametrize( - "batch_size", - [ - 1, - 2, - ], - ids=["one", "two"], - ) - def test_should_raise_if_predict_function_returns_wrong_datatype(self, batch_size: int, device: Device) -> None: - configure_test_with_device(device) - fitted_model = NeuralNetworkClassifier( - InputConversionTable(), - [ForwardLayer(neuron_count=8), ForwardLayer(neuron_count=1)], - ).fit( - Table.from_dict({"a": [1, 0, 1, 0, 1, 0], "b": [0, 1, 0, 12, 3, 3]}).to_tabular_dataset("a"), - batch_size=batch_size, - ) - predictions = fitted_model.predict(Table.from_dict({"b": [1, 0]})) - assert isinstance(predictions, TabularDataset) + def test_should_raise_if_train_features_mismatch(self, device: Device) -> None: + configure_test_with_device(device) + model = NeuralNetworkClassifier( + InputConversionTable(), + [ForwardLayer(neuron_count=1), ForwardLayer(neuron_count=1)], + ) + learned_model = model.fit( + Table.from_dict({"a": [0.1, 0, 0.2], "b": [0, 0.15, 0.5]}).to_tabular_dataset("b"), + ) + with pytest.raises( + FeatureDataMismatchError, + match="The features in the given table do not match with the specified feature columns names of the model.", + ): + learned_model.fit(Table.from_dict({"k": [0.1, 0, 0.2], "l": [0, 0.15, 0.5]}).to_tabular_dataset("k")) - @pytest.mark.parametrize( - "batch_size", - [ - 1, - 2, - ], - ids=["one", "two"], - ) - def test_should_raise_if_predict_function_returns_wrong_datatype_for_multiclass_classification( - self, - batch_size: int, - device: Device, - ) -> None: - configure_test_with_device(device) - fitted_model = NeuralNetworkClassifier( - InputConversionTable(), - [ForwardLayer(neuron_count=8), ForwardLayer(neuron_count=3)], - ).fit( - Table.from_dict({"a": [0, 1, 2], "b": [0, 15, 51]}).to_tabular_dataset("a"), - batch_size=batch_size, - ) - NeuralNetworkClassifier( - InputConversionTable(), - [ForwardLayer(neuron_count=8), LSTMLayer(neuron_count=3)], - ).fit( - Table.from_dict({"a": [0, 1, 2], "b": [0, 15, 51]}).to_tabular_dataset("a"), - batch_size=batch_size, + @pytest.mark.parametrize( + ("table", "reason"), + [ + ( + Table.from_dict({"a": [1, 2, 3], "b": [1, 2, None], "c": [0, 15, 5]}).to_tabular_dataset("c"), + re.escape("The given Fit Data is invalid:\nThe following Columns contain missing values: ['b']\n"), + ), + ( + Table.from_dict({"a": ["a", "b", "c"], "b": [1, 2, 3], "c": [0, 15, 5]}).to_tabular_dataset("c"), + re.escape( + "The given Fit Data is invalid:\nThe following Columns contain non-numerical data: ['a']", + ), + ), + ( + Table.from_dict({"a": ["a", "b", "c"], "b": [1, 2, None], "c": [0, 15, 5]}).to_tabular_dataset("c"), + re.escape( + "The given Fit Data is invalid:\nThe following Columns contain missing values: ['b']\nThe following Columns contain non-numerical data: ['a']", + ), + ), + ( + Table.from_dict({"a": [1, 2, 3], "b": [1, 2, 3], "c": [0, None, 5]}).to_tabular_dataset("c"), + re.escape( + "The given Fit Data is invalid:\nThe following Columns contain missing values: ['c']\n", + ), + ), + ( + Table.from_dict({"a": [1, 2, 3], "b": [1, 2, 3], "c": ["a", "b", "a"]}).to_tabular_dataset("c"), + re.escape( + "The given Fit Data is invalid:\nThe following Columns contain non-numerical data: ['c']", + ), + ), + ], + ids=[ + "missing value feature", + "non-numerical feature", + "missing value and non-numerical features", + "missing value target", + "non-numerical target", + ], ) - predictions = fitted_model.predict(Table.from_dict({"b": [1, 4, 124]})) - assert isinstance(predictions, TabularDataset) + def test_should_catch_invalid_fit_data(self, device: Device, table: TabularDataset, reason: str) -> None: + configure_test_with_device(device) + model = NeuralNetworkClassifier( + InputConversionTable(), + [ForwardLayer(neuron_count=4), ForwardLayer(1)], + ) + with pytest.raises( + InvalidFitDataError, + match=reason, + ): + model.fit(table) - def test_should_raise_if_model_has_not_been_fitted(self, device: Device) -> None: - configure_test_with_device(device) - with pytest.raises(ModelNotFittedError, match="The model has not been fitted yet."): - NeuralNetworkClassifier( + def test_should_raise_if_fit_doesnt_batch_callback(self, device: Device) -> None: + configure_test_with_device(device) + model = NeuralNetworkClassifier( InputConversionTable(), [ForwardLayer(neuron_count=1)], - ).predict( - Table.from_dict({"a": [1]}), ) - def test_should_raise_if_is_fitted_is_set_correctly_for_binary_classification(self, device: Device) -> None: - configure_test_with_device(device) - model = NeuralNetworkClassifier( - InputConversionTable(), - [ForwardLayer(neuron_count=1)], - ) - model_2 = NeuralNetworkClassifier( - InputConversionTable(), - [LSTMLayer(neuron_count=1)], - ) - assert not model.is_fitted - assert not model_2.is_fitted - model = model.fit( - Table.from_dict({"a": [1], "b": [0]}).to_tabular_dataset("a"), - ) - model_2 = model_2.fit( - Table.from_dict({"a": [1], "b": [0]}).to_tabular_dataset("a"), - ) - assert model.is_fitted - assert model_2.is_fitted + class Test: + self.was_called = False - def test_should_raise_if_is_fitted_is_set_correctly_for_multiclass_classification(self, device: Device) -> None: - configure_test_with_device(device) - model = NeuralNetworkClassifier( - InputConversionTable(), - [ForwardLayer(neuron_count=1), ForwardLayer(neuron_count=3)], - ) - model_2 = NeuralNetworkClassifier( - InputConversionTable(), - [ForwardLayer(neuron_count=1), LSTMLayer(neuron_count=3)], - ) - assert not model.is_fitted - assert not model_2.is_fitted - model = model.fit( - Table.from_dict({"a": [1, 0, 2], "b": [0, 15, 5]}).to_tabular_dataset("a"), - ) - model_2 = model_2.fit( - Table.from_dict({"a": [1, 0, 2], "b": [0, 15, 5]}).to_tabular_dataset("a"), - ) - assert model.is_fitted - assert model_2.is_fitted + def cb(self, ind: int, loss: float) -> None: + if ind >= 0 and loss >= 0.0: + self.was_called = True - def test_should_raise_if_test_features_mismatch(self, device: Device) -> None: - configure_test_with_device(device) - model = NeuralNetworkClassifier( - InputConversionTable(), - [ForwardLayer(neuron_count=1), ForwardLayer(neuron_count=3)], - ) - model = model.fit( - Table.from_dict({"a": [1, 0, 2], "b": [0, 15, 5]}).to_tabular_dataset("a"), - ) - with pytest.raises( - FeatureDataMismatchError, - match="The features in the given table do not match with the specified feature columns names of the model.", - ): - model.predict( - Table.from_dict({"a": [1], "c": [2]}), + def callback_was_called(self) -> bool: + return self.was_called + + obj = Test() + model.fit( + Table.from_dict({"a": [1], "b": [0]}).to_tabular_dataset("a"), + callback_on_batch_completion=obj.cb, ) - def test_should_raise_if_train_features_mismatch(self, device: Device) -> None: - configure_test_with_device(device) - model = NeuralNetworkClassifier( - InputConversionTable(), - [ForwardLayer(neuron_count=1), ForwardLayer(neuron_count=1)], - ) - learned_model = model.fit( - Table.from_dict({"a": [0.1, 0, 0.2], "b": [0, 0.15, 0.5]}).to_tabular_dataset("b"), - ) - with pytest.raises( - FeatureDataMismatchError, - match="The features in the given table do not match with the specified feature columns names of the model.", - ): - learned_model.fit(Table.from_dict({"k": [0.1, 0, 0.2], "l": [0, 0.15, 0.5]}).to_tabular_dataset("k")) + assert obj.callback_was_called() is True - @pytest.mark.parametrize( - ("table", "reason"), - [ - ( - Table.from_dict({"a": [1, 2, 3], "b": [1, 2, None], "c": [0, 15, 5]}).to_tabular_dataset("c"), - re.escape("The given Fit Data is invalid:\nThe following Columns contain missing values: ['b']\n"), - ), - ( - Table.from_dict({"a": ["a", "b", "c"], "b": [1, 2, 3], "c": [0, 15, 5]}).to_tabular_dataset("c"), - re.escape("The given Fit Data is invalid:\nThe following Columns contain non-numerical data: ['a']"), - ), - ( - Table.from_dict({"a": ["a", "b", "c"], "b": [1, 2, None], "c": [0, 15, 5]}).to_tabular_dataset("c"), - re.escape( - "The given Fit Data is invalid:\nThe following Columns contain missing values: ['b']\nThe following Columns contain non-numerical data: ['a']", - ), - ), - ( - Table.from_dict({"a": [1, 2, 3], "b": [1, 2, 3], "c": [0, None, 5]}).to_tabular_dataset("c"), - re.escape( - "The given Fit Data is invalid:\nThe following Columns contain missing values: ['c']\n", - ), - ), - ( - Table.from_dict({"a": [1, 2, 3], "b": [1, 2, 3], "c": ["a", "b", "a"]}).to_tabular_dataset("c"), - re.escape("The given Fit Data is invalid:\nThe following Columns contain non-numerical data: ['c']"), - ), - ], - ids=[ - "missing value feature", - "non-numerical feature", - "missing value and non-numerical features", - "missing value target", - "non-numerical target", - ], - ) - def test_should_catch_invalid_fit_data(self, device: Device, table: TabularDataset, reason: str) -> None: - configure_test_with_device(device) - model = NeuralNetworkClassifier( - InputConversionTable(), - [ForwardLayer(neuron_count=4), ForwardLayer(1)], - ) - with pytest.raises( - InvalidFitDataError, - match=reason, - ): - model.fit(table) - - # def test_should_raise_if_table_size_and_input_size_mismatch(self, device: Device) -> None: - # configure_test_with_device(device) - # model = NeuralNetworkClassifier( - # InputConversionTable(), - # [ForwardLayer(neuron_count=1), ForwardLayer(neuron_count=3)], - # ) - # with pytest.raises( - # InputSizeError, - # ): - # model.fit( - # Table.from_dict({"a": [1, 0, 2], "b": [0, 15, 5], "c": [3, 33, 333]}).to_tabular_dataset("a"), - # ) - - def test_should_raise_if_fit_doesnt_batch_callback(self, device: Device) -> None: - configure_test_with_device(device) - model = NeuralNetworkClassifier( - InputConversionTable(), - [ForwardLayer(neuron_count=1)], - ) + def test_should_raise_if_fit_doesnt_epoch_callback(self, device: Device) -> None: + configure_test_with_device(device) + model = NeuralNetworkClassifier( + InputConversionTable(), + [ForwardLayer(neuron_count=1)], + ) - class Test: - self.was_called = False + class Test: + self.was_called = False - def cb(self, ind: int, loss: float) -> None: - if ind >= 0 and loss >= 0.0: - self.was_called = True + def cb(self, ind: int, loss: float) -> None: + if ind >= 0 and loss >= 0.0: + self.was_called = True - def callback_was_called(self) -> bool: - return self.was_called + def callback_was_called(self) -> bool: + return self.was_called - obj = Test() - model.fit(Table.from_dict({"a": [1], "b": [0]}).to_tabular_dataset("a"), callback_on_batch_completion=obj.cb) + obj = Test() + model.fit( + Table.from_dict({"a": [1], "b": [0]}).to_tabular_dataset("a"), + callback_on_epoch_completion=obj.cb, + ) - assert obj.callback_was_called() is True + assert obj.callback_was_called() is True - def test_should_raise_if_fit_doesnt_epoch_callback(self, device: Device) -> None: - configure_test_with_device(device) - model = NeuralNetworkClassifier( - InputConversionTable(), - [ForwardLayer(neuron_count=1)], + class TestFitByExhaustiveSearch: + def test_should_return_input_size(self, device: Device) -> None: + configure_test_with_device(device) + model = NeuralNetworkClassifier( + InputConversionTable(), + [ForwardLayer(neuron_count=Choice(2, 4)), ForwardLayer(1)], + ).fit_by_exhaustive_search( + Table.from_dict({"a": [1, 2, 3, 4], "b": [0, 1, 0, 1]}).to_tabular_dataset("b"), + "accuracy", + ) + assert model.input_size == 1 + + def test_should_raise_if_epoch_size_out_of_bounds_when_fitting_by_exhaustive_search( + self, + device: Device, + ) -> None: + invalid_epoch_size = 0 + configure_test_with_device(device) + with pytest.raises(OutOfBoundsError): + NeuralNetworkClassifier( + InputConversionTable(), + [ForwardLayer(Choice(2, 4)), ForwardLayer(1)], + ).fit_by_exhaustive_search( + Table.from_dict({"a": [1], "b": [0]}).to_tabular_dataset("b"), + "accuracy", + epoch_size=invalid_epoch_size, + ) + + def test_should_raise_if_batch_size_out_of_bounds_when_fitting_by_exhaustive_search( + self, + device: Device, + ) -> None: + invalid_batch_size = 0 + configure_test_with_device(device) + with pytest.raises(OutOfBoundsError): + NeuralNetworkClassifier( + InputConversionTable(), + [ForwardLayer(neuron_count=Choice(2, 4)), ForwardLayer(1)], + ).fit_by_exhaustive_search( + Table.from_dict({"a": [1], "b": [0]}).to_tabular_dataset("b"), + "accuracy", + batch_size=invalid_batch_size, + ) + + def test_should_raise_when_fitting_by_exhaustive_search_without_choice(self, device: Device) -> None: + configure_test_with_device(device) + model = NeuralNetworkClassifier(InputConversionTable(), [ForwardLayer(1)]) + with pytest.raises(FittingWithoutChoiceError): + model.fit_by_exhaustive_search( + Table.from_dict({"a": [1], "b": [0]}).to_tabular_dataset("b"), + "accuracy", + ) + + @pytest.mark.parametrize( + ("metric", "positive_class"), + [ + ( + "accuracy", + None, + ), + ( + "precision", + 0, + ), + ( + "recall", + 0, + ), + ( + "f1_score", + 0, + ), + ], + ids=["accuracy", "precision", "recall", "f1_score"], ) + def test_should_assert_that_is_fitted_is_set_correctly_and_check_return_type( + self, + metric: Literal["accuracy", "precision", "recall", "f1_score"], + positive_class: Any, + device: Device, + ) -> None: + configure_test_with_device(device) + model = NeuralNetworkClassifier(InputConversionTable(), [ForwardLayer(Choice(2, 4)), ForwardLayer(1)]) + assert not model.is_fitted + fitted_model = model.fit_by_exhaustive_search( + Table.from_dict({"a": [1, 2, 3, 4], "b": [0, 1, 0, 1]}).to_tabular_dataset("b"), + optimization_metric=metric, + positive_class=positive_class, + ) + assert fitted_model.is_fitted + assert isinstance(fitted_model, NeuralNetworkClassifier) - class Test: - self.was_called = False + class TestPredict: - def cb(self, ind: int, loss: float) -> None: - if ind >= 0 and loss >= 0.0: - self.was_called = True + @pytest.mark.parametrize( + "batch_size", + [ + 1, + 2, + ], + ids=["one", "two"], + ) + def test_should_raise_if_predict_function_returns_wrong_datatype(self, batch_size: int, device: Device) -> None: + configure_test_with_device(device) + fitted_model = NeuralNetworkClassifier( + InputConversionTable(), + [ForwardLayer(neuron_count=8), ForwardLayer(neuron_count=1)], + ).fit( + Table.from_dict({"a": [1, 0, 1, 0, 1, 0], "b": [0, 1, 0, 12, 3, 3]}).to_tabular_dataset("a"), + batch_size=batch_size, + ) + predictions = fitted_model.predict(Table.from_dict({"b": [1, 0]})) + assert isinstance(predictions, TabularDataset) - def callback_was_called(self) -> bool: - return self.was_called + @pytest.mark.parametrize( + "batch_size", + [ + 1, + 2, + ], + ids=["one", "two"], + ) + def test_should_raise_if_predict_function_returns_wrong_datatype_for_multiclass_classification( + self, + batch_size: int, + device: Device, + ) -> None: + configure_test_with_device(device) + fitted_model = NeuralNetworkClassifier( + InputConversionTable(), + [ForwardLayer(neuron_count=8), ForwardLayer(neuron_count=3)], + ).fit( + Table.from_dict({"a": [0, 1, 2], "b": [0, 15, 51]}).to_tabular_dataset("a"), + batch_size=batch_size, + ) + NeuralNetworkClassifier( + InputConversionTable(), + [ForwardLayer(neuron_count=8), LSTMLayer(neuron_count=3)], + ).fit( + Table.from_dict({"a": [0, 1, 2], "b": [0, 15, 51]}).to_tabular_dataset("a"), + batch_size=batch_size, + ) + predictions = fitted_model.predict(Table.from_dict({"b": [1, 4, 124]})) + assert isinstance(predictions, TabularDataset) - obj = Test() - model.fit(Table.from_dict({"a": [1], "b": [0]}).to_tabular_dataset("a"), callback_on_epoch_completion=obj.cb) + def test_should_raise_if_model_has_not_been_fitted(self, device: Device) -> None: + configure_test_with_device(device) + with pytest.raises(ModelNotFittedError, match="The model has not been fitted yet."): + NeuralNetworkClassifier( + InputConversionTable(), + [ForwardLayer(neuron_count=1)], + ).predict( + Table.from_dict({"a": [1]}), + ) - assert obj.callback_was_called() is True + def test_should_raise_if_test_features_mismatch(self, device: Device) -> None: + configure_test_with_device(device) + model = NeuralNetworkClassifier( + InputConversionTable(), + [ForwardLayer(neuron_count=1), ForwardLayer(neuron_count=3)], + ) + model = model.fit( + Table.from_dict({"a": [1, 0, 2], "b": [0, 15, 5]}).to_tabular_dataset("a"), + ) + with pytest.raises( + FeatureDataMismatchError, + match="The features in the given table do not match with the specified feature columns names of the model.", + ): + model.predict( + Table.from_dict({"a": [1], "c": [2]}), + ) @pytest.mark.parametrize( ("input_conversion", "layers", "error_msg"), @@ -481,6 +571,11 @@ def callback_was_called(self) -> bool: [FlattenLayer()], r"A NeuralNetworkClassifier cannot be used with a InputConversionImage that uses a VariableImageSize.", ), + ( + InputConversionImageToImage(VariableImageSize(1, 1, 1)), + [FlattenLayer()], + r"A NeuralNetworkClassifier cannot be used with images as output.", + ), ], ) def test_should_raise_if_model_has_invalid_structure( @@ -517,252 +612,333 @@ def test_should_be_pickleable(self, device: Device) -> None: @pytest.mark.parametrize("device", get_devices(), ids=get_devices_ids()) class TestRegressionModel: - def test_should_return_input_size(self, device: Device) -> None: - configure_test_with_device(device) - model = NeuralNetworkRegressor( - InputConversionTable(), - [ForwardLayer(neuron_count=1)], - ).fit( - Table.from_dict({"a": [1], "b": [2]}).to_tabular_dataset("a"), - ) - - assert model.input_size == 1 - - @pytest.mark.parametrize( - "epoch_size", - [ - 0, - ], - ids=["epoch_size_out_of_bounds"], - ) - def test_should_raise_if_epoch_size_out_of_bounds(self, epoch_size: int, device: Device) -> None: - configure_test_with_device(device) - with pytest.raises(OutOfBoundsError): - NeuralNetworkRegressor( + class TestFit: + def test_should_return_input_size(self, device: Device) -> None: + configure_test_with_device(device) + model = NeuralNetworkRegressor( InputConversionTable(), [ForwardLayer(neuron_count=1)], ).fit( Table.from_dict({"a": [1], "b": [2]}).to_tabular_dataset("a"), - epoch_size=epoch_size, ) - @pytest.mark.parametrize( - "batch_size", - [ - 0, - ], - ids=["batch_size_out_of_bounds"], - ) - def test_should_raise_if_batch_size_out_of_bounds(self, batch_size: int, device: Device) -> None: - configure_test_with_device(device) - with pytest.raises(OutOfBoundsError): - NeuralNetworkRegressor( + assert model.input_size == 1 + + def test_should_raise_if_epoch_size_out_of_bounds(self, device: Device) -> None: + invalid_epoch_size = 0 + configure_test_with_device(device) + with pytest.raises(OutOfBoundsError): + NeuralNetworkRegressor( + InputConversionTable(), + [ForwardLayer(neuron_count=1)], + ).fit( + Table.from_dict({"a": [1], "b": [2]}).to_tabular_dataset("a"), + epoch_size=invalid_epoch_size, + ) + + def test_should_raise_if_batch_size_out_of_bounds(self, device: Device) -> None: + invalid_batch_size = 0 + configure_test_with_device(device) + with pytest.raises(OutOfBoundsError): + NeuralNetworkRegressor( + InputConversionTable(), + [ForwardLayer(neuron_count=1)], + ).fit( + Table.from_dict({"a": [1], "b": [2]}).to_tabular_dataset("a"), + batch_size=invalid_batch_size, + ) + + @pytest.mark.parametrize( + "batch_size", + [ + 1, + 2, + ], + ids=["one", "two"], + ) + def test_should_raise_if_fit_function_returns_wrong_datatype(self, batch_size: int, device: Device) -> None: + configure_test_with_device(device) + fitted_model = NeuralNetworkRegressor( InputConversionTable(), [ForwardLayer(neuron_count=1)], ).fit( - Table.from_dict({"a": [1], "b": [2]}).to_tabular_dataset("a"), + Table.from_dict({"a": [1, 0, 1], "b": [2, 3, 4]}).to_tabular_dataset("a"), batch_size=batch_size, ) + assert isinstance(fitted_model, NeuralNetworkRegressor) - @pytest.mark.parametrize( - "batch_size", - [ - 1, - 2, - ], - ids=["one", "two"], - ) - def test_should_raise_if_fit_function_returns_wrong_datatype(self, batch_size: int, device: Device) -> None: - configure_test_with_device(device) - fitted_model = NeuralNetworkRegressor( - InputConversionTable(), - [ForwardLayer(neuron_count=1)], - ).fit( - Table.from_dict({"a": [1, 0, 1], "b": [2, 3, 4]}).to_tabular_dataset("a"), - batch_size=batch_size, - ) - assert isinstance(fitted_model, NeuralNetworkRegressor) + def test_should_raise_when_fitting_with_choice(self, device: Device) -> None: + configure_test_with_device(device) + model = NeuralNetworkRegressor(InputConversionTable(), [ForwardLayer(Choice(1, 2))]) + with pytest.raises(FittingWithChoiceError): + model.fit(Table.from_dict({"a": [1], "b": [0]}).to_tabular_dataset("a")) - @pytest.mark.parametrize( - "batch_size", - [ - 1, - 2, - ], - ids=["one", "two"], - ) - def test_should_raise_if_predict_function_returns_wrong_datatype(self, batch_size: int, device: Device) -> None: - configure_test_with_device(device) - fitted_model = NeuralNetworkRegressor( - InputConversionTable(), - [ForwardLayer(neuron_count=1)], - ).fit( - Table.from_dict({"a": [1, 0, 1], "b": [2, 3, 4]}).to_tabular_dataset("a"), - batch_size=batch_size, - ) - predictions = fitted_model.predict(Table.from_dict({"b": [5, 6, 7]})) - assert isinstance(predictions, TabularDataset) + def test_should_raise_if_is_fitted_is_set_correctly(self, device: Device) -> None: + configure_test_with_device(device) + model = NeuralNetworkRegressor( + InputConversionTable(), + [ForwardLayer(neuron_count=1)], + ) + assert not model.is_fitted + model = model.fit( + Table.from_dict({"a": [1], "b": [0]}).to_tabular_dataset("a"), + ) + assert model.is_fitted - def test_should_raise_if_model_has_not_been_fitted(self, device: Device) -> None: - configure_test_with_device(device) - with pytest.raises(ModelNotFittedError, match="The model has not been fitted yet."): - NeuralNetworkRegressor( + def test_should_raise_if_train_features_mismatch(self, device: Device) -> None: + configure_test_with_device(device) + model = NeuralNetworkRegressor( InputConversionTable(), [ForwardLayer(neuron_count=1)], - ).predict( - Table.from_dict({"a": [1]}), ) + trained_model = model.fit( + Table.from_dict({"a": [1, 0, 2], "b": [0, 15, 5]}).to_tabular_dataset("b"), + ) + with pytest.raises( + FeatureDataMismatchError, + match="The features in the given table do not match with the specified feature columns names of the model.", + ): + trained_model.fit( + Table.from_dict({"k": [1, 0, 2], "l": [0, 15, 5]}).to_tabular_dataset("l"), + ) - def test_should_raise_if_is_fitted_is_set_correctly(self, device: Device) -> None: - configure_test_with_device(device) - model = NeuralNetworkRegressor( - InputConversionTable(), - [ForwardLayer(neuron_count=1)], - ) - assert not model.is_fitted - model = model.fit( - Table.from_dict({"a": [1], "b": [0]}).to_tabular_dataset("a"), + @pytest.mark.parametrize( + ("table", "reason"), + [ + ( + Table.from_dict({"a": [1, 2, 3], "b": [1, 2, None], "c": [0, 15, 5]}).to_tabular_dataset("c"), + re.escape("The given Fit Data is invalid:\nThe following Columns contain missing values: ['b']\n"), + ), + ( + Table.from_dict({"a": ["a", "b", "c"], "b": [1, 2, 3], "c": [0, 15, 5]}).to_tabular_dataset("c"), + re.escape( + "The given Fit Data is invalid:\nThe following Columns contain non-numerical data: ['a']", + ), + ), + ( + Table.from_dict({"a": ["a", "b", "c"], "b": [1, 2, None], "c": [0, 15, 5]}).to_tabular_dataset("c"), + re.escape( + "The given Fit Data is invalid:\nThe following Columns contain missing values: ['b']\nThe following Columns contain non-numerical data: ['a']", + ), + ), + ( + Table.from_dict({"a": [1, 2, 3], "b": [1, 2, 3], "c": [0, None, 5]}).to_tabular_dataset("c"), + re.escape( + "The given Fit Data is invalid:\nThe following Columns contain missing values: ['c']\n", + ), + ), + ( + Table.from_dict({"a": [1, 2, 3], "b": [1, 2, 3], "c": ["a", "b", "a"]}).to_tabular_dataset("c"), + re.escape( + "The given Fit Data is invalid:\nThe following Columns contain non-numerical data: ['c']", + ), + ), + ], + ids=[ + "missing value feature", + "non-numerical feature", + "missing value and non-numerical features", + "missing value target", + "non-numerical target", + ], ) - assert model.is_fitted + def test_should_catch_invalid_fit_data(self, device: Device, table: TabularDataset, reason: str) -> None: + configure_test_with_device(device) + model = NeuralNetworkRegressor( + InputConversionTable(), + [ForwardLayer(neuron_count=4), ForwardLayer(1)], + ) + with pytest.raises( + InvalidFitDataError, + match=reason, + ): + model.fit(table) - def test_should_raise_if_test_features_mismatch(self, device: Device) -> None: - configure_test_with_device(device) - model = NeuralNetworkRegressor( - InputConversionTable(), - [ForwardLayer(neuron_count=1)], - ) - model = model.fit( - Table.from_dict({"a": [1, 0, 2], "b": [0, 15, 5]}).to_tabular_dataset("a"), - ) - with pytest.raises( - FeatureDataMismatchError, - match="The features in the given table do not match with the specified feature columns names of the model.", - ): - model.predict( - Table.from_dict({"a": [1], "c": [2]}), + def test_should_raise_if_fit_doesnt_batch_callback(self, device: Device) -> None: + configure_test_with_device(device) + model = NeuralNetworkRegressor( + InputConversionTable(), + [ForwardLayer(neuron_count=1)], ) - def test_should_raise_if_train_features_mismatch(self, device: Device) -> None: - configure_test_with_device(device) - model = NeuralNetworkRegressor( - InputConversionTable(), - [ForwardLayer(neuron_count=1)], - ) - trained_model = model.fit( - Table.from_dict({"a": [1, 0, 2], "b": [0, 15, 5]}).to_tabular_dataset("b"), - ) - with pytest.raises( - FeatureDataMismatchError, - match="The features in the given table do not match with the specified feature columns names of the model.", - ): - trained_model.fit( - Table.from_dict({"k": [1, 0, 2], "l": [0, 15, 5]}).to_tabular_dataset("l"), + class Test: + self.was_called = False + + def cb(self, ind: int, loss: float) -> None: + if ind >= 0 and loss >= 0.0: + self.was_called = True + + def callback_was_called(self) -> bool: + return self.was_called + + obj = Test() + model.fit( + Table.from_dict({"a": [1], "b": [0]}).to_tabular_dataset("a"), + callback_on_batch_completion=obj.cb, ) - @pytest.mark.parametrize( - ("table", "reason"), - [ - ( - Table.from_dict({"a": [1, 2, 3], "b": [1, 2, None], "c": [0, 15, 5]}).to_tabular_dataset("c"), - re.escape("The given Fit Data is invalid:\nThe following Columns contain missing values: ['b']\n"), - ), - ( - Table.from_dict({"a": ["a", "b", "c"], "b": [1, 2, 3], "c": [0, 15, 5]}).to_tabular_dataset("c"), - re.escape("The given Fit Data is invalid:\nThe following Columns contain non-numerical data: ['a']"), - ), - ( - Table.from_dict({"a": ["a", "b", "c"], "b": [1, 2, None], "c": [0, 15, 5]}).to_tabular_dataset("c"), - re.escape( - "The given Fit Data is invalid:\nThe following Columns contain missing values: ['b']\nThe following Columns contain non-numerical data: ['a']", - ), - ), - ( - Table.from_dict({"a": [1, 2, 3], "b": [1, 2, 3], "c": [0, None, 5]}).to_tabular_dataset("c"), - re.escape( - "The given Fit Data is invalid:\nThe following Columns contain missing values: ['c']\n", - ), - ), - ( - Table.from_dict({"a": [1, 2, 3], "b": [1, 2, 3], "c": ["a", "b", "a"]}).to_tabular_dataset("c"), - re.escape("The given Fit Data is invalid:\nThe following Columns contain non-numerical data: ['c']"), - ), - ], - ids=[ - "missing value feature", - "non-numerical feature", - "missing value and non-numerical features", - "missing value target", - "non-numerical target", - ], - ) - def test_should_catch_invalid_fit_data(self, device: Device, table: TabularDataset, reason: str) -> None: - configure_test_with_device(device) - model = NeuralNetworkRegressor( - InputConversionTable(), - [ForwardLayer(neuron_count=4), ForwardLayer(1)], - ) - with pytest.raises( - InvalidFitDataError, - match=reason, - ): - model.fit(table) - - # def test_should_raise_if_table_size_and_input_size_mismatch(self, device: Device) -> None: - # configure_test_with_device(device) - # model = NeuralNetworkRegressor( - # InputConversionTable(), - # [ForwardLayer(neuron_count=1), ForwardLayer(neuron_count=3)], - # ) - # with pytest.raises( - # InputSizeError, - # ): - # model.fit( - # Table.from_dict({"a": [1, 0, 2], "b": [0, 15, 5], "c": [3, 33, 333]}).to_tabular_dataset("a"), - # ) - - def test_should_raise_if_fit_doesnt_batch_callback(self, device: Device) -> None: - configure_test_with_device(device) - model = NeuralNetworkRegressor( - InputConversionTable(), - [ForwardLayer(neuron_count=1)], - ) + assert obj.callback_was_called() is True - class Test: - self.was_called = False + def test_should_raise_if_fit_doesnt_epoch_callback(self, device: Device) -> None: + configure_test_with_device(device) + model = NeuralNetworkRegressor( + InputConversionTable(), + [ForwardLayer(neuron_count=1)], + ) - def cb(self, ind: int, loss: float) -> None: - if ind >= 0 and loss >= 0.0: - self.was_called = True + class Test: + self.was_called = False - def callback_was_called(self) -> bool: - return self.was_called + def cb(self, ind: int, loss: float) -> None: + if ind >= 0 and loss >= 0.0: + self.was_called = True - obj = Test() - model.fit(Table.from_dict({"a": [1], "b": [0]}).to_tabular_dataset("a"), callback_on_batch_completion=obj.cb) + def callback_was_called(self) -> bool: + return self.was_called - assert obj.callback_was_called() is True + obj = Test() + model.fit( + Table.from_dict({"a": [1], "b": [0]}).to_tabular_dataset("a"), + callback_on_epoch_completion=obj.cb, + ) - def test_should_raise_if_fit_doesnt_epoch_callback(self, device: Device) -> None: - configure_test_with_device(device) - model = NeuralNetworkRegressor( - InputConversionTable(), - [ForwardLayer(neuron_count=1)], - ) + assert obj.callback_was_called() is True - class Test: - self.was_called = False + class TestFitByExhaustiveSearch: + def test_should_return_input_size(self, device: Device) -> None: + configure_test_with_device(device) + model = NeuralNetworkRegressor( + InputConversionTable(), + [ForwardLayer(neuron_count=Choice(2, 4)), ForwardLayer(1)], + ).fit_by_exhaustive_search( + Table.from_dict({"a": [1, 2, 3, 4], "b": [1.0, 2.0, 3.0, 4.0]}).to_tabular_dataset("b"), + "mean_squared_error", + ) + assert model.input_size == 1 - def cb(self, ind: int, loss: float) -> None: - if ind >= 0 and loss >= 0.0: - self.was_called = True + def test_should_raise_if_epoch_size_out_of_bounds_when_fitting_by_exhaustive_search( + self, + device: Device, + ) -> None: + invalid_epoch_size = 0 + configure_test_with_device(device) + with pytest.raises(OutOfBoundsError): + NeuralNetworkRegressor( + InputConversionTable(), + [ForwardLayer(Choice(1, 3))], + ).fit_by_exhaustive_search( + Table.from_dict({"a": [1], "b": [1.0]}).to_tabular_dataset("b"), + "mean_squared_error", + epoch_size=invalid_epoch_size, + ) - def callback_was_called(self) -> bool: - return self.was_called + def test_should_raise_if_batch_size_out_of_bounds_when_fitting_by_exhaustive_search( + self, + device: Device, + ) -> None: + invalid_batch_size = 0 + configure_test_with_device(device) + with pytest.raises(OutOfBoundsError): + NeuralNetworkRegressor( + InputConversionTable(), + [ForwardLayer(neuron_count=Choice(1, 3))], + ).fit_by_exhaustive_search( + Table.from_dict({"a": [1], "b": [1.0]}).to_tabular_dataset("b"), + "mean_squared_error", + batch_size=invalid_batch_size, + ) - obj = Test() - model.fit(Table.from_dict({"a": [1], "b": [0]}).to_tabular_dataset("a"), callback_on_epoch_completion=obj.cb) + def test_should_raise_when_fitting_by_exhaustive_search_without_choice(self, device: Device) -> None: + configure_test_with_device(device) + model = NeuralNetworkRegressor(InputConversionTable(), [ForwardLayer(1)]) + with pytest.raises(FittingWithoutChoiceError): + model.fit_by_exhaustive_search( + Table.from_dict({"a": [1], "b": [1.0]}).to_tabular_dataset("b"), + "mean_squared_error", + ) - assert obj.callback_was_called() is True + @pytest.mark.parametrize( + "metric", + [ + "mean_squared_error", + "mean_absolute_error", + "median_absolute_deviation", + "coefficient_of_determination", + ], + ids=[ + "mean_squared_error", + "mean_absolute_error", + "median_absolute_deviation", + "coefficient_of_determination", + ], + ) + def test_should_assert_that_is_fitted_is_set_correctly_and_check_return_type( + self, + metric: Literal[ + "mean_squared_error", + "mean_absolute_error", + "median_absolute_deviation", + "coefficient_of_determination", + ], + device: Device, + ) -> None: + configure_test_with_device(device) + model = NeuralNetworkRegressor(InputConversionTable(), [ForwardLayer(Choice(2, 4)), ForwardLayer(1)]) + assert not model.is_fitted + fitted_model = model.fit_by_exhaustive_search( + Table.from_dict({"a": [1, 2, 3, 4], "b": [1.0, 2.0, 3.0, 4.0]}).to_tabular_dataset("b"), + optimization_metric=metric, + ) + assert fitted_model.is_fitted + assert isinstance(fitted_model, NeuralNetworkRegressor) + + class TestPredict: + @pytest.mark.parametrize( + "batch_size", + [ + 1, + 2, + ], + ids=["one", "two"], + ) + def test_should_raise_if_predict_function_returns_wrong_datatype(self, batch_size: int, device: Device) -> None: + configure_test_with_device(device) + fitted_model = NeuralNetworkRegressor( + InputConversionTable(), + [ForwardLayer(neuron_count=1)], + ).fit( + Table.from_dict({"a": [1, 0, 1], "b": [2, 3, 4]}).to_tabular_dataset("a"), + batch_size=batch_size, + ) + predictions = fitted_model.predict(Table.from_dict({"b": [5, 6, 7]})) + assert isinstance(predictions, TabularDataset) + + def test_should_raise_if_model_has_not_been_fitted(self, device: Device) -> None: + configure_test_with_device(device) + with pytest.raises(ModelNotFittedError, match="The model has not been fitted yet."): + NeuralNetworkRegressor( + InputConversionTable(), + [ForwardLayer(neuron_count=1)], + ).predict( + Table.from_dict({"a": [1]}), + ) + + def test_should_raise_if_test_features_mismatch(self, device: Device) -> None: + configure_test_with_device(device) + model = NeuralNetworkRegressor( + InputConversionTable(), + [ForwardLayer(neuron_count=1)], + ) + model = model.fit( + Table.from_dict({"a": [1, 0, 2], "b": [0, 15, 5]}).to_tabular_dataset("a"), + ) + with pytest.raises( + FeatureDataMismatchError, + match="The features in the given table do not match with the specified feature columns names of the model.", + ): + model.predict( + Table.from_dict({"a": [1], "c": [2]}), + ) @pytest.mark.parametrize( ("input_conversion", "layers", "error_msg"),