From 09c5082215b399a4853fe6dac2a0e44a7915f8f5 Mon Sep 17 00:00:00 2001 From: Lars Reimann Date: Wed, 1 May 2024 17:50:33 +0200 Subject: [PATCH] fix: actually use `kernel` of support vector machines for training (#681) Closes #602 ### Summary of Changes Previously, support vector machines always used an RBF kernel, regardless of the `kernel` requested by the user. This is fixed now. --- .../classification/_support_vector_machine.py | 114 +++++------------ .../regression/_support_vector_machine.py | 118 ++++++------------ .../test_support_vector_machine.py | 55 ++++---- .../regression/test_support_vector_machine.py | 55 ++++---- 4 files changed, 120 insertions(+), 222 deletions(-) diff --git a/src/safeds/ml/classical/classification/_support_vector_machine.py b/src/safeds/ml/classical/classification/_support_vector_machine.py index 6c937a72a..347eef2a6 100644 --- a/src/safeds/ml/classical/classification/_support_vector_machine.py +++ b/src/safeds/ml/classical/classification/_support_vector_machine.py @@ -2,7 +2,7 @@ import sys from abc import ABC, abstractmethod -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any from safeds._utils import _structural_hash from safeds.exceptions import ClosedBound, OpenBound, OutOfBoundsError @@ -21,15 +21,8 @@ class SupportVectorMachineKernel(ABC): """The abstract base class of the different subclasses supported by the `Kernel`.""" @abstractmethod - def _get_sklearn_kernel(self) -> object: - """ - Get the kernel of the given SupportVectorMachine. - - Returns - ------- - kernel: - The kernel of the SupportVectorMachine. - """ + def _get_sklearn_arguments(self) -> dict[str, Any]: + """Return the arguments to pass to scikit-learn.""" @abstractmethod def __eq__(self, other: object) -> bool: @@ -80,16 +73,20 @@ def __hash__(self) -> int: return _structural_hash(Classifier.__hash__(self), self._target_name, self._feature_names, self._c, self.kernel) def __init__(self, *, c: float = 1.0, kernel: SupportVectorMachineKernel | None = None) -> None: + # Inputs + if c <= 0: + raise OutOfBoundsError(c, name="c", lower_bound=OpenBound(0)) + if kernel is None: + kernel = self.Kernel.RadialBasisFunction() + # Internal state self._wrapped_classifier: sk_SVC | None = None self._feature_names: list[str] | None = None self._target_name: str | None = None # Hyperparameters - if c <= 0: - raise OutOfBoundsError(c, name="c", lower_bound=OpenBound(0)) - self._c = c - self._kernel = kernel + self._c: float = c + self._kernel: SupportVectorMachineKernel = kernel @property def c(self) -> float: @@ -104,7 +101,7 @@ def c(self) -> float: return self._c @property - def kernel(self) -> SupportVectorMachineKernel | None: + def kernel(self) -> SupportVectorMachineKernel: """ Get the type of kernel used. @@ -117,16 +114,10 @@ def kernel(self) -> SupportVectorMachineKernel | None: class Kernel: class Linear(SupportVectorMachineKernel): - def _get_sklearn_kernel(self) -> str: - """ - Get the name of the linear kernel. - - Returns - ------- - result: - The name of the linear kernel. - """ - return "linear" + def _get_sklearn_arguments(self) -> dict[str, Any]: + return { + "kernel": "linear", + } def __eq__(self, other: object) -> bool: if not isinstance(other, SupportVectorMachineClassifier.Kernel.Linear): @@ -141,16 +132,16 @@ def __init__(self, degree: int): raise OutOfBoundsError(degree, name="degree", lower_bound=ClosedBound(1)) self._degree = degree - def _get_sklearn_kernel(self) -> str: - """ - Get the name of the polynomial kernel. + @property + def degree(self) -> int: + """The degree of the polynomial kernel.""" + return self._degree - Returns - ------- - result: - The name of the polynomial kernel. - """ - return "poly" + def _get_sklearn_arguments(self) -> dict[str, Any]: + return { + "kernel": "poly", + "degree": self._degree, + } def __eq__(self, other: object) -> bool: if not isinstance(other, SupportVectorMachineClassifier.Kernel.Polynomial): @@ -172,16 +163,10 @@ def __sizeof__(self) -> int: return sys.getsizeof(self._degree) class Sigmoid(SupportVectorMachineKernel): - def _get_sklearn_kernel(self) -> str: - """ - Get the name of the sigmoid kernel. - - Returns - ------- - result: - The name of the sigmoid kernel. - """ - return "sigmoid" + def _get_sklearn_arguments(self) -> dict[str, Any]: + return { + "kernel": "sigmoid", + } def __eq__(self, other: object) -> bool: if not isinstance(other, SupportVectorMachineClassifier.Kernel.Sigmoid): @@ -191,16 +176,10 @@ def __eq__(self, other: object) -> bool: __hash__ = SupportVectorMachineKernel.__hash__ class RadialBasisFunction(SupportVectorMachineKernel): - def _get_sklearn_kernel(self) -> str: - """ - Get the name of the radial basis function (RBF) kernel. - - Returns - ------- - result: - The name of the RBF kernel. - """ - return "rbf" + def _get_sklearn_arguments(self) -> dict[str, Any]: + return { + "kernel": "rbf", + } def __eq__(self, other: object) -> bool: if not isinstance(other, SupportVectorMachineClassifier.Kernel.RadialBasisFunction): @@ -209,31 +188,6 @@ def __eq__(self, other: object) -> bool: __hash__ = SupportVectorMachineKernel.__hash__ - def _get_kernel_name(self) -> str: - """ - Get the name of the kernel. - - Returns - ------- - result: - The name of the kernel. - - Raises - ------ - TypeError - If the kernel type is invalid. - """ - if isinstance(self.kernel, SupportVectorMachineClassifier.Kernel.Linear): - return "linear" - elif isinstance(self.kernel, SupportVectorMachineClassifier.Kernel.Polynomial): - return "poly" - elif isinstance(self.kernel, SupportVectorMachineClassifier.Kernel.Sigmoid): - return "sigmoid" - elif isinstance(self.kernel, SupportVectorMachineClassifier.Kernel.RadialBasisFunction): - return "rbf" - else: - raise TypeError("Invalid kernel type.") - def fit(self, training_set: TabularDataset) -> SupportVectorMachineClassifier: """ Create a copy of this classifier and fit it with the given training data. @@ -322,4 +276,4 @@ def _get_sklearn_classifier(self) -> ClassifierMixin: """ from sklearn.svm import SVC as sk_SVC # noqa: N811 - return sk_SVC(C=self._c) + return sk_SVC(C=self._c, **(self._kernel._get_sklearn_arguments())) diff --git a/src/safeds/ml/classical/regression/_support_vector_machine.py b/src/safeds/ml/classical/regression/_support_vector_machine.py index 695603030..ab92a72d3 100644 --- a/src/safeds/ml/classical/regression/_support_vector_machine.py +++ b/src/safeds/ml/classical/regression/_support_vector_machine.py @@ -2,7 +2,7 @@ import sys from abc import ABC, abstractmethod -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any from safeds._utils import _structural_hash from safeds.exceptions import ClosedBound, OpenBound, OutOfBoundsError @@ -11,7 +11,7 @@ if TYPE_CHECKING: from sklearn.base import RegressorMixin - from sklearn.svm import SVR as sk_SVR # noqa: N811 + from sklearn.svm import SVC as sk_SVR # noqa: N811 from safeds.data.labeled.containers import TabularDataset from safeds.data.tabular.containers import Table @@ -21,15 +21,8 @@ class SupportVectorMachineKernel(ABC): """The abstract base class of the different subclasses supported by the `Kernel`.""" @abstractmethod - def _get_sklearn_kernel(self) -> object: - """ - Get the kernel of the given SupportVectorMachine. - - Returns - ------- - kernel: - The kernel of the SupportVectorMachine. - """ + def _get_sklearn_arguments(self) -> dict[str, Any]: + """Return the arguments to pass to scikit-learn.""" @abstractmethod def __eq__(self, other: object) -> bool: @@ -80,16 +73,20 @@ def __hash__(self) -> int: return _structural_hash(Regressor.__hash__(self), self._target_name, self._feature_names, self._c, self.kernel) def __init__(self, *, c: float = 1.0, kernel: SupportVectorMachineKernel | None = None) -> None: + # Inputs + if c <= 0: + raise OutOfBoundsError(c, name="c", lower_bound=OpenBound(0)) + if kernel is None: + kernel = self.Kernel.RadialBasisFunction() + # Internal state self._wrapped_regressor: sk_SVR | None = None self._feature_names: list[str] | None = None self._target_name: str | None = None # Hyperparameters - if c <= 0: - raise OutOfBoundsError(c, name="c", lower_bound=OpenBound(0)) - self._c = c - self._kernel = kernel + self._c: float = c + self._kernel: SupportVectorMachineKernel = kernel @property def c(self) -> float: @@ -104,7 +101,7 @@ def c(self) -> float: return self._c @property - def kernel(self) -> SupportVectorMachineKernel | None: + def kernel(self) -> SupportVectorMachineKernel: """ Get the type of kernel used. @@ -117,16 +114,10 @@ def kernel(self) -> SupportVectorMachineKernel | None: class Kernel: class Linear(SupportVectorMachineKernel): - def _get_sklearn_kernel(self) -> str: - """ - Get the name of the linear kernel. - - Returns - ------- - result: - The name of the linear kernel. - """ - return "linear" + def _get_sklearn_arguments(self) -> dict[str, Any]: + return { + "kernel": "linear", + } def __eq__(self, other: object) -> bool: if not isinstance(other, SupportVectorMachineRegressor.Kernel.Linear): @@ -141,16 +132,16 @@ def __init__(self, degree: int): raise OutOfBoundsError(degree, name="degree", lower_bound=ClosedBound(1)) self._degree = degree - def _get_sklearn_kernel(self) -> str: - """ - Get the name of the polynomial kernel. + @property + def degree(self) -> int: + """The degree of the polynomial kernel.""" + return self._degree - Returns - ------- - result: - The name of the polynomial kernel. - """ - return "poly" + def _get_sklearn_arguments(self) -> dict[str, Any]: + return { + "kernel": "poly", + "degree": self._degree, + } def __eq__(self, other: object) -> bool: if not isinstance(other, SupportVectorMachineRegressor.Kernel.Polynomial): @@ -172,16 +163,10 @@ def __sizeof__(self) -> int: return sys.getsizeof(self._degree) class Sigmoid(SupportVectorMachineKernel): - def _get_sklearn_kernel(self) -> str: - """ - Get the name of the sigmoid kernel. - - Returns - ------- - result: - The name of the sigmoid kernel. - """ - return "sigmoid" + def _get_sklearn_arguments(self) -> dict[str, Any]: + return { + "kernel": "sigmoid", + } def __eq__(self, other: object) -> bool: if not isinstance(other, SupportVectorMachineRegressor.Kernel.Sigmoid): @@ -191,16 +176,10 @@ def __eq__(self, other: object) -> bool: __hash__ = SupportVectorMachineKernel.__hash__ class RadialBasisFunction(SupportVectorMachineKernel): - def _get_sklearn_kernel(self) -> str: - """ - Get the name of the radial basis function (RBF) kernel. - - Returns - ------- - result: - The name of the RBF kernel. - """ - return "rbf" + def _get_sklearn_arguments(self) -> dict[str, Any]: + return { + "kernel": "rbf", + } def __eq__(self, other: object) -> bool: if not isinstance(other, SupportVectorMachineRegressor.Kernel.RadialBasisFunction): @@ -209,31 +188,6 @@ def __eq__(self, other: object) -> bool: __hash__ = SupportVectorMachineKernel.__hash__ - def _get_kernel_name(self) -> str: - """ - Get the name of the kernel. - - Returns - ------- - result: - The name of the kernel. - - Raises - ------ - TypeError - If the kernel type is invalid. - """ - if isinstance(self.kernel, SupportVectorMachineRegressor.Kernel.Linear): - return "linear" - elif isinstance(self.kernel, SupportVectorMachineRegressor.Kernel.Polynomial): - return "poly" - elif isinstance(self.kernel, SupportVectorMachineRegressor.Kernel.Sigmoid): - return "sigmoid" - elif isinstance(self.kernel, SupportVectorMachineRegressor.Kernel.RadialBasisFunction): - return "rbf" - else: - raise TypeError("Invalid kernel type.") - def fit(self, training_set: TabularDataset) -> SupportVectorMachineRegressor: """ Create a copy of this regressor and fit it with the given training data. @@ -320,6 +274,6 @@ def _get_sklearn_regressor(self) -> RegressorMixin: wrapped_regressor: The sklearn Regressor. """ - from sklearn.svm import SVR as sk_SVR # noqa: N811 + from sklearn.svm import SVC as sk_SVR # noqa: N811 - return sk_SVR(C=self._c) + return sk_SVR(C=self._c, **(self._kernel._get_sklearn_arguments())) diff --git a/tests/safeds/ml/classical/classification/test_support_vector_machine.py b/tests/safeds/ml/classical/classification/test_support_vector_machine.py index c68c5cd46..21df23650 100644 --- a/tests/safeds/ml/classical/classification/test_support_vector_machine.py +++ b/tests/safeds/ml/classical/classification/test_support_vector_machine.py @@ -62,52 +62,47 @@ def test_should_be_passed_to_sklearn(self, training_set: TabularDataset) -> None assert fitted_model._wrapped_classifier is not None assert isinstance(fitted_model.kernel, SupportVectorMachineClassifier.Kernel.Linear) - def test_should_get_sklearn_kernel_linear(self) -> None: + def test_should_get_sklearn_arguments_linear(self) -> None: svm = SupportVectorMachineClassifier(c=2, kernel=SupportVectorMachineClassifier.Kernel.Linear()) assert isinstance(svm.kernel, SupportVectorMachineClassifier.Kernel.Linear) - linear_kernel = svm.kernel._get_sklearn_kernel() - assert linear_kernel == "linear" + linear_kernel = svm.kernel._get_sklearn_arguments() + assert linear_kernel == { + "kernel": "linear", + } @pytest.mark.parametrize("degree", [-1, 0], ids=["minus_one", "zero"]) def test_should_raise_if_degree_less_than_1(self, degree: int) -> None: with pytest.raises(OutOfBoundsError, match=rf"degree \(={degree}\) is not inside \[1, \u221e\)\."): SupportVectorMachineClassifier.Kernel.Polynomial(degree=degree) - def test_should_get_sklearn_kernel_polynomial(self) -> None: + def test_should_get_sklearn_arguments_polynomial(self) -> None: svm = SupportVectorMachineClassifier(c=2, kernel=SupportVectorMachineClassifier.Kernel.Polynomial(degree=2)) assert isinstance(svm.kernel, SupportVectorMachineClassifier.Kernel.Polynomial) - poly_kernel = svm.kernel._get_sklearn_kernel() - assert poly_kernel == "poly" + poly_kernel = svm.kernel._get_sklearn_arguments() + assert poly_kernel == { + "kernel": "poly", + "degree": 2, + } - def test_should_get_sklearn_kernel_sigmoid(self) -> None: + def test_should_get_degree(self) -> None: + kernel = SupportVectorMachineClassifier.Kernel.Polynomial(degree=3) + assert kernel.degree == 3 + + def test_should_get_sklearn_arguments_sigmoid(self) -> None: svm = SupportVectorMachineClassifier(c=2, kernel=SupportVectorMachineClassifier.Kernel.Sigmoid()) assert isinstance(svm.kernel, SupportVectorMachineClassifier.Kernel.Sigmoid) - sigmoid_kernel = svm.kernel._get_sklearn_kernel() - assert sigmoid_kernel == "sigmoid" + sigmoid_kernel = svm.kernel._get_sklearn_arguments() + assert sigmoid_kernel == { + "kernel": "sigmoid", + } - def test_should_get_sklearn_kernel_rbf(self) -> None: + def test_should_get_sklearn_arguments_rbf(self) -> None: svm = SupportVectorMachineClassifier(c=2, kernel=SupportVectorMachineClassifier.Kernel.RadialBasisFunction()) assert isinstance(svm.kernel, SupportVectorMachineClassifier.Kernel.RadialBasisFunction) - rbf_kernel = svm.kernel._get_sklearn_kernel() - assert rbf_kernel == "rbf" - - def test_should_get_kernel_name(self) -> None: - svm = SupportVectorMachineClassifier(c=2, kernel=SupportVectorMachineClassifier.Kernel.Linear()) - assert svm._get_kernel_name() == "linear" - - svm = SupportVectorMachineClassifier(c=2, kernel=SupportVectorMachineClassifier.Kernel.Polynomial(degree=2)) - assert svm._get_kernel_name() == "poly" - - svm = SupportVectorMachineClassifier(c=2, kernel=SupportVectorMachineClassifier.Kernel.Sigmoid()) - assert svm._get_kernel_name() == "sigmoid" - - svm = SupportVectorMachineClassifier(c=2, kernel=SupportVectorMachineClassifier.Kernel.RadialBasisFunction()) - assert svm._get_kernel_name() == "rbf" - - def test_should_get_kernel_name_invalid_kernel_type(self) -> None: - svm = SupportVectorMachineClassifier(c=2) - with pytest.raises(TypeError, match="Invalid kernel type."): - svm._get_kernel_name() + rbf_kernel = svm.kernel._get_sklearn_arguments() + assert rbf_kernel == { + "kernel": "rbf", + } @pytest.mark.parametrize( ("kernel1", "kernel2"), diff --git a/tests/safeds/ml/classical/regression/test_support_vector_machine.py b/tests/safeds/ml/classical/regression/test_support_vector_machine.py index dc3df125d..6ed483b9d 100644 --- a/tests/safeds/ml/classical/regression/test_support_vector_machine.py +++ b/tests/safeds/ml/classical/regression/test_support_vector_machine.py @@ -62,52 +62,47 @@ def test_should_be_passed_to_sklearn(self, training_set: TabularDataset) -> None assert fitted_model._wrapped_regressor is not None assert isinstance(fitted_model.kernel, SupportVectorMachineRegressor.Kernel.Linear) - def test_should_get_sklearn_kernel_linear(self) -> None: + def test_should_get_sklearn_arguments_linear(self) -> None: svm = SupportVectorMachineRegressor(c=2, kernel=SupportVectorMachineRegressor.Kernel.Linear()) assert isinstance(svm.kernel, SupportVectorMachineRegressor.Kernel.Linear) - linear_kernel = svm.kernel._get_sklearn_kernel() - assert linear_kernel == "linear" + linear_kernel = svm.kernel._get_sklearn_arguments() + assert linear_kernel == { + "kernel": "linear", + } @pytest.mark.parametrize("degree", [-1, 0], ids=["minus_one", "zero"]) def test_should_raise_if_degree_less_than_1(self, degree: int) -> None: with pytest.raises(OutOfBoundsError, match=rf"degree \(={degree}\) is not inside \[1, \u221e\)\."): SupportVectorMachineRegressor.Kernel.Polynomial(degree=degree) - def test_should_get_sklearn_kernel_polynomial(self) -> None: + def test_should_get_sklearn_arguments_polynomial(self) -> None: svm = SupportVectorMachineRegressor(c=2, kernel=SupportVectorMachineRegressor.Kernel.Polynomial(degree=2)) assert isinstance(svm.kernel, SupportVectorMachineRegressor.Kernel.Polynomial) - poly_kernel = svm.kernel._get_sklearn_kernel() - assert poly_kernel == "poly" + poly_kernel = svm.kernel._get_sklearn_arguments() + assert poly_kernel == { + "kernel": "poly", + "degree": 2, + } - def test_should_get_sklearn_kernel_sigmoid(self) -> None: + def test_should_get_degree(self) -> None: + kernel = SupportVectorMachineRegressor.Kernel.Polynomial(degree=3) + assert kernel.degree == 3 + + def test_should_get_sklearn_arguments_sigmoid(self) -> None: svm = SupportVectorMachineRegressor(c=2, kernel=SupportVectorMachineRegressor.Kernel.Sigmoid()) assert isinstance(svm.kernel, SupportVectorMachineRegressor.Kernel.Sigmoid) - sigmoid_kernel = svm.kernel._get_sklearn_kernel() - assert sigmoid_kernel == "sigmoid" + sigmoid_kernel = svm.kernel._get_sklearn_arguments() + assert sigmoid_kernel == { + "kernel": "sigmoid", + } - def test_should_get_sklearn_kernel_rbf(self) -> None: + def test_should_get_sklearn_arguments_rbf(self) -> None: svm = SupportVectorMachineRegressor(c=2, kernel=SupportVectorMachineRegressor.Kernel.RadialBasisFunction()) assert isinstance(svm.kernel, SupportVectorMachineRegressor.Kernel.RadialBasisFunction) - rbf_kernel = svm.kernel._get_sklearn_kernel() - assert rbf_kernel == "rbf" - - def test_should_get_kernel_name(self) -> None: - svm = SupportVectorMachineRegressor(c=2, kernel=SupportVectorMachineRegressor.Kernel.Linear()) - assert svm._get_kernel_name() == "linear" - - svm = SupportVectorMachineRegressor(c=2, kernel=SupportVectorMachineRegressor.Kernel.Polynomial(degree=2)) - assert svm._get_kernel_name() == "poly" - - svm = SupportVectorMachineRegressor(c=2, kernel=SupportVectorMachineRegressor.Kernel.Sigmoid()) - assert svm._get_kernel_name() == "sigmoid" - - svm = SupportVectorMachineRegressor(c=2, kernel=SupportVectorMachineRegressor.Kernel.RadialBasisFunction()) - assert svm._get_kernel_name() == "rbf" - - def test_should_get_kernel_name_invalid_kernel_type(self) -> None: - svm = SupportVectorMachineRegressor(c=2) - with pytest.raises(TypeError, match="Invalid kernel type."): - svm._get_kernel_name() + rbf_kernel = svm.kernel._get_sklearn_arguments() + assert rbf_kernel == { + "kernel": "rbf", + } @pytest.mark.parametrize( ("kernel1", "kernel2"),