From 2df292bba19165d609105dd0482dc8d11b87e66a Mon Sep 17 00:00:00 2001 From: Valentin Laurent Date: Tue, 21 Jan 2025 18:27:01 +0100 Subject: [PATCH 1/2] DOC: change words conformity and conf to conformalization/conformalize --- README.rst | 6 +++--- doc/quick_start.rst | 6 +++--- doc/split_cross_conformal.rst | 2 +- doc/v1_migration_guide.rst | 4 ++-- mapie_v1/classification.py | 8 +++---- mapie_v1/regression.py | 40 +++++++++++++++++++---------------- 6 files changed, 35 insertions(+), 31 deletions(-) diff --git a/README.rst b/README.rst index a3f3f9279..1681b3eb5 100644 --- a/README.rst +++ b/README.rst @@ -115,8 +115,8 @@ As **MAPIE** is compatible with the standard scikit-learn API, you can see that from mapie_v1.regression import SplitConformalRegressor X, y = make_regression(n_samples=500, n_features=1, noise=20, random_state=59) - X_train_conf, X_test, y_train_conf, y_test = train_test_split(X, y, test_size=0.5) - X_train, X_conf, y_train, y_conf = train_test_split(X_train_conf, y_train_conf, test_size=0.5) + X_train_conformalize, X_test, y_train_conformalize, y_test = train_test_split(X, y, test_size=0.5) + X_train, X_conformalize, y_train, y_conformalize = train_test_split(X_train_conformalize, y_train_conformalize, test_size=0.5) regressor = LinearRegression() mapie_regressor = SplitConformalRegressor( @@ -124,7 +124,7 @@ As **MAPIE** is compatible with the standard scikit-learn API, you can see that confidence_level=[0.95, 0.68], ) mapie_regressor.fit(X_train, y_train) - mapie_regressor.conformalize(X_conf, y_conf) + mapie_regressor.conformalize(X_conformalize, y_conformalize) y_pred = mapie_regressor.predict(X_test) y_pred_intervals = mapie_regressor.predict_set(X_test) diff --git a/doc/quick_start.rst b/doc/quick_start.rst index 6ad9109f6..13e92bf02 100644 --- a/doc/quick_start.rst +++ b/doc/quick_start.rst @@ -45,8 +45,8 @@ Here, we generate one-dimensional noisy data that we fit with a linear model. regressor = LinearRegression() X, y = make_regression(n_samples=500, n_features=1, noise=20, random_state=59) - X_train_conf, X_test, y_train_conf, y_test = train_test_split(X, y, test_size=0.5) - X_train, X_conf, y_train, y_conf = train_test_split(X_train_conf, y_train_conf, + X_train_conformalize, X_test, y_train_conformalize, y_test = train_test_split(X, y, test_size=0.5) + X_train, X_conformalize, y_train, y_conformalize = train_test_split(X_train_conformalize, y_train_conformalize, test_size=0.5) # We follow a sequential ``fit``, ``conformalize``, and ``predict`` process. @@ -60,7 +60,7 @@ Here, we generate one-dimensional noisy data that we fit with a linear model. confidence_level=[0.95, 0.68], ) mapie_regressor.fit(X_train, y_train) - mapie_regressor.conformalize(X_conf, y_conf) + mapie_regressor.conformalize(X_conformalize, y_conformalize) y_pred = mapie_regressor.predict(X_test) y_pred_intervals = mapie_regressor.predict_set(X_test) diff --git a/doc/split_cross_conformal.rst b/doc/split_cross_conformal.rst index ca588f106..8c804efc9 100644 --- a/doc/split_cross_conformal.rst +++ b/doc/split_cross_conformal.rst @@ -1,5 +1,5 @@ ################################################################ -The calibration (or "conformity") set +The calibration (or "conformalization") set ################################################################ **MAPIE** is based on two types of techniques: diff --git a/doc/v1_migration_guide.rst b/doc/v1_migration_guide.rst index 26582c285..e55545c7e 100644 --- a/doc/v1_migration_guide.rst +++ b/doc/v1_migration_guide.rst @@ -51,7 +51,7 @@ In v1.0: MAPIE separates between training and calibration. We decided to name th - Additional fitting parameters, like ``sample_weight``, should be included in ``fit_params``, keeping this method focused on training alone. - ``.conformalize()`` method: - - This new method performs conformalization after fitting, using separate conformity data ``(X_conf, y_conf)``. + - This new method performs conformalization after fitting, using separate conformalization data ``(X_conformalize, y_conformalize)``. - ``predict_params`` can be passed here, allowing independent control over conformalization and prediction stages. Step 4: Making predictions (``predict`` and ``predict_set`` methods) @@ -159,7 +159,7 @@ Example 1: Split Conformal Prediction Description ############ -Split conformal prediction is a widely used method for generating prediction intervals, it splits the data into training, conformity, and test sets. The model is trained on the training set, calibrated on the conformity set, and then used to make predictions on the test set. In `MAPIE v1`, the `SplitConformalRegressor` replaces the older `MapieRegressor` with a more modular design and simplified API. +Split conformal prediction is a widely used method for generating prediction intervals, it splits the data into training, conformalization, and test sets. The model is trained on the training set, calibrated on the conformity set, and then used to make predictions on the test set. In `MAPIE v1`, the `SplitConformalRegressor` replaces the older `MapieRegressor` with a more modular design and simplified API. MAPIE v0.9 Code ############### diff --git a/mapie_v1/classification.py b/mapie_v1/classification.py index 0e609b8ed..671aeeda8 100644 --- a/mapie_v1/classification.py +++ b/mapie_v1/classification.py @@ -35,8 +35,8 @@ def fit( def conformalize( self, - X_conf: ArrayLike, - y_conf: ArrayLike, + X_conformalize: ArrayLike, + y_conformalize: ArrayLike, predict_params: Optional[dict] = None, ) -> Self: return self @@ -93,8 +93,8 @@ def fit( def conformalize( self, - X_conf: ArrayLike, - y_conf: ArrayLike, + X_conformalize: ArrayLike, + y_conformalize: ArrayLike, predict_params: Optional[dict] = None ) -> Self: return self diff --git a/mapie_v1/regression.py b/mapie_v1/regression.py index fbc9191dd..e4e9e72bf 100644 --- a/mapie_v1/regression.py +++ b/mapie_v1/regression.py @@ -65,7 +65,7 @@ class SplitConformalRegressor: Notes ----- This implementation currently uses a ShuffleSplit cross-validation scheme - for splitting the conformity set. Future implementations may allow the use + for splitting the conformalization set. Future implementations may allow the use of groups. Examples @@ -77,12 +77,14 @@ class SplitConformalRegressor: >>> X, y = make_regression(n_samples=500, n_features=2, noise=1.0) >>> X_train, X_conf_test, y_train, y_conf_test = train_test_split(X, y) - >>> X_conf, X_test, y_conf, y_test = train_test_split(X_conf_test, y_conf_test) + >>> X_conformalize, X_test, y_conformalize, y_test = train_test_split( + ... X_conf_test, y_conf_test + ... ) >>> mapie_regressor = SplitConformalRegressor( ... estimator=Ridge(), ... confidence_level=0.95 - ... ).fit(X_train, y_train).conformalize(X_conf, y_conf) + ... ).fit(X_train, y_train).conformalize(X_conformalize, y_conformalize) >>> prediction_points = mapie_regressor.predict(X_test) >>> prediction_intervals = mapie_regressor.predict_set(X_test) @@ -155,8 +157,8 @@ def fit( def conformalize( self, - X_conf: ArrayLike, - y_conf: ArrayLike, + X_conformalize: ArrayLike, + y_conformalize: ArrayLike, predict_params: Optional[dict] = None, ) -> Self: """ @@ -165,10 +167,10 @@ def conformalize( Parameters ---------- - X_conf : ArrayLike + X_conformalize : ArrayLike Features for the conformity set. - y_conf : ArrayLike + y_conformalize : ArrayLike Target values for the conformity set. predict_params : Optional[dict], default=None @@ -181,8 +183,8 @@ def conformalize( The conformalized SplitConformalRegressor instance. """ predict_params = {} if predict_params is None else predict_params - self._mapie_regressor.fit(X_conf, - y_conf, + self._mapie_regressor.fit(X_conformalize, + y_conformalize, predict_params=predict_params) return self @@ -907,7 +909,7 @@ class ConformalizedQuantileRegressor: Trains the base quantile regression estimator on the provided data. Not applicable if `prefit=True`. - conformalize(X_conf, y_conf, predict_params=None) -> Self + conformalize(X_conformalize, y_conformalize, predict_params=None) -> Self Calibrates the model on provided data, adjusting the prediction intervals to achieve the specified confidence levels. @@ -938,12 +940,14 @@ class ConformalizedQuantileRegressor: >>> X, y = make_regression(n_samples=500, n_features=2, noise=1.0) >>> X_train, X_conf_test, y_train, y_conf_test = train_test_split(X, y) - >>> X_conf, X_test, y_conf, y_test = train_test_split(X_conf_test, y_conf_test) + >>> X_conformalize, X_test, y_conformalize, y_test = train_test_split( + ... X_conf_test, y_conf_test + ... ) >>> mapie_regressor = ConformalizedQuantileRegressor( ... estimator=QuantileRegressor(), ... confidence_level=0.95, - ... ).fit(X_train, y_train).conformalize(X_conf, y_conf) + ... ).fit(X_train, y_train).conformalize(X_conformalize, y_conformalize) >>> prediction_points = mapie_regressor.predict(X_test) >>> prediction_intervals = mapie_regressor.predict_set(X_test) @@ -1026,8 +1030,8 @@ def fit( def conformalize( self, - X_conf: ArrayLike, - y_conf: ArrayLike, + X_conformalize: ArrayLike, + y_conformalize: ArrayLike, predict_params: Optional[dict] = None, ) -> Self: """ @@ -1038,10 +1042,10 @@ def conformalize( Parameters ---------- - X_conf : ArrayLike + X_conformalize : ArrayLike Features for the calibration (conformity) data. - y_conf : ArrayLike + y_conformalize : ArrayLike Target values for the calibration (conformity) data. predict_params : Optional[dict], default=None @@ -1057,8 +1061,8 @@ def conformalize( self.predict_params = predict_params if predict_params else {} self._mapie_quantile_regressor.conformalize( - X_conf, - y_conf, + X_conformalize, + y_conformalize, **self.predict_params ) From 4c25e029098245407d3aa748f159c2e4eec37443 Mon Sep 17 00:00:00 2001 From: Valentin Laurent Date: Wed, 22 Jan 2025 12:03:35 +0100 Subject: [PATCH 2/2] ENH: implement several changes to the API: - make prefit=True default for SplitConformalRegressor - change predict_set to predict_interval, and make it return point predictions - make mean aggregation default for predictions in cross conformal methods --- README.rst | 5 +- doc/quick_start.rst | 4 +- doc/v1_migration_guide.rst | 28 +-- mapie_v1/_utils.py | 18 +- mapie_v1/classification.py | 2 +- .../tests/test_regression.py | 26 +- mapie_v1/regression.py | 223 ++++++++---------- 7 files changed, 139 insertions(+), 167 deletions(-) diff --git a/README.rst b/README.rst index 1681b3eb5..a4960cb5a 100644 --- a/README.rst +++ b/README.rst @@ -119,15 +119,14 @@ As **MAPIE** is compatible with the standard scikit-learn API, you can see that X_train, X_conformalize, y_train, y_conformalize = train_test_split(X_train_conformalize, y_train_conformalize, test_size=0.5) regressor = LinearRegression() + regressor.fit(X_train, y_train) mapie_regressor = SplitConformalRegressor( regressor, confidence_level=[0.95, 0.68], ) - mapie_regressor.fit(X_train, y_train) mapie_regressor.conformalize(X_conformalize, y_conformalize) - y_pred = mapie_regressor.predict(X_test) - y_pred_intervals = mapie_regressor.predict_set(X_test) + y_pred, y_pred_intervals = mapie_regressor.predict_interval(X_test) .. code:: python diff --git a/doc/quick_start.rst b/doc/quick_start.rst index 13e92bf02..f556ac0b3 100644 --- a/doc/quick_start.rst +++ b/doc/quick_start.rst @@ -58,12 +58,12 @@ Here, we generate one-dimensional noisy data that we fit with a linear model. mapie_regressor = SplitConformalRegressor( regressor, confidence_level=[0.95, 0.68], + prefit=False, ) mapie_regressor.fit(X_train, y_train) mapie_regressor.conformalize(X_conformalize, y_conformalize) - y_pred = mapie_regressor.predict(X_test) - y_pred_intervals = mapie_regressor.predict_set(X_test) + y_pred, y_pred_intervals = mapie_regressor.predict_interval(X_test) # MAPIE's ``predict`` method returns point predictions as a ``np.ndarray`` of shape ``(n_samples)``. # The ``predict_set`` method returns prediction intervals as a ``np.ndarray`` of shape ``(n_samples, 2, 2)`` diff --git a/doc/v1_migration_guide.rst b/doc/v1_migration_guide.rst index e55545c7e..320047246 100644 --- a/doc/v1_migration_guide.rst +++ b/doc/v1_migration_guide.rst @@ -38,7 +38,7 @@ Step 1: Data splitting ~~~~~~~~~~~~~~~~~~~~~~ In v0.9, data splitting is handled by MAPIE. -In v1, the data splitting is left to the user, with the exception of cross-conformal methods (``CrossConformalRegressor``). The user can split the data into training, conformalization, and test sets using scikit-learn's ``train_test_split`` or other methods. +In v1, the data splitting is left to the user, with the exception of cross-conformal methods (``CrossConformalRegressor`` and ``JackknifeAfterBootstrapRegressor``). The user can split the data into training, conformalization, and test sets using scikit-learn's ``train_test_split`` or other methods. Step 2 & 3: Model training and conformalization (ie: calibration) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -54,13 +54,12 @@ In v1.0: MAPIE separates between training and calibration. We decided to name th - This new method performs conformalization after fitting, using separate conformalization data ``(X_conformalize, y_conformalize)``. - ``predict_params`` can be passed here, allowing independent control over conformalization and prediction stages. -Step 4: Making predictions (``predict`` and ``predict_set`` methods) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Step 4: Making predictions (``predict`` and ``predict_interval`` methods) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ In MAPIE v0.9, both point predictions and prediction intervals were produced through the ``predict`` method. -MAPIE v1 introduces two distinct methods for prediction: -- ``.predict_set()`` is dedicated to generating prediction intervals (i.e., lower and upper bounds), clearly separating interval predictions from point predictions. -- ``.predict()`` now focuses solely on producing point predictions. +MAPIE v1 introduces a new method for prediction, ``.predict_interval()``, that behaves like v0.9 ``.predict(alpha=...)`` method. Namely, it predicts points and intervals. +The ``.predict()`` method now focuses solely on producing point predictions. @@ -107,7 +106,7 @@ The ``groups`` parameter is used to specify group labels for cross-validation, e Controls whether the model has been pre-fitted before applying conformal prediction. - **v0.9**: Indicated through ``cv="prefit"`` in ``MapieRegressor``. -- **v1**: ``prefit`` is now a separate boolean parameter, allowing explicit control over whether the model has been pre-fitted before applying conformal methods. +- **v1**: ``prefit`` is now a separate boolean parameter, allowing explicit control over whether the model has been pre-fitted before applying conformal methods. It is set by default to ``True`` for ``SplitConformalRegressor``, as we believe this will become MAPIE nominal usage. ``fit_params`` (includes ``sample_weight``) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -125,10 +124,12 @@ Defines additional parameters exclusively for prediction. ``agg_function``, ``aggregation_method``, ``aggregate_predictions``, and ``ensemble`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -The aggregation method and technique for combining predictions in ensemble methods. +The aggregation method and technique for combining predictions in cross conformal methods. - **v0.9**: Previously, the ``agg_function`` parameter had two usage: to aggregate predictions when setting ``ensemble=True`` in the ``predict`` method, and to specify the aggregation technique in ``JackknifeAfterBootstrapRegressor``. -- **v1**: The ``agg_function`` parameter has been split into two distinct parameters: ``aggregate_predictions`` and ``aggregation_method``. ``aggregate_predictions`` is specific to ``CrossConformalRegressor``, and it specifies how predictions from multiple conformal regressors are aggregated when making point predictions. ``aggregation_method`` is specific to ``JackknifeAfterBootstrapRegressor``, and it specifies the aggregation technique for combining predictions across different bootstrap samples during conformalization. +- **v1**: + - The ``agg_function`` parameter has been split into two distinct parameters: ``aggregate_predictions`` and ``aggregation_method``. ``aggregate_predictions`` is specific to ``CrossConformalRegressor``, and it specifies how predictions from multiple conformal regressors are aggregated when making point predictions. ``aggregation_method`` is specific to ``JackknifeAfterBootstrapRegressor``, and it specifies the aggregation technique for combining predictions across different bootstrap samples during conformalization. + - Note that for both cross conformal methods, predictions points are now computed by default using mean aggregation. This is to avoid prediction points outside of prediction intervals in the default setting. ``random_state`` ~~~~~~~~~~~~~~~~~~ @@ -189,7 +190,7 @@ Below is a MAPIE v0.9 code for split conformal prediction in case of pre-fitted v0.fit(X_conf, y_conf) - prediction_intervals_v0 = v0.predict(X_test, alpha=0.1)[1][:, :, 0] + prediction_points_v0, prediction_intervals_v0 = v0.predict(X_test, alpha=0.1) prediction_points_v0 = v0.predict(X_test) Equivalent MAPIE v1 code @@ -215,13 +216,12 @@ Below is the equivalent MAPIE v1 code for split conformal prediction: estimator=prefit_model, confidence_level=0.9, conformity_score="residual_normalized", - prefit=True ) # Here we're not using v1.fit(), because the provided model is already fitted v1.conformalize(X_conf, y_conf) - prediction_intervals_v1 = v1.predict_set(X_test) + prediction_points_v1, prediction_intervals_v1 = v1.predict_interval(X_test) prediction_points_v1 = v1.predict(X_test) Example 2: Cross-Conformal Prediction @@ -263,7 +263,7 @@ Below is a MAPIE v0.9 code for cross-conformal prediction: v0.fit(X, y, sample_weight=sample_weight, groups=groups) - prediction_intervals_v0 = v0.predict(X_test, alpha=0.1)[1][:, :, 0] + prediction_points_v0, prediction_intervals_v0 = v0.predict(X_test, alpha=0.1) prediction_points_v0 = v0.predict(X_test, ensemble=True) Equivalent MAPIE v1 code @@ -299,5 +299,5 @@ Below is the equivalent MAPIE v1 code for cross-conformal prediction: v1.fit(X, y, fit_params={"sample_weight": sample_weight}) v1.conformalize(X, y, groups=groups) - prediction_intervals_v1 = v1.predict_set(X_test) + prediction_points_v1, prediction_intervals_v1 = v1.predict_interval(X_test) prediction_points_v1 = v1.predict(X_test, aggregate_predictions="median") diff --git a/mapie_v1/_utils.py b/mapie_v1/_utils.py index 2bac773af..49a3381d1 100644 --- a/mapie_v1/_utils.py +++ b/mapie_v1/_utils.py @@ -52,19 +52,13 @@ def check_if_X_y_different_from_fit( ) -def make_intervals_single_if_single_alpha( - intervals: NDArray, - alphas: Union[float, List[float]] -) -> NDArray: - if isinstance(alphas, float): - return intervals[:, :, 0] - if isinstance(alphas, list) and len(alphas) == 1: - return intervals[:, :, 0] - return intervals - - def cast_point_predictions_to_ndarray( point_predictions: Union[NDArray, Tuple[NDArray, NDArray]] ) -> NDArray: - # This will be useless when we split .predict and .predict_set in back-end return cast(NDArray, point_predictions) + + +def cast_predictions_to_ndarray_tuple( + predictions: Union[NDArray, Tuple[NDArray, NDArray]] +) -> Tuple[NDArray, NDArray]: + return cast(Tuple[NDArray, NDArray], predictions) diff --git a/mapie_v1/classification.py b/mapie_v1/classification.py index 671aeeda8..183c37ac8 100644 --- a/mapie_v1/classification.py +++ b/mapie_v1/classification.py @@ -18,7 +18,7 @@ def __init__( estimator: ClassifierMixin = LogisticRegression(), confidence_level: Union[float, List[float]] = 0.9, conformity_score: Union[str, BaseClassificationScore] = "lac", - prefit: bool = False, + prefit: bool = True, n_jobs: Optional[int] = None, verbose: int = 0, random_state: Optional[Union[int, np.random.RandomState]] = None, diff --git a/mapie_v1/integration_tests/tests/test_regression.py b/mapie_v1/integration_tests/tests/test_regression.py index 1ad7de845..4d48c7495 100644 --- a/mapie_v1/integration_tests/tests/test_regression.py +++ b/mapie_v1/integration_tests/tests/test_regression.py @@ -126,6 +126,7 @@ "v1": { "estimator": positive_predictor, "confidence_level": 0.9, + "prefit": False, "conformity_score": GammaConformityScore(), "test_size": 0.3, "minimize_interval_width": True @@ -183,6 +184,8 @@ def test_intervals_and_predictions_exact_equality_split(params_split): "alpha": [0.5, 0.5], "conformity_score": GammaConformityScore(), "cv": LeaveOneOut(), + "agg_function": "mean", + "ensemble": True, "method": "plus", "optimize_beta": True, "random_state": RANDOM_STATE, @@ -210,6 +213,7 @@ def test_intervals_and_predictions_exact_equality_split(params_split): "cv": GroupKFold(), "groups": groups, "method": "minmax", + "aggregate_predictions": None, "allow_infinite_bounds": True, "random_state": RANDOM_STATE, } @@ -262,6 +266,7 @@ def test_intervals_and_predictions_exact_equality_cross(params_cross): "alpha": [0.5, 0.5], "conformity_score": GammaConformityScore(), "agg_function": "mean", + "ensemble": True, "cv": Subsample(n_resamplings=20, replace=True, random_state=RANDOM_STATE), @@ -448,9 +453,15 @@ def compare_model_predictions_and_intervals( v1_params: Dict = {}, prefit: bool = False, test_size: Optional[float] = None, - sample_weight: Optional[ArrayLike] = None, random_state: int = RANDOM_STATE, ) -> None: + if v0_params.get("alpha"): + if isinstance(v0_params["alpha"], float): + n_alpha = 1 + else: + n_alpha = len(v0_params["alpha"]) + else: + n_alpha = 1 if test_size is not None: X_train, X_conf, y_train, y_conf = train_test_split_shuffle( @@ -496,14 +507,17 @@ def compare_model_predictions_and_intervals( v0_predict_params.pop('alpha') v1_predict_params = filter_params(v1.predict, v1_params) - v1_predict_set_params = filter_params(v1.predict_set, v1_params) + v1_predict_interval_params = filter_params(v1.predict_interval, v1_params) v0_preds, v0_pred_intervals = v0.predict(X_conf, **v0_predict_params) - v1_pred_intervals = v1.predict_set(X_conf, **v1_predict_set_params) - if v1_pred_intervals.ndim == 2: - v1_pred_intervals = np.expand_dims(v1_pred_intervals, axis=2) + v1_preds, v1_pred_intervals = v1.predict_interval( + X_conf, **v1_predict_interval_params + ) - v1_preds: ArrayLike = v1.predict(X_conf, **v1_predict_params) + v1_preds_using_predict: ArrayLike = v1.predict(X_conf, **v1_predict_params) np.testing.assert_array_equal(v0_preds, v1_preds) np.testing.assert_array_equal(v0_pred_intervals, v1_pred_intervals) + np.testing.assert_array_equal(v1_preds_using_predict, v1_preds) + if not v0_params.get("optimize_beta"): + assert v1_pred_intervals.shape == (len(X_conf), 2, n_alpha) diff --git a/mapie_v1/regression.py b/mapie_v1/regression.py index e4e9e72bf..8d9b590ad 100644 --- a/mapie_v1/regression.py +++ b/mapie_v1/regression.py @@ -1,5 +1,5 @@ import copy -from typing import Optional, Union, List, cast +from typing import Optional, Union, List, cast, Tuple from typing_extensions import Self import numpy as np @@ -18,8 +18,8 @@ ) from mapie_v1._utils import transform_confidence_level_to_alpha_list, \ check_if_param_in_allowed_values, check_cv_not_string, hash_X_y, \ - check_if_X_y_different_from_fit, make_intervals_single_if_single_alpha, \ - cast_point_predictions_to_ndarray + check_if_X_y_different_from_fit, \ + cast_point_predictions_to_ndarray, cast_predictions_to_ndarray_tuple class SplitConformalRegressor: @@ -83,11 +83,11 @@ class SplitConformalRegressor: >>> mapie_regressor = SplitConformalRegressor( ... estimator=Ridge(), - ... confidence_level=0.95 + ... confidence_level=0.95, + ... prefit=False, ... ).fit(X_train, y_train).conformalize(X_conformalize, y_conformalize) - >>> prediction_points = mapie_regressor.predict(X_test) - >>> prediction_intervals = mapie_regressor.predict_set(X_test) + >>> predicted_points, predicted_intervals = mapie_regressor.predict_interval(X_test) """ def __init__( @@ -95,7 +95,7 @@ def __init__( estimator: RegressorMixin = LinearRegression(), confidence_level: Union[float, List[float]] = 0.9, conformity_score: Union[str, BaseRegressionScore] = "absolute", - prefit: bool = False, + prefit: bool = True, n_jobs: Optional[int] = None, verbose: int = 0, ) -> None: @@ -189,15 +189,15 @@ def conformalize( return self - def predict_set( + def predict_interval( self, X: ArrayLike, minimize_interval_width: bool = False, allow_infinite_bounds: bool = False, - ) -> NDArray: + ) -> Tuple[NDArray, NDArray]: """ - Generates prediction intervals for the input data `X` based on - conformity scores and confidence level(s). + Generates prediction intervals (and prediction points) for the input data `X` + based on conformity scores and confidence level(s). Parameters ---------- @@ -212,23 +212,18 @@ def predict_set( Returns ------- - NDArray - An array containing the prediction intervals with shape - `(n_samples, 2)` if `confidence_level` is a single float, or - `(n_samples, 2, n_confidence_levels)` if `confidence_level` is a - list of floats. + Tuple[NDArray, NDArray] + Two arrays: + - Prediction points, of shape `(n_samples,)` + - Prediction intervals, of shape `(n_samples, 2, n_confidence_levels)` """ - _, intervals = self._mapie_regressor.predict( + predictions = self._mapie_regressor.predict( X, alpha=self._alphas, optimize_beta=minimize_interval_width, allow_infinite_bounds=allow_infinite_bounds ) - - return make_intervals_single_if_single_alpha( - intervals, - self._alphas - ) + return cast_predictions_to_ndarray_tuple(predictions) def predict( self, @@ -310,14 +305,6 @@ class CrossConformalRegressor: A seed or random state instance to ensure reproducibility in any random operations within the regressor. - Returns - ------- - NDArray - An array containing the prediction intervals with shape: - - `(n_samples, 2)` if `confidence_level` is a single float - - `(n_samples, 2, n_confidence_levels)` if `confidence_level` - is a list of floats. - Examples -------- >>> from mapie_v1.regression import CrossConformalRegressor @@ -334,8 +321,7 @@ class CrossConformalRegressor: ... cv=10 ... ).fit(X, y).conformalize(X, y) - >>> prediction_points = mapie_regressor.predict(X_test) - >>> prediction_intervals = mapie_regressor.predict_set(X_test) + >>> predicted_points, predicted_intervals = mapie_regressor.predict_interval(X_test) """ _VALID_METHODS = ["base", "plus", "minmax"] @@ -470,21 +456,29 @@ def conformalize( return self - def predict_set( + def predict_interval( self, X: ArrayLike, + aggregate_predictions: Optional[str] = "mean", minimize_interval_width: bool = False, allow_infinite_bounds: bool = False, - ) -> NDArray: + ) -> Tuple[NDArray, NDArray]: """ - Generates prediction intervals for the input data `X` based on - conformity scores and confidence level(s). + Generates prediction intervals (and prediction points) for the input data `X` + based on conformity scores and confidence level(s). Parameters ---------- X : ArrayLike Data features for generating prediction intervals. + aggregate_predictions : Optional[str], default="mean" + The method to aggregate point predictions across folds. Options: + - None: No aggregation, returns predictions from the estimator + trained on the entire dataset + - "mean": Returns the mean prediction across folds. + - "median": Returns the median prediction across folds. + minimize_interval_width : bool, default=False If True, attempts to minimize the interval width. @@ -494,43 +488,39 @@ def predict_set( Returns ------- - NDArray - An array containing the prediction intervals with shape - `(n_samples, 2)` if `confidence_level` is a single float, or - `(n_samples, 2, n_confidence_levels)` if `confidence_level` is a - list of floats. + Tuple[NDArray, NDArray] + Two arrays: + - Prediction points, of shape `(n_samples,)` + - Prediction intervals, of shape `(n_samples, 2, n_confidence_levels)` """ - # TODO: factorize this function once the v0 backend is updated with - # correct param names - _, intervals = self._mapie_regressor.predict( + ensemble = self._check_aggregate_predictions_and_return_ensemble( + aggregate_predictions + ) + predictions = self._mapie_regressor.predict( X, alpha=self._alphas, optimize_beta=minimize_interval_width, - allow_infinite_bounds=allow_infinite_bounds - ) - - return make_intervals_single_if_single_alpha( - intervals, - self._alphas + allow_infinite_bounds=allow_infinite_bounds, + ensemble=ensemble, ) + return cast_predictions_to_ndarray_tuple(predictions) def predict( self, X: ArrayLike, - aggregate_predictions: Optional[str] = None, + aggregate_predictions: Optional[str] = "mean", ) -> NDArray: """ Generates point predictions for the input data `X`: - - using the model fitted on the entire dataset - - or if aggregation_method is provided, aggregating predictions from - the models fitted on each fold + - aggregating predictions from the models fitted on each fold + - or using the model fitted on the entire dataset if aggregate_predictions=None Parameters ---------- X : ArrayLike Data features for generating point predictions. - aggregate_predictions : Optional[str], default=None + aggregate_predictions : Optional[str], default="mean" The method to aggregate predictions across folds. Options: - None: No aggregation, returns predictions from the estimator trained on the entire dataset @@ -542,17 +532,24 @@ def predict( NDArray Array of point predictions, with shape `(n_samples,)`. """ + ensemble = self._check_aggregate_predictions_and_return_ensemble( + aggregate_predictions + ) + predictions = self._mapie_regressor.predict( + X, alpha=None, ensemble=ensemble + ) + return cast_point_predictions_to_ndarray(predictions) + + def _check_aggregate_predictions_and_return_ensemble( + self, aggregate_predictions: Optional[str] + ) -> bool: if not aggregate_predictions: ensemble = False else: ensemble = True self._mapie_regressor._check_agg_function(aggregate_predictions) self._mapie_regressor.agg_function = aggregate_predictions - - predictions = self._mapie_regressor.predict( - X, alpha=None, ensemble=ensemble - ) - return cast_point_predictions_to_ndarray(predictions) + return ensemble class JackknifeAfterBootstrapRegressor: @@ -615,13 +612,6 @@ class JackknifeAfterBootstrapRegressor: A seed or random state instance to ensure reproducibility in any random operations within the regressor. - Returns - ------- - NDArray - An array containing the prediction intervals with shape - `(n_samples, 2)`, where each row represents the lower and - upper bounds for each sample. - Examples -------- >>> from mapie_v1.regression import JackknifeAfterBootstrapRegressor @@ -638,8 +628,7 @@ class JackknifeAfterBootstrapRegressor: ... resampling=25, ... ).fit(X, y).conformalize(X, y) - >>> prediction_points = mapie_regressor.predict(X_test) - >>> prediction_intervals = mapie_regressor.predict_set(X_test) + >>> predicted_points, predicted_intervals = mapie_regressor.predict_interval(X_test) """ _VALID_METHODS = ["plus", "minmax"] @@ -785,14 +774,16 @@ def conformalize( return self - def predict_set( + def predict_interval( self, X: ArrayLike, + ensemble: bool = True, minimize_interval_width: bool = False, allow_infinite_bounds: bool = False, - ) -> NDArray: + ) -> Tuple[NDArray, NDArray]: """ - Computes prediction intervals for each sample in `X` based on + Generates prediction intervals (and prediction points) for the input data `X` + based on conformity scores and confidence level(s), following the jackknife-after-bootstrap framework. Parameters @@ -800,6 +791,13 @@ def predict_set( X : ArrayLike Test data for prediction intervals. + ensemble : bool, default=True + If True, aggregates point predictions across models fitted on each + bootstrap samples, this is using the aggregation method defined + during the initialization of the model. + If False, returns predictions from the estimator trained on the + entire dataset. + minimize_interval_width : bool, default=False If True, minimizes the width of prediction intervals while maintaining coverage. @@ -810,37 +808,35 @@ def predict_set( Returns ------- - NDArray - Prediction intervals of shape (n_samples, 2), - with lower and upper bounds for each sample. + Tuple[NDArray, NDArray] + Two arrays: + - Prediction points, of shape `(n_samples,)` + - Prediction intervals, of shape `(n_samples, 2, n_confidence_levels)` """ - _, intervals = self._mapie_regressor.predict( + predictions = self._mapie_regressor.predict( X, alpha=self._alphas, optimize_beta=minimize_interval_width, - allow_infinite_bounds=allow_infinite_bounds - ) - - return make_intervals_single_if_single_alpha( - intervals, - self._alphas + allow_infinite_bounds=allow_infinite_bounds, + ensemble=ensemble, ) + return cast_predictions_to_ndarray_tuple(predictions) def predict( self, X: ArrayLike, - ensemble: bool = False, + ensemble: bool = True, ) -> NDArray: """ Generates point predictions for the input data using the fitted model, - with optional aggregation over bootstrap samples. + with aggregation over bootstrap samples. Parameters ---------- X : ArrayLike Data features for generating point predictions. - ensemble : bool, default=False + ensemble : bool, default=True If True, aggregates predictions across models fitted on each bootstrap samples, this is using the aggregation method defined during the initialization of the model. @@ -895,7 +891,7 @@ class ConformalizedQuantileRegressor: * ``median quantile = 0.5`` confidence_level : float default=0.9 - The confidence level(s) for the prediction intervals, indicating the + The confidence level for the prediction intervals, indicating the desired coverage probability of the prediction intervals. prefit : bool, default=False @@ -903,34 +899,6 @@ class ConformalizedQuantileRegressor: When set to `True`, the `fit` method cannot be called and the provided estimators should be pre-trained. - Methods - ------- - fit(X_train, y_train, fit_params=None) -> Self - Trains the base quantile regression estimator on the provided data. - Not applicable if `prefit=True`. - - conformalize(X_conformalize, y_conformalize, predict_params=None) -> Self - Calibrates the model on provided data, adjusting the prediction - intervals to achieve the specified confidence levels. - - predict(X) -> NDArray - Generates point predictions for the input data `X`. - - predict_set(X, - allow_infinite_bounds=False, - minimize_interval_width=False, - symmetric_intervals=True) -> NDArray - Generates prediction intervals for the input data `X`, - adjusted for desired scoverage based on the calibrated - quantile predictions. - - Returns - ------- - NDArray - An array containing the prediction intervals with shape - `(n_samples, 2)`, where each row represents the lower and - upper bounds for each sample. - Examples -------- >>> from mapie_v1.regression import ConformalizedQuantileRegressor @@ -949,8 +917,7 @@ class ConformalizedQuantileRegressor: ... confidence_level=0.95, ... ).fit(X_train, y_train).conformalize(X_conformalize, y_conformalize) - >>> prediction_points = mapie_regressor.predict(X_test) - >>> prediction_intervals = mapie_regressor.predict_set(X_test) + >>> predicted_points, predicted_intervals = mapie_regressor.predict_interval(X_test) """ def __init__( @@ -1068,16 +1035,17 @@ def conformalize( return self - def predict_set( + def predict_interval( self, X: ArrayLike, allow_infinite_bounds: bool = False, minimize_interval_width: bool = False, symmetric_intervals: bool = True, - ) -> NDArray: + ) -> Tuple[NDArray, NDArray]: """ - Computes prediction intervals for quantile regression based - on calibrated predictions. + Generates prediction intervals (and prediction points) for the input data `X` + based on conformity scores and confidence level(s), following + the conformalize quantile regression framework. Parameters ---------- @@ -1100,22 +1068,19 @@ def predict_set( Returns ------- - NDArray - Prediction intervals with shape `(n_samples, 2)`, with lower - and upper bounds for each sample. + Tuple[NDArray, NDArray] + Two arrays: + - Prediction points, of shape `(n_samples,)` + - Prediction intervals, of shape `(n_samples, 2, 1)` """ - _, intervals = self._mapie_quantile_regressor.predict( + predictions = self._mapie_quantile_regressor.predict( X, optimize_beta=minimize_interval_width, allow_infinite_bounds=allow_infinite_bounds, symmetry=symmetric_intervals, **self.predict_params ) - - return make_intervals_single_if_single_alpha( - intervals, - self._alpha - ) + return cast_predictions_to_ndarray_tuple(predictions) def predict( self,