REFACTOR: restucture the MapieQuantileRegressor Fit - Split the fit into prefit_estimators, fit_estimators and conformalize

.gitignore

@@ -49,7 +49,7 @@ var/
 pip-log.txt
 pip-delete-this-directory.txt
 
-# Unit test / coverage reports
+# Tests / coverage reports
 htmlcov/
 .tox/
 .coverage
@@ -59,6 +59,7 @@ nosetests.xml
 coverage.xml
 *,cover
 .hypothesis/
+mapie_v1/integration_tests/mapie_v0_package
 
 # Translations
 *.mo

Makefile

@@ -1,14 +1,25 @@
 .PHONY: tests doc build
 
+mapie_v0_folder_name = mapie_v0_package
+
 lint:
 	flake8 . --exclude=doc
 
 type-check:
 	mypy mapie
 
+v1-type-check:
+	mypy mapie_v1 --exclude $(mapie_v0_folder_name)
+
 tests:
 	pytest -vs --doctest-modules mapie
 
+integration-tests-v1:
+	@pip install mapie --no-dependencies --target=./mapie_v1/integration_tests/$(mapie_v0_folder_name) >/dev/null 2>&1
+	@mv ./mapie_v1/integration_tests/$(mapie_v0_folder_name)/mapie ./mapie_v1/integration_tests/$(mapie_v0_folder_name)/mapiev0
+	@- export PYTHONPATH="${PYTHONPATH}:./mapie_v1/integration_tests/$(mapie_v0_folder_name)"; pytest -vs mapie_v1/integration_tests/tests  -k $(pattern)
+	@mv ./mapie_v1/integration_tests/$(mapie_v0_folder_name)/mapiev0 ./mapie_v1/integration_tests/$(mapie_v0_folder_name)/mapie
+
 coverage:
 	pytest -vs \
 		--doctest-modules \

doc/api.rst

@@ -2,18 +2,28 @@
 MAPIE API
 #########
 
-.. currentmodule:: mapie
+Regression V1 (from mapie_v1)
+=============================
 
-Regression
-==========
+.. autosummary::
+   :toctree: generated/
+   :template: class.rst
+
+   mapie_v1.regression.SplitConformalRegressor
+   mapie_v1.regression.CrossConformalRegressor
+   mapie_v1.regression.JackknifeAfterBootstrapRegressor
+   mapie_v1.regression.ConformalizedQuantileRegressor
+
+Regression (from mapie)
+=======================
 
 .. autosummary::
    :toctree: generated/
    :template: class.rst
 
-   regression.MapieRegressor
-   regression.MapieQuantileRegressor
-   regression.MapieTimeSeriesRegressor
+   mapie.regression.MapieRegressor
+   mapie.regression.MapieQuantileRegressor
+   mapie.regression.MapieTimeSeriesRegressor
 
 Classification
 ==============
@@ -22,7 +32,7 @@ Classification
    :toctree: generated/
    :template: class.rst
 
-   classification.MapieClassifier
+   mapie.classification.MapieClassifier
 
 Multi-Label Classification
 ==========================
@@ -31,7 +41,7 @@ Multi-Label Classification
    :toctree: generated/
    :template: class.rst
 
-   multi_label_classification.MapieMultiLabelClassifier
+   mapie.multi_label_classification.MapieMultiLabelClassifier
 
 Calibration
 ===========
@@ -40,64 +50,64 @@ Calibration
    :toctree: generated/
    :template: class.rst
 
-   calibration.MapieCalibrator
+   mapie.calibration.MapieCalibrator
 
 Metrics
 =======
 
 .. autosummary::
    :toctree: generated/
    :template: function.rst
-   
-   metrics.classification_coverage_score
-   metrics.classification_coverage_score_v2
-   metrics.classification_mean_width_score
-   metrics.classification_ssc
-   metrics.classification_ssc_score
-   metrics.cumulative_differences
-   metrics.expected_calibration_error
-   metrics.hsic
-   metrics.kolmogorov_smirnov_cdf
-   metrics.kolmogorov_smirnov_p_value
-   metrics.kolmogorov_smirnov_statistic
-   metrics.kuiper_cdf
-   metrics.kuiper_p_value
-   metrics.kuiper_statistic
-   metrics.length_scale
-   metrics.regression_coverage_score
-   metrics.regression_coverage_score_v2
-   metrics.regression_mean_width_score
-   metrics.regression_ssc
-   metrics.regression_ssc_score
-   metrics.spiegelhalter_p_value
-   metrics.spiegelhalter_statistic
-   metrics.top_label_ece
-
-Conformity scores (regression)
+
+   mapie.metrics.classification_coverage_score
+   mapie.metrics.classification_coverage_score_v2
+   mapie.metrics.classification_mean_width_score
+   mapie.metrics.classification_ssc
+   mapie.metrics.classification_ssc_score
+   mapie.metrics.cumulative_differences
+   mapie.metrics.expected_calibration_error
+   mapie.metrics.hsic
+   mapie.metrics.kolmogorov_smirnov_cdf
+   mapie.metrics.kolmogorov_smirnov_p_value
+   mapie.metrics.kolmogorov_smirnov_statistic
+   mapie.metrics.kuiper_cdf
+   mapie.metrics.kuiper_p_value
+   mapie.metrics.kuiper_statistic
+   mapie.metrics.length_scale
+   mapie.metrics.regression_coverage_score
+   mapie.metrics.regression_coverage_score_v2
+   mapie.metrics.regression_mean_width_score
+   mapie.metrics.regression_ssc
+   mapie.metrics.regression_ssc_score
+   mapie.metrics.spiegelhalter_p_value
+   mapie.metrics.spiegelhalter_statistic
+   mapie.metrics.top_label_ece
+
+Conformity Scores (Regression)
 ==============================
 
 .. autosummary::
    :toctree: generated/
    :template: class.rst
 
-   conformity_scores.BaseRegressionScore
-   conformity_scores.AbsoluteConformityScore
-   conformity_scores.GammaConformityScore
-   conformity_scores.ResidualNormalisedScore
+   mapie.conformity_scores.BaseRegressionScore
+   mapie.conformity_scores.AbsoluteConformityScore
+   mapie.conformity_scores.GammaConformityScore
+   mapie.conformity_scores.ResidualNormalisedScore
 
-Conformity scores (classification)
+Conformity Scores (Classification)
 ==================================
 
 .. autosummary::
    :toctree: generated/
    :template: class.rst
 
-   conformity_scores.BaseClassificationScore
-   conformity_scores.NaiveConformityScore
-   conformity_scores.LACConformityScore
-   conformity_scores.APSConformityScore
-   conformity_scores.RAPSConformityScore
-   conformity_scores.TopKConformityScore
+   mapie.conformity_scores.BaseClassificationScore
+   mapie.conformity_scores.NaiveConformityScore
+   mapie.conformity_scores.LACConformityScore
+   mapie.conformity_scores.APSConformityScore
+   mapie.conformity_scores.RAPSConformityScore
+   mapie.conformity_scores.TopKConformityScore
 
 Resampling
 ==========
@@ -106,15 +116,14 @@ Resampling
    :toctree: generated/
    :template: class.rst
 
-   subsample.BlockBootstrap
-   subsample.Subsample
-
+   mapie.subsample.BlockBootstrap
+   mapie.subsample.Subsample
 
 Mondrian
-==========
+========
 
 .. autosummary::
    :toctree: generated/
    :template: class.rst
 
-   mondrian.MondrianCP
+   mapie.mondrian.MondrianCP

doc/choosing_the_right_algorithm.rst

@@ -0,0 +1,20 @@
+############################################
+Choosing the right algorithm
+############################################
+
+Following are simple decision trees to help you you getting started quickly with MAPIE.
+
+Reality is of course a bit more complex, so feel free to browse the documentation for nuanced explanations.
+
+
+1. Regression
+==================================
+
+.. image:: images/decision_tree_regression.png
+    :align: center
+
+2. Classification
+=====================
+
+.. image:: images/decision_tree_classification.png
+    :align: center

doc/choosing_the_right_algorithm_classification.rst

@@ -0,0 +1,10 @@
+############################################
+Choosing the right algorithm
+############################################
+
+Following is a simple decision tree to help you you getting started quickly with MAPIE.
+
+Reality is of course a bit more complex, so feel free to browse the documentation for nuanced explanations.
+
+.. image:: images/decision_tree_classification.png
+    :align: center

doc/choosing_the_right_algorithm_regression.rst

@@ -0,0 +1,10 @@
+############################################
+Choosing the right algorithm
+############################################
+
+Following is a simple decision trees to help you you getting started quickly with MAPIE.
+
+Reality is of course a bit more complex, so feel free to browse the documentation for nuanced explanations.
+
+.. image:: images/decision_tree_regression.png
+    :align: center

doc/conf.py

@@ -146,7 +146,9 @@
 # Theme options are theme-specific and customize the look and feel of a theme
 # further.  For a list of options available for each theme, see the
 # documentation.
-# html_theme_options = {}
+html_theme_options = {
+    'collapse_navigation': False,
+}
 
 # Add any paths that contain custom themes here, relative to this directory.
 html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]

doc/index.rst

@@ -7,43 +7,21 @@
 
    quick_start
    split_cross_conformal
+   choosing_the_right_algorithm
+   v1_migration_guide
 
 .. toctree::
    :maxdepth: 2
    :hidden:
-   :caption: REGRESSION
+   :caption: Measure predictions uncertainty
 
-   theoretical_description_regression
-   theoretical_description_conformity_scores
-   examples_regression/4-tutorials/plot_main-tutorial-regression
-   examples_regression/4-tutorials/plot_cqr_tutorial
-   examples_regression/4-tutorials/plot_ts-tutorial
-   examples_regression/index
-   notebooks_regression
+   index_regression
+   index_classification
 
 .. toctree::
    :maxdepth: 2
    :hidden:
-   :caption: CLASSIFICATION
-
-   theoretical_description_classification
-   examples_classification/4-tutorials/plot_main-tutorial-classification
-   examples_classification/4-tutorials/plot_crossconformal
-   examples_classification/index
-   notebooks_classification
-
-.. toctree::
-   :maxdepth: 2
-   :hidden:
-   :caption: BINARY CLASSIFICATION
-
-   theoretical_description_binary_classification
-   examples_classification/4-tutorials/plot_main-tutorial-binary-classification
-
-.. toctree::
-   :maxdepth: 2
-   :hidden:
-   :caption: MULTI-LABEL CLASSIFICATION
+   :caption: Control prediction errors
 
    theoretical_description_multilabel_classification
    examples_multilabel_classification/1-quickstart/plot_tutorial_multilabel_classification
@@ -52,15 +30,7 @@
 .. toctree::
    :maxdepth: 2
    :hidden:
-   :caption: MONDRIAN
-
-   theoretical_description_mondrian
-   examples_mondrian/1-quickstart/plot_main-tutorial-mondrian-regression
-
-.. toctree::
-   :maxdepth: 2
-   :hidden:
-   :caption: CALIBRATION
+   :caption: Calibrate multi-class predictions
 
    theoretical_description_calibration
    examples_calibration/index
@@ -69,9 +39,10 @@
 .. toctree::
    :maxdepth: 2
    :hidden:
-   :caption: METRICS
+   :caption: Question & Answers
 
-   theoretical_description_metrics
+   Metrics: how to measure conformal prediction performance? <theoretical_description_metrics>
+   index_mondrian
 
 .. toctree::
    :maxdepth: 2

doc/index_binary_classification.rst

@@ -0,0 +1,8 @@
+The binary classification case
+================================
+
+.. toctree::
+   :maxdepth: 2
+
+   examples_classification/4-tutorials/plot_main-tutorial-binary-classification
+   theoretical_description_binary_classification

doc/index_classification.rst

@@ -0,0 +1,13 @@
+Prediction sets (classification)
+================================
+
+.. toctree::
+   :maxdepth: 2
+
+   choosing_the_right_algorithm_classification
+   examples_classification/4-tutorials/plot_main-tutorial-classification
+   examples_classification/4-tutorials/plot_crossconformal
+   examples_classification/index
+   notebooks_classification
+   theoretical_description_classification
+   index_binary_classification

doc/index_mondrian.rst

@@ -0,0 +1,8 @@
+Mondrian: how to use prior knowledge on groups when measuring uncertainty?
+==============================================================================
+
+.. toctree::
+   :maxdepth: 2
+
+   theoretical_description_mondrian
+   examples_mondrian/1-quickstart/plot_main-tutorial-mondrian-regression