Pushing the docs to dev/ for branch: main, commit f22ebd2548aa45d8da8689ccd2548a73557299fa

Author: sklearn-ci

dev/_downloads/07fcc19ba03226cd3d83d4e40ec44385/auto_examples_python.zip

@@ -79,7 +79,7 @@
 
 print("Learning the dictionary...")
 t0 = time()
-dico = MiniBatchDictionaryLearning(n_components=100, alpha=1, n_iter=500)
+dico = MiniBatchDictionaryLearning(n_components=50, alpha=1, n_iter=250)
 V = dico.fit(data).components_
 dt = time() - t0
 print("done in %.2fs." % dt)
@@ -139,7 +139,7 @@ def show_with_diff(image, reference, title):
 transform_algorithms = [
     ("Orthogonal Matching Pursuit\n1 atom", "omp", {"transform_n_nonzero_coefs": 1}),
     ("Orthogonal Matching Pursuit\n2 atoms", "omp", {"transform_n_nonzero_coefs": 2}),
-    ("Least-angle regression\n5 atoms", "lars", {"transform_n_nonzero_coefs": 5}),
+    ("Least-angle regression\n4 atoms", "lars", {"transform_n_nonzero_coefs": 4}),
     ("Thresholding\n alpha=0.1", "threshold", {"transform_alpha": 0.1}),
 ]
 

dev/_downloads/149ff4a0ff65a845f675cc7a0fcb86ea/plot_image_denoising.py

@@ -62,7 +62,7 @@ def f(x):
 all_models = {}
 common_params = dict(
     learning_rate=0.05,
-    n_estimators=250,
+    n_estimators=200,
     max_depth=2,
     min_samples_leaf=9,
     min_samples_split=9,
@@ -97,7 +97,7 @@ def f(x):
 fig = plt.figure(figsize=(10, 10))
 plt.plot(xx, f(xx), "g:", linewidth=3, label=r"$f(x) = x\,\sin(x)$")
 plt.plot(X_test, y_test, "b.", markersize=10, label="Test observations")
-plt.plot(xx, y_med, "r-", label="Predicted median", color="orange")
+plt.plot(xx, y_med, "r-", label="Predicted median")
 plt.plot(xx, y_pred, "r-", label="Predicted mean")
 plt.plot(xx, y_upper, "k-")
 plt.plot(xx, y_lower, "k-")
@@ -224,25 +224,24 @@ def coverage_fraction(y, y_low, y_high):
 # underfit and could not adapt to sinusoidal shape of the signal.
 #
 # The hyper-parameters of the model were approximately hand-tuned for the
-# median regressor and there is no reason than the same hyper-parameters are
+# median regressor and there is no reason that the same hyper-parameters are
 # suitable for the 5th percentile regressor.
 #
 # To confirm this hypothesis, we tune the hyper-parameters of a new regressor
 # of the 5th percentile by selecting the best model parameters by
 # cross-validation on the pinball loss with alpha=0.05:
 
 # %%
-from sklearn.model_selection import RandomizedSearchCV
+from sklearn.experimental import enable_halving_search_cv  # noqa
+from sklearn.model_selection import HalvingRandomSearchCV
 from sklearn.metrics import make_scorer
 from pprint import pprint
 
-
 param_grid = dict(
-    learning_rate=[0.01, 0.05, 0.1],
-    n_estimators=[100, 150, 200, 250, 300],
-    max_depth=[2, 5, 10, 15, 20],
-    min_samples_leaf=[1, 5, 10, 20, 30, 50],
-    min_samples_split=[2, 5, 10, 20, 30, 50],
+    learning_rate=[0.05, 0.1, 0.2],
+    max_depth=[2, 5, 10],
+    min_samples_leaf=[1, 5, 10, 20],
+    min_samples_split=[5, 10, 20, 30, 50],
 )
 alpha = 0.05
 neg_mean_pinball_loss_05p_scorer = make_scorer(
@@ -251,20 +250,22 @@ def coverage_fraction(y, y_low, y_high):
     greater_is_better=False,  # maximize the negative loss
 )
 gbr = GradientBoostingRegressor(loss="quantile", alpha=alpha, random_state=0)
-search_05p = RandomizedSearchCV(
+search_05p = HalvingRandomSearchCV(
     gbr,
     param_grid,
-    n_iter=10,  # increase this if computational budget allows
+    resource="n_estimators",
+    max_resources=250,
+    min_resources=50,
     scoring=neg_mean_pinball_loss_05p_scorer,
     n_jobs=2,
     random_state=0,
 ).fit(X_train, y_train)
 pprint(search_05p.best_params_)
 
 # %%
-# We observe that the search procedure identifies that deeper trees are needed
-# to get a good fit for the 5th percentile regressor. Deeper trees are more
-# expressive and less likely to underfit.
+# We observe that the hyper-parameters that were hand-tuned for the median
+# regressor are in the same range as the hyper-parameters suitable for the 5th
+# percentile regressor.
 #
 # Let's now tune the hyper-parameters for the 95th percentile regressor. We
 # need to redefine the `scoring` metric used to select the best model, along
@@ -286,15 +287,14 @@ def coverage_fraction(y, y_low, y_high):
 pprint(search_95p.best_params_)
 
 # %%
-# This time, shallower trees are selected and lead to a more constant piecewise
-# and therefore more robust estimation of the 95th percentile. This is
-# beneficial as it avoids overfitting the large outliers of the log-normal
-# additive noise.
-#
-# We can confirm this intuition by displaying the predicted 90% confidence
-# interval comprised by the predictions of those two tuned quantile regressors:
-# the prediction of the upper 95th percentile has a much coarser shape than the
-# prediction of the lower 5th percentile:
+# The result shows that the hyper-parameters for the 95th percentile regressor
+# identified by the search procedure are roughly in the same range as the hand-
+# tuned hyper-parameters for the median regressor and the hyper-parameters
+# identified by the search procedure for the 5th percentile regressor. However,
+# the hyper-parameter searches did lead to an improved 90% confidence interval
+# that is comprised by the predictions of those two tuned quantile regressors.
+# Note that the prediction of the upper 95th percentile has a much coarser shape
+# than the prediction of the lower 5th percentile because of the outliers:
 y_lower = search_05p.predict(xx)
 y_upper = search_95p.predict(xx)
 

dev/_downloads/2f3ef774a6d7e52e1e6b7ccbb75d25f0/plot_gradient_boosting_quantile.py

@@ -87,7 +87,7 @@
       },
       "outputs": [],
       "source": [
-        "from sklearn.ensemble import GradientBoostingRegressor\nfrom sklearn.metrics import mean_pinball_loss, mean_squared_error\n\n\nall_models = {}\ncommon_params = dict(\n    learning_rate=0.05,\n    n_estimators=250,\n    max_depth=2,\n    min_samples_leaf=9,\n    min_samples_split=9,\n)\nfor alpha in [0.05, 0.5, 0.95]:\n    gbr = GradientBoostingRegressor(loss=\"quantile\", alpha=alpha, **common_params)\n    all_models[\"q %1.2f\" % alpha] = gbr.fit(X_train, y_train)"
+        "from sklearn.ensemble import GradientBoostingRegressor\nfrom sklearn.metrics import mean_pinball_loss, mean_squared_error\n\n\nall_models = {}\ncommon_params = dict(\n    learning_rate=0.05,\n    n_estimators=200,\n    max_depth=2,\n    min_samples_leaf=9,\n    min_samples_split=9,\n)\nfor alpha in [0.05, 0.5, 0.95]:\n    gbr = GradientBoostingRegressor(loss=\"quantile\", alpha=alpha, **common_params)\n    all_models[\"q %1.2f\" % alpha] = gbr.fit(X_train, y_train)"
       ]
     },
     {
@@ -141,7 +141,7 @@
       },
       "outputs": [],
       "source": [
-        "import matplotlib.pyplot as plt\n\n\ny_pred = all_models[\"mse\"].predict(xx)\ny_lower = all_models[\"q 0.05\"].predict(xx)\ny_upper = all_models[\"q 0.95\"].predict(xx)\ny_med = all_models[\"q 0.50\"].predict(xx)\n\nfig = plt.figure(figsize=(10, 10))\nplt.plot(xx, f(xx), \"g:\", linewidth=3, label=r\"$f(x) = x\\,\\sin(x)$\")\nplt.plot(X_test, y_test, \"b.\", markersize=10, label=\"Test observations\")\nplt.plot(xx, y_med, \"r-\", label=\"Predicted median\", color=\"orange\")\nplt.plot(xx, y_pred, \"r-\", label=\"Predicted mean\")\nplt.plot(xx, y_upper, \"k-\")\nplt.plot(xx, y_lower, \"k-\")\nplt.fill_between(\n    xx.ravel(), y_lower, y_upper, alpha=0.4, label=\"Predicted 90% interval\"\n)\nplt.xlabel(\"$x$\")\nplt.ylabel(\"$f(x)$\")\nplt.ylim(-10, 25)\nplt.legend(loc=\"upper left\")\nplt.show()"
+        "import matplotlib.pyplot as plt\n\n\ny_pred = all_models[\"mse\"].predict(xx)\ny_lower = all_models[\"q 0.05\"].predict(xx)\ny_upper = all_models[\"q 0.95\"].predict(xx)\ny_med = all_models[\"q 0.50\"].predict(xx)\n\nfig = plt.figure(figsize=(10, 10))\nplt.plot(xx, f(xx), \"g:\", linewidth=3, label=r\"$f(x) = x\\,\\sin(x)$\")\nplt.plot(X_test, y_test, \"b.\", markersize=10, label=\"Test observations\")\nplt.plot(xx, y_med, \"r-\", label=\"Predicted median\")\nplt.plot(xx, y_pred, \"r-\", label=\"Predicted mean\")\nplt.plot(xx, y_upper, \"k-\")\nplt.plot(xx, y_lower, \"k-\")\nplt.fill_between(\n    xx.ravel(), y_lower, y_upper, alpha=0.4, label=\"Predicted 90% interval\"\n)\nplt.xlabel(\"$x$\")\nplt.ylabel(\"$f(x)$\")\nplt.ylim(-10, 25)\nplt.legend(loc=\"upper left\")\nplt.show()"
       ]
     },
     {
@@ -220,7 +220,7 @@
       "cell_type": "markdown",
       "metadata": {},
       "source": [
-        "On the test set, the estimated confidence interval is slightly too narrow.\nNote, however, that we would need to wrap those metrics in a cross-validation\nloop to assess their variability under data resampling.\n\n## Tuning the hyper-parameters of the quantile regressors\n\nIn the plot above, we observed that the 5th percentile regressor seems to\nunderfit and could not adapt to sinusoidal shape of the signal.\n\nThe hyper-parameters of the model were approximately hand-tuned for the\nmedian regressor and there is no reason than the same hyper-parameters are\nsuitable for the 5th percentile regressor.\n\nTo confirm this hypothesis, we tune the hyper-parameters of a new regressor\nof the 5th percentile by selecting the best model parameters by\ncross-validation on the pinball loss with alpha=0.05:\n\n"
+        "On the test set, the estimated confidence interval is slightly too narrow.\nNote, however, that we would need to wrap those metrics in a cross-validation\nloop to assess their variability under data resampling.\n\n## Tuning the hyper-parameters of the quantile regressors\n\nIn the plot above, we observed that the 5th percentile regressor seems to\nunderfit and could not adapt to sinusoidal shape of the signal.\n\nThe hyper-parameters of the model were approximately hand-tuned for the\nmedian regressor and there is no reason that the same hyper-parameters are\nsuitable for the 5th percentile regressor.\n\nTo confirm this hypothesis, we tune the hyper-parameters of a new regressor\nof the 5th percentile by selecting the best model parameters by\ncross-validation on the pinball loss with alpha=0.05:\n\n"
       ]
     },
     {
@@ -231,14 +231,14 @@
       },
       "outputs": [],
       "source": [
-        "from sklearn.model_selection import RandomizedSearchCV\nfrom sklearn.metrics import make_scorer\nfrom pprint import pprint\n\n\nparam_grid = dict(\n    learning_rate=[0.01, 0.05, 0.1],\n    n_estimators=[100, 150, 200, 250, 300],\n    max_depth=[2, 5, 10, 15, 20],\n    min_samples_leaf=[1, 5, 10, 20, 30, 50],\n    min_samples_split=[2, 5, 10, 20, 30, 50],\n)\nalpha = 0.05\nneg_mean_pinball_loss_05p_scorer = make_scorer(\n    mean_pinball_loss,\n    alpha=alpha,\n    greater_is_better=False,  # maximize the negative loss\n)\ngbr = GradientBoostingRegressor(loss=\"quantile\", alpha=alpha, random_state=0)\nsearch_05p = RandomizedSearchCV(\n    gbr,\n    param_grid,\n    n_iter=10,  # increase this if computational budget allows\n    scoring=neg_mean_pinball_loss_05p_scorer,\n    n_jobs=2,\n    random_state=0,\n).fit(X_train, y_train)\npprint(search_05p.best_params_)"
+        "from sklearn.experimental import enable_halving_search_cv  # noqa\nfrom sklearn.model_selection import HalvingRandomSearchCV\nfrom sklearn.metrics import make_scorer\nfrom pprint import pprint\n\nparam_grid = dict(\n    learning_rate=[0.05, 0.1, 0.2],\n    max_depth=[2, 5, 10],\n    min_samples_leaf=[1, 5, 10, 20],\n    min_samples_split=[5, 10, 20, 30, 50],\n)\nalpha = 0.05\nneg_mean_pinball_loss_05p_scorer = make_scorer(\n    mean_pinball_loss,\n    alpha=alpha,\n    greater_is_better=False,  # maximize the negative loss\n)\ngbr = GradientBoostingRegressor(loss=\"quantile\", alpha=alpha, random_state=0)\nsearch_05p = HalvingRandomSearchCV(\n    gbr,\n    param_grid,\n    resource=\"n_estimators\",\n    max_resources=250,\n    min_resources=50,\n    scoring=neg_mean_pinball_loss_05p_scorer,\n    n_jobs=2,\n    random_state=0,\n).fit(X_train, y_train)\npprint(search_05p.best_params_)"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {},
       "source": [
-        "We observe that the search procedure identifies that deeper trees are needed\nto get a good fit for the 5th percentile regressor. Deeper trees are more\nexpressive and less likely to underfit.\n\nLet's now tune the hyper-parameters for the 95th percentile regressor. We\nneed to redefine the `scoring` metric used to select the best model, along\nwith adjusting the alpha parameter of the inner gradient boosting estimator\nitself:\n\n"
+        "We observe that the hyper-parameters that were hand-tuned for the median\nregressor are in the same range as the hyper-parameters suitable for the 5th\npercentile regressor.\n\nLet's now tune the hyper-parameters for the 95th percentile regressor. We\nneed to redefine the `scoring` metric used to select the best model, along\nwith adjusting the alpha parameter of the inner gradient boosting estimator\nitself:\n\n"
       ]
     },
     {
@@ -256,7 +256,7 @@
       "cell_type": "markdown",
       "metadata": {},
       "source": [
-        "This time, shallower trees are selected and lead to a more constant piecewise\nand therefore more robust estimation of the 95th percentile. This is\nbeneficial as it avoids overfitting the large outliers of the log-normal\nadditive noise.\n\nWe can confirm this intuition by displaying the predicted 90% confidence\ninterval comprised by the predictions of those two tuned quantile regressors:\nthe prediction of the upper 95th percentile has a much coarser shape than the\nprediction of the lower 5th percentile:\n\n"
+        "The result shows that the hyper-parameters for the 95th percentile regressor\nidentified by the search procedure are roughly in the same range as the hand-\ntuned hyper-parameters for the median regressor and the hyper-parameters\nidentified by the search procedure for the 5th percentile regressor. However,\nthe hyper-parameter searches did lead to an improved 90% confidence interval\nthat is comprised by the predictions of those two tuned quantile regressors.\nNote that the prediction of the upper 95th percentile has a much coarser shape\nthan the prediction of the lower 5th percentile because of the outliers:\n\n"
       ]
     },
     {