Pushing the docs to dev/ for branch: master, commit 104b736d4446d1d3a996bceb8340776dff1882a5

Author: sklearn-ci

dev/_downloads/3409d9766d352cc9f9b169d4a799a87a/auto_examples_python.zip

@@ -6,25 +6,25 @@
 This example demonstrates Gradient Boosting to produce a predictive
 model from an ensemble of weak predictive models. Gradient boosting can be used
 for regression and classification problems. Here, we will train a model to
-tackle a diabetes regression task.
-
-We will obtain the results from
+tackle a diabetes regression task. We will obtain the results from
 :class:`~sklearn.ensemble.GradientBoostingRegressor` with least squares loss
 and 500 regression trees of depth 4.
 
+Note: For larger datasets (n_samples >= 10000), please refer to
+:class:`sklearn.ensemble.HistGradientBoostingRegressor`
 """
 print(__doc__)
 
 # Author: Peter Prettenhofer <[email protected]>
 #         Maria Telenczuk <https://github.com/maikia>
+#         Katrina Ni <https://github.com/nilichen>
 #
 # License: BSD 3 clause
 
-import numpy as np
 import matplotlib.pyplot as plt
-
-from sklearn import ensemble
-from sklearn import datasets
+import numpy as np
+from sklearn import datasets, ensemble
+from sklearn.inspection import permutation_importance
 from sklearn.metrics import mean_squared_error
 from sklearn.model_selection import train_test_split
 
@@ -47,23 +47,25 @@
 # regression model. You can play with those parameters to see how the
 # results change:
 #
-# Here:
-# n_estimators : is the number of boosting stages which will be performed.
-#     Later, we will plot and see how the deviance changes with those boosting
-#     operations.
-# max_depth : this limits the number of nodes in the tree. The best value
-#     depends on the interaction of the input variables.
-# min_samples_split : is the minimum number of samples required to split an
-#     internal node.
-# learning_rate: tells how much the contribution of each tree will shrink
-# loss: here, we decided to use least squeares as a loss function, however
-#     there are many other options (check
-#     :class:`~sklearn.ensemble.GradientBoostingRegressor` to see what are
-#     other possibilities)
-
-X_train, X_test, y_train, y_test = train_test_split(X, y,
-                                                    test_size=0.1,
-                                                    random_state=13)
+# n_estimators : the number of boosting stages which will be performed.
+# Later, we will plot and see how the deviance changes with those boosting
+# operations.
+#
+# max_depth : limits the number of nodes in the tree.
+# The best value depends on the interaction of the input variables.
+#
+# min_samples_split : the minimum number of samples required to split an
+# internal node.
+#
+# learning_rate : how much the contribution of each tree will shrink
+#
+# loss : here, we decided to use least squeares as a loss function.
+# However there are many other options (check
+# :class:`~sklearn.ensemble.GradientBoostingRegressor` to see what are
+# other possibilities)
+
+X_train, X_test, y_train, y_test = train_test_split(
+    X, y, test_size=0.1, random_state=13)
 
 params = {'n_estimators': 500,
           'max_depth': 4,
@@ -92,13 +94,11 @@
 # test set deviance and then plot it.
 
 test_score = np.zeros((params['n_estimators'],), dtype=np.float64)
-
 for i, y_pred in enumerate(clf.staged_predict(X_test)):
     test_score[i] = clf.loss_(y_test, y_pred)
 
-fig = plt.figure(figsize=(12, 8))
-
-plt.subplot(1, 2, 1)
+fig = plt.figure(figsize=(6, 6))
+plt.subplot(1, 1, 1)
 plt.title('Deviance')
 plt.plot(np.arange(params['n_estimators']) + 1, clf.train_score_, 'b-',
          label='Training Set Deviance')
@@ -107,25 +107,39 @@
 plt.legend(loc='upper right')
 plt.xlabel('Boosting Iterations')
 plt.ylabel('Deviance')
+fig.tight_layout()
+plt.show()
 
 ##############################################################################
-# Plot impurity-based feature importance
+# Plot feature importance
 # -------------------------------------
 #
 # Careful, impurity-based feature importances can be misleading for
-# high cardinality features (many unique values). See
-# :func:`sklearn.inspection.permutation_importance` as an alternative.
+# high cardinality features (many unique values). As an alternative,
+# the permutation importances of ``clf`` are computed on a
+# held out test set. See :ref:`permutation_importance` for more details.
+#
+# In this case, the two methods agree to identify the same top 2 features
+# as strongly predictive features but not in the same order. The third most
+# predictive feature, "bp", is also the same for the 2 methods. The remaining
+# features are less predictive and the error bars of the permutation plot
+# show that they overlap with 0.
 
 feature_importance = clf.feature_importances_
-# make importances relative to max importance
-feature_importance = 100.0 * (feature_importance / feature_importance.max())
 sorted_idx = np.argsort(feature_importance)
 pos = np.arange(sorted_idx.shape[0]) + .5
-plt.subplot(1, 2, 2)
+fig = plt.figure(figsize=(12, 6))
+plt.subplot(1, 2, 1)
 plt.barh(pos, feature_importance[sorted_idx], align='center')
 plt.yticks(pos, np.array(diabetes.feature_names)[sorted_idx])
-plt.xlabel('Relative Importance')
-plt.title('Variable Importance')
-fig.tight_layout()
+plt.title('Feature Importance (MDI)')
 
+result = permutation_importance(clf, X_test, y_test, n_repeats=10,
+                                random_state=42, n_jobs=2)
+sorted_idx = result.importances_mean.argsort()
+plt.subplot(1, 2, 2)
+plt.boxplot(result.importances[sorted_idx].T,
+            vert=False, labels=np.array(diabetes.feature_names)[sorted_idx])
+plt.title("Permutation Importance (test set)")
+fig.tight_layout()
 plt.show()

dev/_downloads/679566501b743cb339497968edb9d62f/plot_gradient_boosting_regression.py

@@ -15,7 +15,7 @@
       "cell_type": "markdown",
       "metadata": {},
       "source": [
-        "\n# Gradient Boosting regression\n\n\nThis example demonstrates Gradient Boosting to produce a predictive\nmodel from an ensemble of weak predictive models. Gradient boosting can be used\nfor regression and classification problems. Here, we will train a model to\ntackle a diabetes regression task.\n\nWe will obtain the results from\n:class:`~sklearn.ensemble.GradientBoostingRegressor` with least squares loss\nand 500 regression trees of depth 4.\n"
+        "\n# Gradient Boosting regression\n\n\nThis example demonstrates Gradient Boosting to produce a predictive\nmodel from an ensemble of weak predictive models. Gradient boosting can be used\nfor regression and classification problems. Here, we will train a model to\ntackle a diabetes regression task. We will obtain the results from\n:class:`~sklearn.ensemble.GradientBoostingRegressor` with least squares loss\nand 500 regression trees of depth 4.\n\nNote: For larger datasets (n_samples >= 10000), please refer to\n:class:`sklearn.ensemble.HistGradientBoostingRegressor`\n"
       ]
     },
     {
@@ -26,7 +26,7 @@
       },
       "outputs": [],
       "source": [
-        "print(__doc__)\n\n# Author: Peter Prettenhofer <[email protected]>\n#         Maria Telenczuk <https://github.com/maikia>\n#\n# License: BSD 3 clause\n\nimport numpy as np\nimport matplotlib.pyplot as plt\n\nfrom sklearn import ensemble\nfrom sklearn import datasets\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.model_selection import train_test_split"
+        "print(__doc__)\n\n# Author: Peter Prettenhofer <[email protected]>\n#         Maria Telenczuk <https://github.com/maikia>\n#         Katrina Ni <https://github.com/nilichen>\n#\n# License: BSD 3 clause\n\nimport matplotlib.pyplot as plt\nimport numpy as np\nfrom sklearn import datasets, ensemble\nfrom sklearn.inspection import permutation_importance\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.model_selection import train_test_split"
       ]
     },
     {
@@ -51,7 +51,7 @@
       "cell_type": "markdown",
       "metadata": {},
       "source": [
-        "Data preprocessing\n-------------------------------------\n\nNext, we will split our dataset to use 90% for training and leave the rest\nfor testing. We will also prepare the parameters we want to use to fit our\nregression model. You can play with those parameters to see how the\nresults change:\n\nHere:\nn_estimators : is the number of boosting stages which will be performed.\n    Later, we will plot and see how the deviance changes with those boosting\n    operations.\nmax_depth : this limits the number of nodes in the tree. The best value\n    depends on the interaction of the input variables.\nmin_samples_split : is the minimum number of samples required to split an\n    internal node.\nlearning_rate: tells how much the contribution of each tree will shrink\nloss: here, we decided to use least squeares as a loss function, however\n    there are many other options (check\n    :class:`~sklearn.ensemble.GradientBoostingRegressor` to see what are\n    other possibilities)\n\n"
+        "Data preprocessing\n-------------------------------------\n\nNext, we will split our dataset to use 90% for training and leave the rest\nfor testing. We will also prepare the parameters we want to use to fit our\nregression model. You can play with those parameters to see how the\nresults change:\n\nn_estimators : the number of boosting stages which will be performed.\nLater, we will plot and see how the deviance changes with those boosting\noperations.\n\nmax_depth : limits the number of nodes in the tree.\nThe best value depends on the interaction of the input variables.\n\nmin_samples_split : the minimum number of samples required to split an\ninternal node.\n\nlearning_rate : how much the contribution of each tree will shrink\n\nloss : here, we decided to use least squeares as a loss function.\nHowever there are many other options (check\n:class:`~sklearn.ensemble.GradientBoostingRegressor` to see what are\nother possibilities)\n\n"
       ]
     },
     {
@@ -62,7 +62,7 @@
       },
       "outputs": [],
       "source": [
-        "X_train, X_test, y_train, y_test = train_test_split(X, y,\n                                                    test_size=0.1,\n                                                    random_state=13)\n\nparams = {'n_estimators': 500,\n          'max_depth': 4,\n          'min_samples_split': 5,\n          'learning_rate': 0.01,\n          'loss': 'ls'}"
+        "X_train, X_test, y_train, y_test = train_test_split(\n    X, y, test_size=0.1, random_state=13)\n\nparams = {'n_estimators': 500,\n          'max_depth': 4,\n          'min_samples_split': 5,\n          'learning_rate': 0.01,\n          'loss': 'ls'}"
       ]
     },
     {
@@ -98,14 +98,14 @@
       },
       "outputs": [],
       "source": [
-        "test_score = np.zeros((params['n_estimators'],), dtype=np.float64)\n\nfor i, y_pred in enumerate(clf.staged_predict(X_test)):\n    test_score[i] = clf.loss_(y_test, y_pred)\n\nfig = plt.figure(figsize=(12, 8))\n\nplt.subplot(1, 2, 1)\nplt.title('Deviance')\nplt.plot(np.arange(params['n_estimators']) + 1, clf.train_score_, 'b-',\n         label='Training Set Deviance')\nplt.plot(np.arange(params['n_estimators']) + 1, test_score, 'r-',\n         label='Test Set Deviance')\nplt.legend(loc='upper right')\nplt.xlabel('Boosting Iterations')\nplt.ylabel('Deviance')"
+        "test_score = np.zeros((params['n_estimators'],), dtype=np.float64)\nfor i, y_pred in enumerate(clf.staged_predict(X_test)):\n    test_score[i] = clf.loss_(y_test, y_pred)\n\nfig = plt.figure(figsize=(6, 6))\nplt.subplot(1, 1, 1)\nplt.title('Deviance')\nplt.plot(np.arange(params['n_estimators']) + 1, clf.train_score_, 'b-',\n         label='Training Set Deviance')\nplt.plot(np.arange(params['n_estimators']) + 1, test_score, 'r-',\n         label='Test Set Deviance')\nplt.legend(loc='upper right')\nplt.xlabel('Boosting Iterations')\nplt.ylabel('Deviance')\nfig.tight_layout()\nplt.show()"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {},
       "source": [
-        "Plot impurity-based feature importance\n-------------------------------------\n\nCareful, impurity-based feature importances can be misleading for\nhigh cardinality features (many unique values). See\n:func:`sklearn.inspection.permutation_importance` as an alternative.\n\n"
+        "Plot feature importance\n-------------------------------------\n\nCareful, impurity-based feature importances can be misleading for\nhigh cardinality features (many unique values). As an alternative,\nthe permutation importances of ``clf`` are computed on a\nheld out test set. See `permutation_importance` for more details.\n\nIn this case, the two methods agree to identify the same top 2 features\nas strongly predictive features but not in the same order. The third most\npredictive feature, \"bp\", is also the same for the 2 methods. The remaining\nfeatures are less predictive and the error bars of the permutation plot\nshow that they overlap with 0.\n\n"
       ]
     },
     {
@@ -116,7 +116,7 @@
       },
       "outputs": [],
       "source": [
-        "feature_importance = clf.feature_importances_\n# make importances relative to max importance\nfeature_importance = 100.0 * (feature_importance / feature_importance.max())\nsorted_idx = np.argsort(feature_importance)\npos = np.arange(sorted_idx.shape[0]) + .5\nplt.subplot(1, 2, 2)\nplt.barh(pos, feature_importance[sorted_idx], align='center')\nplt.yticks(pos, np.array(diabetes.feature_names)[sorted_idx])\nplt.xlabel('Relative Importance')\nplt.title('Variable Importance')\nfig.tight_layout()\n\nplt.show()"
+        "feature_importance = clf.feature_importances_\nsorted_idx = np.argsort(feature_importance)\npos = np.arange(sorted_idx.shape[0]) + .5\nfig = plt.figure(figsize=(12, 6))\nplt.subplot(1, 2, 1)\nplt.barh(pos, feature_importance[sorted_idx], align='center')\nplt.yticks(pos, np.array(diabetes.feature_names)[sorted_idx])\nplt.title('Feature Importance (MDI)')\n\nresult = permutation_importance(clf, X_test, y_test, n_repeats=10,\n                                random_state=42, n_jobs=2)\nsorted_idx = result.importances_mean.argsort()\nplt.subplot(1, 2, 2)\nplt.boxplot(result.importances[sorted_idx].T,\n            vert=False, labels=np.array(diabetes.feature_names)[sorted_idx])\nplt.title(\"Permutation Importance (test set)\")\nfig.tight_layout()\nplt.show()"
       ]
     }
   ],