Pushing the docs to dev/ for branch: master, commit b65c53d292097bf2382344cfdab8e7c126d215df

Author: sklearn-ci

dev/_downloads/3409d9766d352cc9f9b169d4a799a87a/auto_examples_python.zip

@@ -3,14 +3,20 @@
 Gradient Boosting regression
 ============================
 
-Demonstrate Gradient Boosting on the Boston housing dataset.
+This example demonstrates Gradient Boosting to produce a predictive
+model from an ensemble of weak predictive models. Gradient boosting can be used
+for regression and classification problems. Here, we will train a model to
+tackle a diabetes regression task.
+
+We will obtain the results from
+:class:`~sklearn.ensemble.GradientBoostingRegressor` with least squares loss
+and 500 regression trees of depth 4.
 
-This example fits a Gradient Boosting model with least squares loss and
-500 regression trees of depth 4.
 """
 print(__doc__)
 
 # Author: Peter Prettenhofer <[email protected]>
+#         Maria Telenczuk <https://github.com/maikia>
 #
 # License: BSD 3 clause
 
@@ -19,38 +25,79 @@
 
 from sklearn import ensemble
 from sklearn import datasets
-from sklearn.utils import shuffle
 from sklearn.metrics import mean_squared_error
+from sklearn.model_selection import train_test_split
+
+##############################################################################
+# Load the data
+# -------------------------------------
+#
+# First we need to load the data. We set random state to be consistent with the
+# result.
+
+diabetes = datasets.load_diabetes()
+X, y = diabetes.data, diabetes.target
+
+##############################################################################
+# Data preprocessing
+# -------------------------------------
+#
+# Next, we will split our dataset to use 90% for training and leave the rest
+# for testing. We will also prepare the parameters we want to use to fit our
+# regression model. You can play with those parameters to see how the
+# results change:
+#
+# Here:
+# n_estimators : is the number of boosting stages which will be performed.
+#     Later, we will plot and see how the deviance changes with those boosting
+#     operations.
+# max_depth : this limits the number of nodes in the tree. The best value
+#     depends on the interaction of the input variables.
+# min_samples_split : is the minimum number of samples required to split an
+#     internal node.
+# learning_rate: tells how much the contribution of each tree will shrink
+# loss: here, we decided to use least squeares as a loss function, however
+#     there are many other options (check
+#     :class:`~sklearn.ensemble.GradientBoostingRegressor` to see what are
+#     other possibilities)
+
+X_train, X_test, y_train, y_test = train_test_split(X, y,
+                                                    test_size=0.1,
+                                                    random_state=13)
 
-# #############################################################################
-# Load data
-boston = datasets.load_boston()
-X, y = shuffle(boston.data, boston.target, random_state=13)
-X = X.astype(np.float32)
-offset = int(X.shape[0] * 0.9)
-X_train, y_train = X[:offset], y[:offset]
-X_test, y_test = X[offset:], y[offset:]
+params = {'n_estimators': 500,
+          'max_depth': 4,
+          'min_samples_split': 5,
+          'learning_rate': 0.01,
+          'loss': 'ls'}
 
-# #############################################################################
+##############################################################################
 # Fit regression model
-params = {'n_estimators': 500, 'max_depth': 4, 'min_samples_split': 2,
-          'learning_rate': 0.01, 'loss': 'ls'}
-clf = ensemble.GradientBoostingRegressor(**params)
+# -------------------------------------
+#
+# Now we will initiate the gradient boosting regressors and fit it with our
+# training data. Let's also look and the mean squared error on the test data.
 
+clf = ensemble.GradientBoostingRegressor(**params)
 clf.fit(X_train, y_train)
+
 mse = mean_squared_error(y_test, clf.predict(X_test))
-print("MSE: %.4f" % mse)
+print("The mean squared error (MSE) on test set: {:.4f}".format(mse))
 
-# #############################################################################
+##############################################################################
 # Plot training deviance
+# -------------------------------------
+#
+# Finally, we will visualize the results. To do that we will first compute the
+# test set deviance and then plot it.
 
-# compute test set deviance
 test_score = np.zeros((params['n_estimators'],), dtype=np.float64)
 
 for i, y_pred in enumerate(clf.staged_predict(X_test)):
     test_score[i] = clf.loss_(y_test, y_pred)
 
-plt.figure(figsize=(12, 6))
+fig = plt.figure(figsize=(12, 8))
+
 plt.subplot(1, 2, 1)
 plt.title('Deviance')
 plt.plot(np.arange(params['n_estimators']) + 1, clf.train_score_, 'b-',
@@ -61,10 +108,11 @@
 plt.xlabel('Boosting Iterations')
 plt.ylabel('Deviance')
 
-# #############################################################################
+##############################################################################
 # Plot impurity-based feature importance
+# -------------------------------------
 #
-# Warning: impurity-based feature importances can be misleading for
+# Careful, impurity-based feature importances can be misleading for
 # high cardinality features (many unique values). See
 # :func:`sklearn.inspection.permutation_importance` as an alternative.
 
@@ -75,7 +123,9 @@
 pos = np.arange(sorted_idx.shape[0]) + .5
 plt.subplot(1, 2, 2)
 plt.barh(pos, feature_importance[sorted_idx], align='center')
-plt.yticks(pos, boston.feature_names[sorted_idx])
+plt.yticks(pos, np.array(diabetes.feature_names)[sorted_idx])
 plt.xlabel('Relative Importance')
 plt.title('Variable Importance')
+fig.tight_layout()
+
 plt.show()

dev/_downloads/679566501b743cb339497968edb9d62f/plot_gradient_boosting_regression.py

@@ -15,7 +15,7 @@
       "cell_type": "markdown",
       "metadata": {},
       "source": [
-        "\n# Gradient Boosting regression\n\n\nDemonstrate Gradient Boosting on the Boston housing dataset.\n\nThis example fits a Gradient Boosting model with least squares loss and\n500 regression trees of depth 4.\n"
+        "\n# Gradient Boosting regression\n\n\nThis example demonstrates Gradient Boosting to produce a predictive\nmodel from an ensemble of weak predictive models. Gradient boosting can be used\nfor regression and classification problems. Here, we will train a model to\ntackle a diabetes regression task.\n\nWe will obtain the results from\n:class:`~sklearn.ensemble.GradientBoostingRegressor` with least squares loss\nand 500 regression trees of depth 4.\n"
       ]
     },
     {
@@ -26,7 +26,97 @@
       },
       "outputs": [],
       "source": [
-        "print(__doc__)\n\n# Author: Peter Prettenhofer <[email protected]>\n#\n# License: BSD 3 clause\n\nimport numpy as np\nimport matplotlib.pyplot as plt\n\nfrom sklearn import ensemble\nfrom sklearn import datasets\nfrom sklearn.utils import shuffle\nfrom sklearn.metrics import mean_squared_error\n\n# #############################################################################\n# Load data\nboston = datasets.load_boston()\nX, y = shuffle(boston.data, boston.target, random_state=13)\nX = X.astype(np.float32)\noffset = int(X.shape[0] * 0.9)\nX_train, y_train = X[:offset], y[:offset]\nX_test, y_test = X[offset:], y[offset:]\n\n# #############################################################################\n# Fit regression model\nparams = {'n_estimators': 500, 'max_depth': 4, 'min_samples_split': 2,\n          'learning_rate': 0.01, 'loss': 'ls'}\nclf = ensemble.GradientBoostingRegressor(**params)\n\nclf.fit(X_train, y_train)\nmse = mean_squared_error(y_test, clf.predict(X_test))\nprint(\"MSE: %.4f\" % mse)\n\n# #############################################################################\n# Plot training deviance\n\n# compute test set deviance\ntest_score = np.zeros((params['n_estimators'],), dtype=np.float64)\n\nfor i, y_pred in enumerate(clf.staged_predict(X_test)):\n    test_score[i] = clf.loss_(y_test, y_pred)\n\nplt.figure(figsize=(12, 6))\nplt.subplot(1, 2, 1)\nplt.title('Deviance')\nplt.plot(np.arange(params['n_estimators']) + 1, clf.train_score_, 'b-',\n         label='Training Set Deviance')\nplt.plot(np.arange(params['n_estimators']) + 1, test_score, 'r-',\n         label='Test Set Deviance')\nplt.legend(loc='upper right')\nplt.xlabel('Boosting Iterations')\nplt.ylabel('Deviance')\n\n# #############################################################################\n# Plot impurity-based feature importance\n#\n# Warning: impurity-based feature importances can be misleading for\n# high cardinality features (many unique values). See\n# :func:`sklearn.inspection.permutation_importance` as an alternative.\n\nfeature_importance = clf.feature_importances_\n# make importances relative to max importance\nfeature_importance = 100.0 * (feature_importance / feature_importance.max())\nsorted_idx = np.argsort(feature_importance)\npos = np.arange(sorted_idx.shape[0]) + .5\nplt.subplot(1, 2, 2)\nplt.barh(pos, feature_importance[sorted_idx], align='center')\nplt.yticks(pos, boston.feature_names[sorted_idx])\nplt.xlabel('Relative Importance')\nplt.title('Variable Importance')\nplt.show()"
+        "print(__doc__)\n\n# Author: Peter Prettenhofer <[email protected]>\n#         Maria Telenczuk <https://github.com/maikia>\n#\n# License: BSD 3 clause\n\nimport numpy as np\nimport matplotlib.pyplot as plt\n\nfrom sklearn import ensemble\nfrom sklearn import datasets\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.model_selection import train_test_split"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "Load the data\n-------------------------------------\n\nFirst we need to load the data. We set random state to be consistent with the\nresult.\n\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "collapsed": false
+      },
+      "outputs": [],
+      "source": [
+        "diabetes = datasets.load_diabetes()\nX, y = diabetes.data, diabetes.target"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "Data preprocessing\n-------------------------------------\n\nNext, we will split our dataset to use 90% for training and leave the rest\nfor testing. We will also prepare the parameters we want to use to fit our\nregression model. You can play with those parameters to see how the\nresults change:\n\nHere:\nn_estimators : is the number of boosting stages which will be performed.\n    Later, we will plot and see how the deviance changes with those boosting\n    operations.\nmax_depth : this limits the number of nodes in the tree. The best value\n    depends on the interaction of the input variables.\nmin_samples_split : is the minimum number of samples required to split an\n    internal node.\nlearning_rate: tells how much the contribution of each tree will shrink\nloss: here, we decided to use least squeares as a loss function, however\n    there are many other options (check\n    :class:`~sklearn.ensemble.GradientBoostingRegressor` to see what are\n    other possibilities)\n\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "collapsed": false
+      },
+      "outputs": [],
+      "source": [
+        "X_train, X_test, y_train, y_test = train_test_split(X, y,\n                                                    test_size=0.1,\n                                                    random_state=13)\n\nparams = {'n_estimators': 500,\n          'max_depth': 4,\n          'min_samples_split': 5,\n          'learning_rate': 0.01,\n          'loss': 'ls'}"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "Fit regression model\n-------------------------------------\n\nNow we will initiate the gradient boosting regressors and fit it with our\ntraining data. Let's also look and the mean squared error on the test data.\n\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "collapsed": false
+      },
+      "outputs": [],
+      "source": [
+        "clf = ensemble.GradientBoostingRegressor(**params)\nclf.fit(X_train, y_train)\n\nmse = mean_squared_error(y_test, clf.predict(X_test))\nprint(\"The mean squared error (MSE) on test set: {:.4f}\".format(mse))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "Plot training deviance\n-------------------------------------\n\nFinally, we will visualize the results. To do that we will first compute the\ntest set deviance and then plot it.\n\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "collapsed": false
+      },
+      "outputs": [],
+      "source": [
+        "test_score = np.zeros((params['n_estimators'],), dtype=np.float64)\n\nfor i, y_pred in enumerate(clf.staged_predict(X_test)):\n    test_score[i] = clf.loss_(y_test, y_pred)\n\nfig = plt.figure(figsize=(12, 8))\n\nplt.subplot(1, 2, 1)\nplt.title('Deviance')\nplt.plot(np.arange(params['n_estimators']) + 1, clf.train_score_, 'b-',\n         label='Training Set Deviance')\nplt.plot(np.arange(params['n_estimators']) + 1, test_score, 'r-',\n         label='Test Set Deviance')\nplt.legend(loc='upper right')\nplt.xlabel('Boosting Iterations')\nplt.ylabel('Deviance')"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "Plot impurity-based feature importance\n-------------------------------------\n\nCareful, impurity-based feature importances can be misleading for\nhigh cardinality features (many unique values). See\n:func:`sklearn.inspection.permutation_importance` as an alternative.\n\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "collapsed": false
+      },
+      "outputs": [],
+      "source": [
+        "feature_importance = clf.feature_importances_\n# make importances relative to max importance\nfeature_importance = 100.0 * (feature_importance / feature_importance.max())\nsorted_idx = np.argsort(feature_importance)\npos = np.arange(sorted_idx.shape[0]) + .5\nplt.subplot(1, 2, 2)\nplt.barh(pos, feature_importance[sorted_idx], align='center')\nplt.yticks(pos, np.array(diabetes.feature_names)[sorted_idx])\nplt.xlabel('Relative Importance')\nplt.title('Variable Importance')\nfig.tight_layout()\n\nplt.show()"
       ]
     }
   ],