"print(__doc__)\n\n# Author: Jan Hendrik Metzen <
[email protected]>\n# License: BSD Style.\n\n\nimport matplotlib.pyplot as plt\n\nimport numpy as np\n\nfrom sklearn.datasets import make_blobs\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.calibration import CalibratedClassifierCV\nfrom sklearn.metrics import log_loss\n\nnp.random.seed(0)\n\n# Generate data\nX, y = make_blobs(n_samples=1000, n_features=2, random_state=42,\n cluster_std=5.0)\nX_train, y_train = X[:600], y[:600]\nX_valid, y_valid = X[600:800], y[600:800]\nX_train_valid, y_train_valid = X[:800], y[:800]\nX_test, y_test = X[800:], y[800:]\n\n# Train uncalibrated random forest classifier on whole train and validation\n# data and evaluate on test data\nclf = RandomForestClassifier(n_estimators=25)\nclf.fit(X_train_valid, y_train_valid)\nclf_probs = clf.predict_proba(X_test)\nscore = log_loss(y_test, clf_probs)\n\n# Train random forest classifier, calibrate on validation data and evaluate\n# on test data\nclf = RandomForestClassifier(n_estimators=25)\nclf.fit(X_train, y_train)\nclf_probs = clf.predict_proba(X_test)\nsig_clf = CalibratedClassifierCV(clf, method=\"sigmoid\", cv=\"prefit\")\nsig_clf.fit(X_valid, y_valid)\nsig_clf_probs = sig_clf.predict_proba(X_test)\nsig_score = log_loss(y_test, sig_clf_probs)\n\n# Plot changes in predicted probabilities via arrows\nplt.figure()\ncolors = [\"r\", \"g\", \"b\"]\nfor i in range(clf_probs.shape[0]):\n plt.arrow(clf_probs[i, 0], clf_probs[i, 1],\n sig_clf_probs[i, 0] - clf_probs[i, 0],\n sig_clf_probs[i, 1] - clf_probs[i, 1],\n color=colors[y_test[i]], head_width=1e-2)\n\n# Plot perfect predictions\nplt.plot([1.0], [0.0], 'ro', ms=20, label=\"Class 1\")\nplt.plot([0.0], [1.0], 'go', ms=20, label=\"Class 2\")\nplt.plot([0.0], [0.0], 'bo', ms=20, label=\"Class 3\")\n\n# Plot boundaries of unit simplex\nplt.plot([0.0, 1.0, 0.0, 0.0], [0.0, 0.0, 1.0, 0.0], 'k', label=\"Simplex\")\n\n# Annotate points on the simplex\nplt.annotate(r'($\\frac{1}{3}$, $\\frac{1}{3}$, $\\frac{1}{3}$)',\n xy=(1.0/3, 1.0/3), xytext=(1.0/3, .23), xycoords='data',\n arrowprops=dict(facecolor='black', shrink=0.05),\n horizontalalignment='center', verticalalignment='center')\nplt.plot([1.0/3], [1.0/3], 'ko', ms=5)\nplt.annotate(r'($\\frac{1}{2}$, $0$, $\\frac{1}{2}$)',\n xy=(.5, .0), xytext=(.5, .1), xycoords='data',\n arrowprops=dict(facecolor='black', shrink=0.05),\n horizontalalignment='center', verticalalignment='center')\nplt.annotate(r'($0$, $\\frac{1}{2}$, $\\frac{1}{2}$)',\n xy=(.0, .5), xytext=(.1, .5), xycoords='data',\n arrowprops=dict(facecolor='black', shrink=0.05),\n horizontalalignment='center', verticalalignment='center')\nplt.annotate(r'($\\frac{1}{2}$, $\\frac{1}{2}$, $0$)',\n xy=(.5, .5), xytext=(.6, .6), xycoords='data',\n arrowprops=dict(facecolor='black', shrink=0.05),\n horizontalalignment='center', verticalalignment='center')\nplt.annotate(r'($0$, $0$, $1$)',\n xy=(0, 0), xytext=(.1, .1), xycoords='data',\n arrowprops=dict(facecolor='black', shrink=0.05),\n horizontalalignment='center', verticalalignment='center')\nplt.annotate(r'($1$, $0$, $0$)',\n xy=(1, 0), xytext=(1, .1), xycoords='data',\n arrowprops=dict(facecolor='black', shrink=0.05),\n horizontalalignment='center', verticalalignment='center')\nplt.annotate(r'($0$, $1$, $0$)',\n xy=(0, 1), xytext=(.1, 1), xycoords='data',\n arrowprops=dict(facecolor='black', shrink=0.05),\n horizontalalignment='center', verticalalignment='center')\n# Add grid\nplt.grid(False)\nfor x in [0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0]:\n plt.plot([0, x], [x, 0], 'k', alpha=0.2)\n plt.plot([0, 0 + (1-x)/2], [x, x + (1-x)/2], 'k', alpha=0.2)\n plt.plot([x, x + (1-x)/2], [0, 0 + (1-x)/2], 'k', alpha=0.2)\n\nplt.title(\"Change of predicted probabilities after sigmoid calibration\")\nplt.xlabel(\"Probability class 1\")\nplt.ylabel(\"Probability class 2\")\nplt.xlim(-0.05, 1.05)\nplt.ylim(-0.05, 1.05)\nplt.legend(loc=\"best\")\n\nprint(\"Log-loss of\")\nprint(\" * uncalibrated classifier trained on 800 datapoints: %.3f \"\n % score)\nprint(\" * classifier trained on 600 datapoints and calibrated on \"\n \"200 datapoint: %.3f\" % sig_score)\n\n# Illustrate calibrator\nplt.figure()\n# generate grid over 2-simplex\np1d = np.linspace(0, 1, 20)\np0, p1 = np.meshgrid(p1d, p1d)\np2 = 1 - p0 - p1\np = np.c_[p0.ravel(), p1.ravel(), p2.ravel()]\np = p[p[:, 2] >= 0]\n\ncalibrated_classifier = sig_clf.calibrated_classifiers_[0]\nprediction = np.vstack([calibrator.predict(this_p)\n for calibrator, this_p in\n zip(calibrated_classifier.calibrators_, p.T)]).T\nprediction /= prediction.sum(axis=1)[:, None]\n\n# Plot modifications of calibrator\nfor i in range(prediction.shape[0]):\n plt.arrow(p[i, 0], p[i, 1],\n prediction[i, 0] - p[i, 0], prediction[i, 1] - p[i, 1],\n head_width=1e-2, color=colors[np.argmax(p[i])])\n# Plot boundaries of unit simplex\nplt.plot([0.0, 1.0, 0.0, 0.0], [0.0, 0.0, 1.0, 0.0], 'k', label=\"Simplex\")\n\nplt.grid(False)\nfor x in [0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0]:\n plt.plot([0, x], [x, 0], 'k', alpha=0.2)\n plt.plot([0, 0 + (1-x)/2], [x, x + (1-x)/2], 'k', alpha=0.2)\n plt.plot([x, x + (1-x)/2], [0, 0 + (1-x)/2], 'k', alpha=0.2)\n\nplt.title(\"Illustration of sigmoid calibrator\")\nplt.xlabel(\"Probability class 1\")\nplt.ylabel(\"Probability class 2\")\nplt.xlim(-0.05, 1.05)\nplt.ylim(-0.05, 1.05)\n\nplt.show()"
0 commit comments