add 0.18 stable website

Author: amueller

0.18/.buildinfo

@@ -0,0 +1,4 @@
+# Sphinx build info version 1
+# This file hashes the configuration used when building these files. When it is not found, a full rebuild will be done.
+config: ee0995393e1e51351d5bec758afe160b
+tags: 645f666f9bcd5a90fca523b33c5a78b7

0.18/_downloads/bicluster_newsgroups.ipynb

@@ -0,0 +1,54 @@
+{
+  "nbformat_minor": 0,
+  "cells": [
+    {
+      "source": [
+        "%matplotlib inline"
+      ],
+      "metadata": {
+        "collapsed": false
+      },
+      "execution_count": null,
+      "outputs": [],
+      "cell_type": "code"
+    },
+    {
+      "source": [
+        "\n# Biclustering documents with the Spectral Co-clustering algorithm\n\n\nThis example demonstrates the Spectral Co-clustering algorithm on the\ntwenty newsgroups dataset. The 'comp.os.ms-windows.misc' category is\nexcluded because it contains many posts containing nothing but data.\n\nThe TF-IDF vectorized posts form a word frequency matrix, which is\nthen biclustered using Dhillon's Spectral Co-Clustering algorithm. The\nresulting document-word biclusters indicate subsets words used more\noften in those subsets documents.\n\nFor a few of the best biclusters, its most common document categories\nand its ten most important words get printed. The best biclusters are\ndetermined by their normalized cut. The best words are determined by\ncomparing their sums inside and outside the bicluster.\n\nFor comparison, the documents are also clustered using\nMiniBatchKMeans. The document clusters derived from the biclusters\nachieve a better V-measure than clusters found by MiniBatchKMeans.\n\nOutput::\n\n    Vectorizing...\n    Coclustering...\n    Done in 9.53s. V-measure: 0.4455\n    MiniBatchKMeans...\n    Done in 12.00s. V-measure: 0.3309\n\n    Best biclusters:\n    ----------------\n    bicluster 0 : 1951 documents, 4373 words\n    categories   : 23% talk.politics.guns, 19% talk.politics.misc, 14% sci.med\n    words        : gun, guns, geb, banks, firearms, drugs, gordon, clinton, cdt, amendment\n\n    bicluster 1 : 1165 documents, 3304 words\n    categories   : 29% talk.politics.mideast, 26% soc.religion.christian, 25% alt.atheism\n    words        : god, jesus, christians, atheists, kent, sin, morality, belief, resurrection, marriage\n\n    bicluster 2 : 2219 documents, 2830 words\n    categories   : 18% comp.sys.mac.hardware, 16% comp.sys.ibm.pc.hardware, 16% comp.graphics\n    words        : voltage, dsp, board, receiver, circuit, shipping, packages, stereo, compression, package\n\n    bicluster 3 : 1860 documents, 2745 words\n    categories   : 26% rec.motorcycles, 23% rec.autos, 13% misc.forsale\n    words        : bike, car, dod, engine, motorcycle, ride, honda, cars, bmw, bikes\n\n    bicluster 4 : 12 documents, 155 words\n    categories   : 100% rec.sport.hockey\n    words        : scorer, unassisted, reichel, semak, sweeney, kovalenko, ricci, audette, momesso, nedved\n\n"
+      ],
+      "metadata": {},
+      "cell_type": "markdown"
+    },
+    {
+      "source": [
+        "from __future__ import print_function\n\nprint(__doc__)\n\nfrom collections import defaultdict\nimport operator\nimport re\nfrom time import time\n\nimport numpy as np\n\nfrom sklearn.cluster.bicluster import SpectralCoclustering\nfrom sklearn.cluster import MiniBatchKMeans\nfrom sklearn.externals.six import iteritems\nfrom sklearn.datasets.twenty_newsgroups import fetch_20newsgroups\nfrom sklearn.feature_extraction.text import TfidfVectorizer\nfrom sklearn.metrics.cluster import v_measure_score\n\n\ndef number_aware_tokenizer(doc):\n    \"\"\" Tokenizer that maps all numeric tokens to a placeholder.\n\n    For many applications, tokens that begin with a number are not directly\n    useful, but the fact that such a token exists can be relevant.  By applying\n    this form of dimensionality reduction, some methods may perform better.\n    \"\"\"\n    token_pattern = re.compile(u'(?u)\\\\b\\\\w\\\\w+\\\\b')\n    tokens = token_pattern.findall(doc)\n    tokens = [\"#NUMBER\" if token[0] in \"0123456789_\" else token\n              for token in tokens]\n    return tokens\n\n# exclude 'comp.os.ms-windows.misc'\ncategories = ['alt.atheism', 'comp.graphics',\n              'comp.sys.ibm.pc.hardware', 'comp.sys.mac.hardware',\n              'comp.windows.x', 'misc.forsale', 'rec.autos',\n              'rec.motorcycles', 'rec.sport.baseball',\n              'rec.sport.hockey', 'sci.crypt', 'sci.electronics',\n              'sci.med', 'sci.space', 'soc.religion.christian',\n              'talk.politics.guns', 'talk.politics.mideast',\n              'talk.politics.misc', 'talk.religion.misc']\nnewsgroups = fetch_20newsgroups(categories=categories)\ny_true = newsgroups.target\n\nvectorizer = TfidfVectorizer(stop_words='english', min_df=5,\n                             tokenizer=number_aware_tokenizer)\ncocluster = SpectralCoclustering(n_clusters=len(categories),\n                                 svd_method='arpack', random_state=0)\nkmeans = MiniBatchKMeans(n_clusters=len(categories), batch_size=20000,\n                         random_state=0)\n\nprint(\"Vectorizing...\")\nX = vectorizer.fit_transform(newsgroups.data)\n\nprint(\"Coclustering...\")\nstart_time = time()\ncocluster.fit(X)\ny_cocluster = cocluster.row_labels_\nprint(\"Done in {:.2f}s. V-measure: {:.4f}\".format(\n    time() - start_time,\n    v_measure_score(y_cocluster, y_true)))\n\nprint(\"MiniBatchKMeans...\")\nstart_time = time()\ny_kmeans = kmeans.fit_predict(X)\nprint(\"Done in {:.2f}s. V-measure: {:.4f}\".format(\n    time() - start_time,\n    v_measure_score(y_kmeans, y_true)))\n\nfeature_names = vectorizer.get_feature_names()\ndocument_names = list(newsgroups.target_names[i] for i in newsgroups.target)\n\n\ndef bicluster_ncut(i):\n    rows, cols = cocluster.get_indices(i)\n    if not (np.any(rows) and np.any(cols)):\n        import sys\n        return sys.float_info.max\n    row_complement = np.nonzero(np.logical_not(cocluster.rows_[i]))[0]\n    col_complement = np.nonzero(np.logical_not(cocluster.columns_[i]))[0]\n    # Note: the following is identical to X[rows[:, np.newaxis], cols].sum() but\n    # much faster in scipy <= 0.16\n    weight = X[rows][:, cols].sum()\n    cut = (X[row_complement][:, cols].sum() +\n           X[rows][:, col_complement].sum())\n    return cut / weight\n\n\ndef most_common(d):\n    \"\"\"Items of a defaultdict(int) with the highest values.\n\n    Like Counter.most_common in Python >=2.7.\n    \"\"\"\n    return sorted(iteritems(d), key=operator.itemgetter(1), reverse=True)\n\n\nbicluster_ncuts = list(bicluster_ncut(i)\n                       for i in range(len(newsgroups.target_names)))\nbest_idx = np.argsort(bicluster_ncuts)[:5]\n\nprint()\nprint(\"Best biclusters:\")\nprint(\"----------------\")\nfor idx, cluster in enumerate(best_idx):\n    n_rows, n_cols = cocluster.get_shape(cluster)\n    cluster_docs, cluster_words = cocluster.get_indices(cluster)\n    if not len(cluster_docs) or not len(cluster_words):\n        continue\n\n    # categories\n    counter = defaultdict(int)\n    for i in cluster_docs:\n        counter[document_names[i]] += 1\n    cat_string = \", \".join(\"{:.0f}% {}\".format(float(c) / n_rows * 100, name)\n                           for name, c in most_common(counter)[:3])\n\n    # words\n    out_of_cluster_docs = cocluster.row_labels_ != cluster\n    out_of_cluster_docs = np.where(out_of_cluster_docs)[0]\n    word_col = X[:, cluster_words]\n    word_scores = np.array(word_col[cluster_docs, :].sum(axis=0) -\n                           word_col[out_of_cluster_docs, :].sum(axis=0))\n    word_scores = word_scores.ravel()\n    important_words = list(feature_names[cluster_words[i]]\n                           for i in word_scores.argsort()[:-11:-1])\n\n    print(\"bicluster {} : {} documents, {} words\".format(\n        idx, n_rows, n_cols))\n    print(\"categories   : {}\".format(cat_string))\n    print(\"words        : {}\\n\".format(', '.join(important_words)))"
+      ],
+      "metadata": {
+        "collapsed": false
+      },
+      "execution_count": null,
+      "outputs": [],
+      "cell_type": "code"
+    }
+  ],
+  "metadata": {
+    "kernelspec": {
+      "language": "python",
+      "name": "python3",
+      "display_name": "Python 3"
+    },
+    "language_info": {
+      "pygments_lexer": "ipython3",
+      "version": "3.5.2",
+      "name": "python",
+      "mimetype": "text/x-python",
+      "codemirror_mode": {
+        "version": 3,
+        "name": "ipython"
+      },
+      "nbconvert_exporter": "python",
+      "file_extension": ".py"
+    }
+  },
+  "nbformat": 4
+}

0.18/_downloads/bicluster_newsgroups.py

@@ -0,0 +1,184 @@
+"""
+================================================================
+Biclustering documents with the Spectral Co-clustering algorithm
+================================================================
+
+This example demonstrates the Spectral Co-clustering algorithm on the
+twenty newsgroups dataset. The 'comp.os.ms-windows.misc' category is
+excluded because it contains many posts containing nothing but data.
+
+The TF-IDF vectorized posts form a word frequency matrix, which is
+then biclustered using Dhillon's Spectral Co-Clustering algorithm. The
+resulting document-word biclusters indicate subsets words used more
+often in those subsets documents.
+
+For a few of the best biclusters, its most common document categories
+and its ten most important words get printed. The best biclusters are
+determined by their normalized cut. The best words are determined by
+comparing their sums inside and outside the bicluster.
+
+For comparison, the documents are also clustered using
+MiniBatchKMeans. The document clusters derived from the biclusters
+achieve a better V-measure than clusters found by MiniBatchKMeans.
+
+Output::
+
+    Vectorizing...
+    Coclustering...
+    Done in 9.53s. V-measure: 0.4455
+    MiniBatchKMeans...
+    Done in 12.00s. V-measure: 0.3309
+
+    Best biclusters:
+    ----------------
+    bicluster 0 : 1951 documents, 4373 words
+    categories   : 23% talk.politics.guns, 19% talk.politics.misc, 14% sci.med
+    words        : gun, guns, geb, banks, firearms, drugs, gordon, clinton, cdt, amendment
+
+    bicluster 1 : 1165 documents, 3304 words
+    categories   : 29% talk.politics.mideast, 26% soc.religion.christian, 25% alt.atheism
+    words        : god, jesus, christians, atheists, kent, sin, morality, belief, resurrection, marriage
+
+    bicluster 2 : 2219 documents, 2830 words
+    categories   : 18% comp.sys.mac.hardware, 16% comp.sys.ibm.pc.hardware, 16% comp.graphics
+    words        : voltage, dsp, board, receiver, circuit, shipping, packages, stereo, compression, package
+
+    bicluster 3 : 1860 documents, 2745 words
+    categories   : 26% rec.motorcycles, 23% rec.autos, 13% misc.forsale
+    words        : bike, car, dod, engine, motorcycle, ride, honda, cars, bmw, bikes
+
+    bicluster 4 : 12 documents, 155 words
+    categories   : 100% rec.sport.hockey
+    words        : scorer, unassisted, reichel, semak, sweeney, kovalenko, ricci, audette, momesso, nedved
+
+"""
+from __future__ import print_function
+
+print(__doc__)
+
+from collections import defaultdict
+import operator
+import re
+from time import time
+
+import numpy as np
+
+from sklearn.cluster.bicluster import SpectralCoclustering
+from sklearn.cluster import MiniBatchKMeans
+from sklearn.externals.six import iteritems
+from sklearn.datasets.twenty_newsgroups import fetch_20newsgroups
+from sklearn.feature_extraction.text import TfidfVectorizer
+from sklearn.metrics.cluster import v_measure_score
+
+
+def number_aware_tokenizer(doc):
+    """ Tokenizer that maps all numeric tokens to a placeholder.
+
+    For many applications, tokens that begin with a number are not directly
+    useful, but the fact that such a token exists can be relevant.  By applying
+    this form of dimensionality reduction, some methods may perform better.
+    """
+    token_pattern = re.compile(u'(?u)\\b\\w\\w+\\b')
+    tokens = token_pattern.findall(doc)
+    tokens = ["#NUMBER" if token[0] in "0123456789_" else token
+              for token in tokens]
+    return tokens
+
+# exclude 'comp.os.ms-windows.misc'
+categories = ['alt.atheism', 'comp.graphics',
+              'comp.sys.ibm.pc.hardware', 'comp.sys.mac.hardware',
+              'comp.windows.x', 'misc.forsale', 'rec.autos',
+              'rec.motorcycles', 'rec.sport.baseball',
+              'rec.sport.hockey', 'sci.crypt', 'sci.electronics',
+              'sci.med', 'sci.space', 'soc.religion.christian',
+              'talk.politics.guns', 'talk.politics.mideast',
+              'talk.politics.misc', 'talk.religion.misc']
+newsgroups = fetch_20newsgroups(categories=categories)
+y_true = newsgroups.target
+
+vectorizer = TfidfVectorizer(stop_words='english', min_df=5,
+                             tokenizer=number_aware_tokenizer)
+cocluster = SpectralCoclustering(n_clusters=len(categories),
+                                 svd_method='arpack', random_state=0)
+kmeans = MiniBatchKMeans(n_clusters=len(categories), batch_size=20000,
+                         random_state=0)
+
+print("Vectorizing...")
+X = vectorizer.fit_transform(newsgroups.data)
+
+print("Coclustering...")
+start_time = time()
+cocluster.fit(X)
+y_cocluster = cocluster.row_labels_
+print("Done in {:.2f}s. V-measure: {:.4f}".format(
+    time() - start_time,
+    v_measure_score(y_cocluster, y_true)))
+
+print("MiniBatchKMeans...")
+start_time = time()
+y_kmeans = kmeans.fit_predict(X)
+print("Done in {:.2f}s. V-measure: {:.4f}".format(
+    time() - start_time,
+    v_measure_score(y_kmeans, y_true)))
+
+feature_names = vectorizer.get_feature_names()
+document_names = list(newsgroups.target_names[i] for i in newsgroups.target)
+
+
+def bicluster_ncut(i):
+    rows, cols = cocluster.get_indices(i)
+    if not (np.any(rows) and np.any(cols)):
+        import sys
+        return sys.float_info.max
+    row_complement = np.nonzero(np.logical_not(cocluster.rows_[i]))[0]
+    col_complement = np.nonzero(np.logical_not(cocluster.columns_[i]))[0]
+    # Note: the following is identical to X[rows[:, np.newaxis], cols].sum() but
+    # much faster in scipy <= 0.16
+    weight = X[rows][:, cols].sum()
+    cut = (X[row_complement][:, cols].sum() +
+           X[rows][:, col_complement].sum())
+    return cut / weight
+
+
+def most_common(d):
+    """Items of a defaultdict(int) with the highest values.
+
+    Like Counter.most_common in Python >=2.7.
+    """
+    return sorted(iteritems(d), key=operator.itemgetter(1), reverse=True)
+
+
+bicluster_ncuts = list(bicluster_ncut(i)
+                       for i in range(len(newsgroups.target_names)))
+best_idx = np.argsort(bicluster_ncuts)[:5]
+
+print()
+print("Best biclusters:")
+print("----------------")
+for idx, cluster in enumerate(best_idx):
+    n_rows, n_cols = cocluster.get_shape(cluster)
+    cluster_docs, cluster_words = cocluster.get_indices(cluster)
+    if not len(cluster_docs) or not len(cluster_words):
+        continue
+
+    # categories
+    counter = defaultdict(int)
+    for i in cluster_docs:
+        counter[document_names[i]] += 1
+    cat_string = ", ".join("{:.0f}% {}".format(float(c) / n_rows * 100, name)
+                           for name, c in most_common(counter)[:3])
+
+    # words
+    out_of_cluster_docs = cocluster.row_labels_ != cluster
+    out_of_cluster_docs = np.where(out_of_cluster_docs)[0]
+    word_col = X[:, cluster_words]
+    word_scores = np.array(word_col[cluster_docs, :].sum(axis=0) -
+                           word_col[out_of_cluster_docs, :].sum(axis=0))
+    word_scores = word_scores.ravel()
+    important_words = list(feature_names[cluster_words[i]]
+                           for i in word_scores.argsort()[:-11:-1])
+
+    print("bicluster {} : {} documents, {} words".format(
+        idx, n_rows, n_cols))
+    print("categories   : {}".format(cat_string))
+    print("words        : {}\n".format(', '.join(important_words)))

0.18/_downloads/digits_classification_exercise.ipynb

@@ -0,0 +1,54 @@
+{
+  "nbformat_minor": 0,
+  "cells": [
+    {
+      "source": [
+        "%matplotlib inline"
+      ],
+      "metadata": {
+        "collapsed": false
+      },
+      "execution_count": null,
+      "outputs": [],
+      "cell_type": "code"
+    },
+    {
+      "source": [
+        "\n# Digits Classification Exercise\n\n\nA tutorial exercise regarding the use of classification techniques on\nthe Digits dataset.\n\nThis exercise is used in the :ref:`clf_tut` part of the\n:ref:`supervised_learning_tut` section of the\n:ref:`stat_learn_tut_index`.\n"
+      ],
+      "metadata": {},
+      "cell_type": "markdown"
+    },
+    {
+      "source": [
+        "print(__doc__)\n\nfrom sklearn import datasets, neighbors, linear_model\n\ndigits = datasets.load_digits()\nX_digits = digits.data\ny_digits = digits.target\n\nn_samples = len(X_digits)\n\nX_train = X_digits[:.9 * n_samples]\ny_train = y_digits[:.9 * n_samples]\nX_test = X_digits[.9 * n_samples:]\ny_test = y_digits[.9 * n_samples:]\n\nknn = neighbors.KNeighborsClassifier()\nlogistic = linear_model.LogisticRegression()\n\nprint('KNN score: %f' % knn.fit(X_train, y_train).score(X_test, y_test))\nprint('LogisticRegression score: %f'\n      % logistic.fit(X_train, y_train).score(X_test, y_test))"
+      ],
+      "metadata": {
+        "collapsed": false
+      },
+      "execution_count": null,
+      "outputs": [],
+      "cell_type": "code"
+    }
+  ],
+  "metadata": {
+    "kernelspec": {
+      "language": "python",
+      "name": "python3",
+      "display_name": "Python 3"
+    },
+    "language_info": {
+      "pygments_lexer": "ipython3",
+      "version": "3.5.2",
+      "name": "python",
+      "mimetype": "text/x-python",
+      "codemirror_mode": {
+        "version": 3,
+        "name": "ipython"
+      },
+      "nbconvert_exporter": "python",
+      "file_extension": ".py"
+    }
+  },
+  "nbformat": 4
+}

0.18/_downloads/digits_classification_exercise.py

@@ -0,0 +1,33 @@
+"""
+================================
+Digits Classification Exercise
+================================
+
+A tutorial exercise regarding the use of classification techniques on
+the Digits dataset.
+
+This exercise is used in the :ref:`clf_tut` part of the
+:ref:`supervised_learning_tut` section of the
+:ref:`stat_learn_tut_index`.
+"""
+print(__doc__)
+
+from sklearn import datasets, neighbors, linear_model
+
+digits = datasets.load_digits()
+X_digits = digits.data
+y_digits = digits.target
+
+n_samples = len(X_digits)
+
+X_train = X_digits[:.9 * n_samples]
+y_train = y_digits[:.9 * n_samples]
+X_test = X_digits[.9 * n_samples:]
+y_test = y_digits[.9 * n_samples:]
+
+knn = neighbors.KNeighborsClassifier()
+logistic = linear_model.LogisticRegression()
+
+print('KNN score: %f' % knn.fit(X_train, y_train).score(X_test, y_test))
+print('LogisticRegression score: %f'
+      % logistic.fit(X_train, y_train).score(X_test, y_test))