Pushing the docs to dev/ for branch: master, commit c97727e8c80537a4a8779f4e9e3e0ed3ce8c184e

Author: sklearn-ci

dev/_downloads/auto_examples_jupyter.zip

@@ -26,7 +26,7 @@
       },
       "outputs": [],
       "source": [
-        "# Author: Matt Terry <[email protected]>\n#\n# License: BSD 3 clause\n\nimport numpy as np\n\nfrom sklearn.base import BaseEstimator, TransformerMixin\nfrom sklearn.datasets import fetch_20newsgroups\nfrom sklearn.datasets.twenty_newsgroups import strip_newsgroup_footer\nfrom sklearn.datasets.twenty_newsgroups import strip_newsgroup_quoting\nfrom sklearn.decomposition import TruncatedSVD\nfrom sklearn.feature_extraction import DictVectorizer\nfrom sklearn.feature_extraction.text import TfidfVectorizer\nfrom sklearn.metrics import classification_report\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.compose import ColumnTransformer\nfrom sklearn.svm import LinearSVC\n\n\nclass TextStats(BaseEstimator, TransformerMixin):\n    \"\"\"Extract features from each document for DictVectorizer\"\"\"\n\n    def fit(self, x, y=None):\n        return self\n\n    def transform(self, posts):\n        return [{'length': len(text),\n                 'num_sentences': text.count('.')}\n                for text in posts]\n\n\nclass SubjectBodyExtractor(BaseEstimator, TransformerMixin):\n    \"\"\"Extract the subject & body from a usenet post in a single pass.\n\n    Takes a sequence of strings and produces a dict of sequences.  Keys are\n    `subject` and `body`.\n    \"\"\"\n    def fit(self, x, y=None):\n        return self\n\n    def transform(self, posts):\n        # construct object dtype array with two columns\n        # first column = 'subject' and second column = 'body'\n        features = np.empty(shape=(len(posts), 2), dtype=object)\n        for i, text in enumerate(posts):\n            headers, _, bod = text.partition('\\n\\n')\n            bod = strip_newsgroup_footer(bod)\n            bod = strip_newsgroup_quoting(bod)\n            features[i, 1] = bod\n\n            prefix = 'Subject:'\n            sub = ''\n            for line in headers.split('\\n'):\n                if line.startswith(prefix):\n                    sub = line[len(prefix):]\n                    break\n            features[i, 0] = sub\n\n        return features\n\n\npipeline = Pipeline([\n    # Extract the subject & body\n    ('subjectbody', SubjectBodyExtractor()),\n\n    # Use ColumnTransformer to combine the features from subject and body\n    ('union', ColumnTransformer(\n        [\n            # Pulling features from the post's subject line (first column)\n            ('subject', TfidfVectorizer(min_df=50), 0),\n\n            # Pipeline for standard bag-of-words model for body (second column)\n            ('body_bow', Pipeline([\n                ('tfidf', TfidfVectorizer()),\n                ('best', TruncatedSVD(n_components=50)),\n            ]), 1),\n\n            # Pipeline for pulling ad hoc features from post's body\n            ('body_stats', Pipeline([\n                ('stats', TextStats()),  # returns a list of dicts\n                ('vect', DictVectorizer()),  # list of dicts -> feature matrix\n            ]), 1),\n        ],\n\n        # weight components in ColumnTransformer\n        transformer_weights={\n            'subject': 0.8,\n            'body_bow': 0.5,\n            'body_stats': 1.0,\n        }\n    )),\n\n    # Use a SVC classifier on the combined features\n    ('svc', LinearSVC()),\n], verbose=True)\n\n# limit the list of categories to make running this example faster.\ncategories = ['alt.atheism', 'talk.religion.misc']\ntrain = fetch_20newsgroups(random_state=1,\n                           subset='train',\n                           categories=categories,\n                           )\ntest = fetch_20newsgroups(random_state=1,\n                          subset='test',\n                          categories=categories,\n                          )\n\npipeline.fit(train.data, train.target)\ny = pipeline.predict(test.data)\nprint(classification_report(y, test.target))"
+        "# Author: Matt Terry <[email protected]>\n#\n# License: BSD 3 clause\n\nimport numpy as np\n\nfrom sklearn.base import BaseEstimator, TransformerMixin\nfrom sklearn.datasets import fetch_20newsgroups\nfrom sklearn.datasets.twenty_newsgroups import strip_newsgroup_footer\nfrom sklearn.datasets.twenty_newsgroups import strip_newsgroup_quoting\nfrom sklearn.decomposition import TruncatedSVD\nfrom sklearn.feature_extraction import DictVectorizer\nfrom sklearn.feature_extraction.text import TfidfVectorizer\nfrom sklearn.metrics import classification_report\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.compose import ColumnTransformer\nfrom sklearn.svm import LinearSVC\n\n\nclass TextStats(BaseEstimator, TransformerMixin):\n    \"\"\"Extract features from each document for DictVectorizer\"\"\"\n\n    def fit(self, x, y=None):\n        return self\n\n    def transform(self, posts):\n        return [{'length': len(text),\n                 'num_sentences': text.count('.')}\n                for text in posts]\n\n\nclass SubjectBodyExtractor(BaseEstimator, TransformerMixin):\n    \"\"\"Extract the subject & body from a usenet post in a single pass.\n\n    Takes a sequence of strings and produces a dict of sequences.  Keys are\n    `subject` and `body`.\n    \"\"\"\n    def fit(self, x, y=None):\n        return self\n\n    def transform(self, posts):\n        # construct object dtype array with two columns\n        # first column = 'subject' and second column = 'body'\n        features = np.empty(shape=(len(posts), 2), dtype=object)\n        for i, text in enumerate(posts):\n            headers, _, bod = text.partition('\\n\\n')\n            bod = strip_newsgroup_footer(bod)\n            bod = strip_newsgroup_quoting(bod)\n            features[i, 1] = bod\n\n            prefix = 'Subject:'\n            sub = ''\n            for line in headers.split('\\n'):\n                if line.startswith(prefix):\n                    sub = line[len(prefix):]\n                    break\n            features[i, 0] = sub\n\n        return features\n\n\npipeline = Pipeline([\n    # Extract the subject & body\n    ('subjectbody', SubjectBodyExtractor()),\n\n    # Use ColumnTransformer to combine the features from subject and body\n    ('union', ColumnTransformer(\n        [\n            # Pulling features from the post's subject line (first column)\n            ('subject', TfidfVectorizer(min_df=50), 0),\n\n            # Pipeline for standard bag-of-words model for body (second column)\n            ('body_bow', Pipeline([\n                ('tfidf', TfidfVectorizer()),\n                ('best', TruncatedSVD(n_components=50)),\n            ]), 1),\n\n            # Pipeline for pulling ad hoc features from post's body\n            ('body_stats', Pipeline([\n                ('stats', TextStats()),  # returns a list of dicts\n                ('vect', DictVectorizer()),  # list of dicts -> feature matrix\n            ]), 1),\n        ],\n\n        # weight components in ColumnTransformer\n        transformer_weights={\n            'subject': 0.8,\n            'body_bow': 0.5,\n            'body_stats': 1.0,\n        }\n    )),\n\n    # Use a SVC classifier on the combined features\n    ('svc', LinearSVC(dual=False)),\n], verbose=True)\n\n# limit the list of categories to make running this example faster.\ncategories = ['alt.atheism', 'talk.religion.misc']\ntrain = fetch_20newsgroups(random_state=1,\n                           subset='train',\n                           categories=categories,\n                           )\ntest = fetch_20newsgroups(random_state=1,\n                          subset='test',\n                          categories=categories,\n                          )\n\npipeline.fit(train.data, train.target)\ny = pipeline.predict(test.data)\nprint(classification_report(y, test.target))"
       ]
     }
   ],

dev/_downloads/auto_examples_python.zip

@@ -116,7 +116,7 @@ def transform(self, posts):
     )),
 
     # Use a SVC classifier on the combined features
-    ('svc', LinearSVC()),
+    ('svc', LinearSVC(dual=False)),
 ], verbose=True)
 
 # limit the list of categories to make running this example faster.