【Scikit-Learn 中文文档】处理文本数据 - scikit-learn 教程 | ApacheCN

处理文本数据

scikit-learn

教程设置

scikit-learn/doc/tutorial/text_analytics/

*.rst files

data

skeletons

solutions

sklearn_tut_workspace

% cp -r skeletons work_directory/sklearn_tut_workspace

$TUTORIAL_HOME/data

fetch_data.py

% cd $TUTORIAL_HOME/data/languages
% less fetch_data.py
% python fetch_data.py

加载这20个新闻组的数据集

20news-bydate-train

sklearn.datasets.load_files

>>> categories = ['alt.atheism', 'soc.religion.christian',
...               'comp.graphics', 'sci.med']

>>> from sklearn.datasets import fetch_20newsgroups
>>> twenty_train = fetch_20newsgroups(subset='train',
...     categories=categories, shuffle=True, random_state=42)

scikit-learn

dict

object

target_names

>>> twenty_train.target_names
['alt.atheism', 'comp.graphics', 'sci.med', 'soc.religion.christian']

data

>>> len(twenty_train.data)
2257
>>> len(twenty_train.filenames)
2257

>>> print("\n".join(twenty_train.data[0].split("\n")[:3]))
From: sd345@city.ac.uk (Michael Collier)
Subject: Converting images to HP LaserJet III?
Nntp-Posting-Host: hampton

>>> print(twenty_train.target_names[twenty_train.target[0]])
comp.graphics

scikit-learn

target_names

target

>>> twenty_train.target[:10]
array([1, 1, 3, 3, 3, 3, 3, 2, 2, 2])

>>> for t in twenty_train.target[:10]:
...     print(twenty_train.target_names[t])
...
comp.graphics
comp.graphics
soc.religion.christian
soc.religion.christian
soc.religion.christian
soc.religion.christian
soc.religion.christian
sci.med
sci.med
sci.med

从文本文件中提取特征

词袋

#i

w

X[i, j]

#j

j

w

n_features

n_samples == 10000

X

scipy.sparse

scikit-learn

使用 

scikit-learn

 来对文本进行分词

>>> from sklearn.feature_extraction.text import CountVectorizer
>>> count_vect = CountVectorizer()
>>> X_train_counts = count_vect.fit_transform(twenty_train.data)
>>> X_train_counts.shape
(2257, 35788)

CountVectorizer

>>> count_vect.vocabulary_.get(u'algorithm')
4690

从出现次数到出现频率

>>> from sklearn.feature_extraction.text import TfidfTransformer
>>> tf_transformer = TfidfTransformer(use_idf=False).fit(X_train_counts)
>>> X_train_tf = tf_transformer.transform(X_train_counts)
>>> X_train_tf.shape
(2257, 35788)

fit(..)

transform(..)

fit_transform(..)

>>> tfidf_transformer = TfidfTransformer()
>>> X_train_tfidf = tfidf_transformer.fit_transform(X_train_counts)
>>> X_train_tfidf.shape
(2257, 35788)

训练分类器

scikit-learn

>>> from sklearn.naive_bayes import MultinomialNB
>>> clf = MultinomialNB().fit(X_train_tfidf, twenty_train.target)

transform

fit_transform

>>> docs_new = ['God is love', 'OpenGL on the GPU is fast']
>>> X_new_counts = count_vect.transform(docs_new)
>>> X_new_tfidf = tfidf_transformer.transform(X_new_counts)

>>> predicted = clf.predict(X_new_tfidf)

>>> for doc, category in zip(docs_new, predicted):
...     print('%r => %s' % (doc, twenty_train.target_names[category]))
...
'God is love' => soc.religion.christian
'OpenGL on the GPU is fast' => comp.graphics

构建 Pipeline（管道）

Pipeline

>>> from sklearn.pipeline import Pipeline
>>> text_clf = Pipeline([('vect', CountVectorizer()),
...                      ('tfidf', TfidfTransformer()),
...                      ('clf', MultinomialNB()),
... ])

vect

tfidf

clf

>>> text_clf.fit(twenty_train.data, twenty_train.target)
Pipeline(...)

评估测试集上的性能

>>> import numpy as np
>>> twenty_test = fetch_20newsgroups(subset='test',
...     categories=categories, shuffle=True, random_state=42)
>>> docs_test = twenty_test.data
>>> predicted = text_clf.predict(docs_test)
>>> np.mean(predicted == twenty_test.target)
0.834...

>>> from sklearn.linear_model import SGDClassifier
>>> text_clf = Pipeline([('vect', CountVectorizer()),
...                      ('tfidf', TfidfTransformer()),
...                      ('clf', SGDClassifier(loss='hinge', penalty='l2',
...                                            alpha=1e-3, random_state=42,
...                                            max_iter=5, tol=None)),
... ])
>>> text_clf.fit(twenty_train.data, twenty_train.target)
Pipeline(...)>>> predicted = text_clf.predict(docs_test)
>>> np.mean(predicted == twenty_test.target)
0.912...

scikit-learn

>>> from sklearn import metrics
>>> print(metrics.classification_report(twenty_test.target, predicted,
...     target_names=twenty_test.target_names))
...
precision    recall  f1-score   support

alt.atheism       0.95      0.81      0.87       319
comp.graphics       0.88      0.97      0.92       389
sci.med       0.94      0.90      0.92       396
soc.religion.christian       0.90      0.95      0.93       398

avg / total       0.92      0.91      0.91      1502

>>> metrics.confusion_matrix(twenty_test.target, predicted)
array([[258,  11,  15,  35],
[  4, 379,   3,   3],
[  5,  33, 355,   3],
[  5,  10,   4, 379]])

使用网格搜索进行参数调优

TfidfTransformer

use_idf

MultinomialNB

alpha

SGDClassifier

alpha

help

>>> from sklearn.model_selection import GridSearchCV
>>> parameters = {'vect__ngram_range': [(1, 1), (1, 2)],
...               'tfidf__use_idf': (True, False),
...               'clf__alpha': (1e-2, 1e-3),
... }

n_jobs

-1

>>> gs_clf = GridSearchCV(text_clf, parameters, n_jobs=-1)

scikit-learn

>>> gs_clf = gs_clf.fit(twenty_train.data[:400], twenty_train.target[:400])

GridSearchCV

fit

predict

>>> twenty_train.target_names[gs_clf.predict(['God is love'])[0]]
'soc.religion.christian'

best_score_

best_params_

>>> gs_clf.best_score_
0.900...
>>> for param_name in sorted(parameters.keys()):
...     print("%s: %r" % (param_name, gs_clf.best_params_[param_name]))
...
clf__alpha: 0.001
tfidf__use_idf: True
vect__ngram_range: (1, 1)

gs_clf.cv_results_

cv_results_

DataFrame

练习

% cp -r skeletons workspace

[1] %run workspace/exercise_XX_script.py arg1 arg2 arg3

%debug

练习 1：语言识别

CharNGramAnalyzer

%run workspace/exercise_01_language_train_model.py data/languages/paragraphs/

练习 2：电影评论上的情绪分析

%run workspace/exercise_02_sentiment.py data/movie_reviews/txt_sentoken/

练习 3：CLI 文本分类实用程序

cPickle

快速链接

CountVectorizer

analyzer

token normalisation

CountVectorizer