用scikit-learn和jieba支持中文文本特征提取和分类的测例

注意：

安装jieba用此命令：easy_install jieba。pip install jieba有时不能正确安装。

中文情况下analyzer='word'参数需要带上，vocabulary=cv.vocabulary_这样的参数是为了使测试集和训练集特征数对齐

代码：

---------

from time import time

import sys

import os

import numpy as np

import scipy.sparse as sp

import matplotlib.pyplot as plt

import jieba

import jieba.posseg as pseg

from sklearn import feature_extraction

from sklearn.feature_extraction.text import TfidfTransformer  

from sklearn.feature_extraction.text import CountVectorizer  

from sklearn.feature_extraction.text import TfidfVectorizer

from sklearn.naive_bayes import MultinomialNB

from sklearn.linear_model import SGDClassifier

from sklearn.metrics import confusion_matrix

from sklearn.metrics import classification_report

corpus=["我爱河南。",#第一类文本切词后的结果，词之间以空格隔开 
"你恨河南。",
"他总是爱河南。",
"我有时候恨河南。"]

tokenized_corpus = []

for text in corpus:
tokenized_corpus.append(" ".join(jieba.cut(text)))

test_corpus = ["我爱河南的胡辣汤。"]

tokenized_test_corpus = []

tokenized_test_corpus.append(" ".join(jieba.cut(test_corpus[0])))

corpus_result = [1, 0, 1, 0]

#下面几个是HashingVectorizer， CountVectorizer+TfidfTransformer，TfidfVectorizer， FeatureHasher的正确用法。

#fh = feature_extraction.FeatureHasher(n_features=15,non_negative=True,input_type='string')

#X_train=fh.fit_transform(tokenized_corpus)

#X_test=fh.fit_transform(tokenized_test_corpus)

#fh = feature_extraction.text.HashingVectorizer(n_features=15,non_negative=True,analyzer='word')

#X_train=fh.fit_transform(tokenized_corpus)

#X_test=fh.fit_transform(tokenized_test_corpus)

#cv=CountVectorizer(analyzer='word')

#transformer=TfidfTransformer()

#X_train=transformer.fit_transform(cv.fit_transform(tokenized_corpus))

#cv2=CountVectorizer(vocabulary=cv.vocabulary_)

#transformer=TfidfTransformer()

#X_test = transformer.fit_transform(cv2.fit_transform(tokenized_test_corpus))

#word=cv.get_feature_names()

#weight=X_train.toarray()

#for i in range(len(weight)):

# print u"-------这里输出第",i,u"类文本的词语tf-idf权重------"  

# for j in range(len(word)):  

#            print word[j],weight[i][j] 

tfidf = TfidfVectorizer(analyzer='word')

X_train=tfidf.fit_transform(tokenized_corpus)

tfidf = TfidfVectorizer(analyzer='word', vocabulary = tfidf.vocabulary_)

X_test=tfidf.fit_transform(tokenized_test_corpus)

y_train = corpus_result

y_test = [1]

def benchmark(clf_class, params, name):

    print("parameters:", params)

    t0 = time()

    clf = clf_class(**params).fit(X_train, y_train)

    print("done in %fs" % (time() - t0))

    if hasattr(clf, 'coef_'):

        print("Percentage of non zeros coef: %f" % (np.mean(clf.coef_ != 0) * 100))

    print("Predicting the outcomes of the testing set")

    t0 = time()

    pred = clf.predict(X_test)

    print("done in %fs" % (time() - t0))

    print("Classification report on test set for classifier:")

    print(clf)

    print()

    print(classification_report(y_test, pred))

    cm = confusion_matrix(y_test, pred)

    print("Confusion matrix:")

    print(cm)

if __name__ == "__main__":  

    print("Testbenching a linear classifier...")

    parameters = {
'loss': 'hinge',
'penalty': 'l2',
'n_iter': 50,
'alpha': 0.00001,
'fit_intercept': True,

    }

    benchmark(SGDClassifier, parameters, 'SGD')

---------

参考：
http://blog.csdn.net/liuxuejiang158blog/article/details/31360765 用count和tfidf
http://www.tuicool.com/articles/vYnIve 用FeatureHasher
http://blog.csdn.net/pat_datamine/article/details/43969631 jieba, 通过复用训练集的vocabulary来达到测试集和训练集特征数对齐的目的
http://blog.csdn.net/abcjennifer/article/details/23615947 复用训练集的vocabulary