python scikit learn 文本分类

<pre style="font-family: 'Courier New'; font-size: 17pt; background-color: rgb(255, 255, 255);"><pre name="code" class="python">#!/usr/bin/env python
#coding:utf-8

import jieba
from sklearn.feature_extraction.text import HashingVectorizer
import sys
import random
from sklearn.naive_bayes import MultinomialNB
import numpy as np

def extract_text_list(filename):
with open(filename,"r") as file:
return [line.strip().decode('utf-8') for line in file]

def split_data(inputlist,split_ratio):
#拆分训练集与测试集
random.shuffle(inputlist)
split_num = int(len(inputlist)*split_ratio)
train_data = inputlist[:split_num]
test_data = inputlist[split_num:]
return (train_data,test_data)

if __name__ == '__main__':
reload(sys)
sys.setdefaultencoding("utf-8")
if len(sys.argv) != 3:
print "Usage: %s <msgFile> <stopWordFile>" % sys.argv[0]
sys.exit(1)

input_file = sys.argv[1]
stop_word_file = sys.argv[2]
cn_stop_words = extract_text_list(stop_word_file)

text_list = extract_text_list(input_file)
print "文件中的消息数：%d" % len(text_list)

train_data, test_data = split_data(text_list,0.6)
print "训练集消息数：%d; 测试集消息数：%d" % (len(train_data),len(test_data))

comma_tokenizer = lambda x: jieba.cut(x, cut_all=False)
vectorizer = HashingVectorizer(encoding='utf-8',decode_error='ignore',tokenizer=comma_tokenizer,stop_words=cn_stop_words,non_negative=True)

train_corpus = [''.join(text.split("\t")[1:]) for text in train_data]
test_corpus = [''.join(text.split("\t")[1:]) for text in test_data]

train_text = vectorizer.fit_transform(train_corpus)
test_text = vectorizer.fit_transform(test_corpus)
train_tags = np.asarray([text.split("\t")[0] for text in train_data])
test_tags = [text.split("\t")[0] for text in test_data]

clf = MultinomialNB()
clf.fit(train_text.todense(),train_tags)
pred = clf.predict(test_text.todense())

true_positive = 0
for i in xrange(len(pred)):
if int(pred[i]) == int(test_tags[i]) and int(test_tags[i]) == 1:
true_positive += 1

pred_positive = sum([int(pred[i]) for i in xrange(len(pred))])
print "预测准确率：%f" % float(true_positive*1.0/pred_positive)

total_positive_num = sum([int(i) for i in test_tags])
print "预测召回率：%f" % float(true_positive*1.0/total_positive_num)