bagofwords tf-idf word2vec特征实践

1 bagofwords + bayes

import pandas as pd
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.naive_bayes import MultinomialNB
from sklearn.metrics import accuracy_score , roc_auc_score , roc_curve
import matplotlib.pyplot as plt

#数据处理
def data_prepare():
df = pd.read_excel("window regulator01.xlsx")
split = 0.7
Nodf = df[df.categories == 0]
print("0样本数量 = %d" %len(Nodf))
Yesdf = df[df.categories == 1]
print("1样本数量 = %d" %len(Yesdf))
d_train = Nodf[:int(split * len(Nodf))]
d_train = pd.concat([d_train , Yesdf[:int(split * len(Yesdf))]])
d_test = Nodf[int(split * len(Nodf)):]
d_test = pd.concat([d_test ,Yesdf[int(split * len(Yesdf) ):] ])
print("训练样本：测试样本 = %.1f" %split)
print("训练样本 = %d" %len(d_train))
print("测试样本 = %d" %len(d_test))
return d_train , d_test

def create_model(d_train , d_test):
vectorizer = CountVectorizer() #词袋特征抽取
features = vectorizer.fit_transform(d_train.title)
print("训练样本特征表长度为 " + str(features.shape))
# print(vectorizer.get_feature_names()[3000:3050]) #特征名展示
test_features = vectorizer.transform(d_test.title)
#贝叶斯构建模型
NBmodle = MultinomialNB()
print("训练中。。。")
NBmodle.fit(features , d_train.categories)
print("测试中。。。")
predict = NBmodle.predict_proba(test_features)
y_true = d_test.categories
acc = accuracy_score(y_true , predict[:,1] > 0.5)
auc = roc_auc_score(y_true, predict[:, 1])
print("acc = %0.2f" %acc )
print("AUC = %0.2f" % auc )
return y_true , predict
#模型评估
def performance(y_true , predict ):
acc = accuracy_score(y_true , predict[:,1] > 0.5)
auc = roc_auc_score(y_true, predict[:, 1])
fpr, tpr , thr = roc_curve(y_true , predict[:,1])
# plt.ion() #开启interactive mode
plt.plot(fpr , tpr )
plt.xlabel("False positive rate")
plt.ylabel("True positive rate")
plt.annotate("Acc : %0.2f" % acc , (0.2 , 0.7) , size = 14)
plt.annotate("AUC:%0.2f" %auc , (0.2 , 0.6) , size = 14)
plt.show()

d_train , d_test = data_prepare()
y_true, predict = create_model(d_train , d_test )
performance(y_true, predict)

2 tf-idf + bayes

import pandas as pd
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.naive_bayes import MultinomialNB
from sklearn.metrics import accuracy_score , roc_auc_score , roc_curve
import matplotlib.pyplot as plt

#数据处理
def data_prepare():
df = pd.read_excel("window regulator01.xlsx")
split = 0.7
Nodf = df[df.categories == 0]
print("0样本数量 = %d" %len(Nodf))
Yesdf = df[df.categories == 1]
print("1样本数量 = %d" %len(Yesdf))
d_train = Nodf[:int(split * len(Nodf))]
d_train = pd.concat([d_train , Yesdf[:int(split * len(Yesdf))]])
d_test = Nodf[int(split * len(Nodf)):]
d_test = pd.concat([d_test ,Yesdf[int(split * len(Yesdf) ):] ])
print("训练样本：测试样本 = %.1f" %split)
print("训练样本 = %d" %len(d_train))
print("测试样本 = %d" %len(d_test))
return d_train , d_test

def create_model(d_train , d_test):
vectorizer = TfidfVectorizer() #词袋特征抽取
features = vectorizer.fit_transform(d_train.title)
print("训练样本特征表长度为 " + str(features.shape))
# print(vectorizer.get_feature_names()[3000:3050]) #特征名展示
test_features = vectorizer.transform(d_test.title)
#贝叶斯构建模型
NBmodle = MultinomialNB()
print("训练中。。。")
NBmodle.fit(features , d_train.categories)
print("测试中。。。")
predict = NBmodle.predict_proba(test_features)
y_true = d_test.categories
acc = accuracy_score(y_true , predict[:,1] > 0.5)
auc = roc_auc_score(y_true, predict[:, 1])
print("acc = %0.2f" %acc )
print("AUC = %0.2f" % auc )
return y_true , predict
#模型评估
def performance(y_true , predict ):
acc = accuracy_score(y_true , predict[:,1] > 0.5)
auc = roc_auc_score(y_true, predict[:, 1])
fpr, tpr , thr = roc_curve(y_true , predict[:,1])
# plt.ion() #开启interactive mode
plt.plot(fpr , tpr  )
plt.xlabel("False positive rate")
plt.ylabel("True positive rate")
plt.annotate("Acc : %0.2f" % acc , (0.2 , 0.7) , size = 14)
plt.annotate("AUC:%0.2f" %auc , (0.2 , 0.6) , size = 14)
plt.show()

d_train , d_test = data_prepare()
y_true, predict = create_model(d_train , d_test )
performance(y_true, predict)

3 tf-idf + bayes 参数优化

import pandas as pd
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.naive_bayes import MultinomialNB
from sklearn.metrics import accuracy_score , roc_auc_score , roc_curve
import matplotlib.pyplot as plt
from itertools import product
import csv

#数据处理
def data_prepare():
df = pd.read_excel("window regulator01.xlsx")
split = 0.7
Nodf = df[df.categories == 0]
print("0样本数量 = %d" %len(Nodf))
Yesdf = df[df.categories == 1]
print("1样本数量 = %d" %len(Yesdf))
d_train = Nodf[:int(split * len(Nodf))]
d_train = pd.concat([d_train , Yesdf[:int(split * len(Yesdf))]])
d_test = Nodf[int(split * len(Nodf)):]
d_test = pd.concat([d_test ,Yesdf[int(split * len(Yesdf) ):] ])
print("训练样本：测试样本 = %.1f" %split)
print("训练样本 = %d" %len(d_train))
print("测试样本 = %d" %len(d_test))
return d_train , d_test

def create_model(d_train , d_test):
vectorizer = TfidfVectorizer() #特征抽取
features = vectorizer.fit_transform(d_train.title)
print("训练样本特征表长度为 " + str(features.shape))
# print(vectorizer.get_feature_names()[3000:3050]) #特征名展示
test_features = vectorizer.transform(d_test.title)
#贝叶斯构建模型
NBmodle = MultinomialNB()
print("训练中。。。")
NBmodle.fit(features , d_train.categories)
print("测试中。。。")
predict = NBmodle.predict_proba(test_features)
y_true = d_test.categories
acc = accuracy_score(y_true , predict[:,1] > 0.5)
auc = roc_auc_score(y_true, predict[:, 1])
print("acc = %0.2f" %acc )
print("AUC = %0.2f" % auc )
return y_true , predict

#参数优化的模型
def create_model_param(d_train , d_test , max_features = None , min_df = 1, nb_alpha = 1.0):
vectorizer = TfidfVectorizer(max_features=max_features , min_df=min_df)
features = vectorizer.fit_transform(d_train.title)
print("训练样本特征表长度为 " + str(features.shape))
# print(vectorizer.get_feature_names()[3000:3050]) #特征名展示
test_features = vectorizer.transform(d_test.title)
#贝叶斯构建模型
NBmodle = MultinomialNB(alpha= nb_alpha)
print("训练中。。。")
NBmodle.fit(features , d_train.categories)
print("测试中。。。")
predict = NBmodle.predict_proba(test_features)
y_true = d_test.categories
auc = roc_auc_score(y_true, predict[:, 1])
print("AUC = %0.2f" % auc )
return {
"max_feature" : max_features,
"min_df":min_df,
"nb_alpha":nb_alpha,
"AUC":auc
}
#模型评估
def performance(y_true , predict ):
acc = accuracy_score(y_true , predict[:,1] > 0.5)
auc = roc_auc_score(y_true, predict[:, 1])
fpr, tpr , thr = roc_curve(y_true , predict[:,1])
# plt.ion() #开启interactive mode
plt.plot(fpr , tpr )
plt.xlabel("False positive rate")
plt.ylabel("True positive rate")
plt.annotate("Acc : %0.2f" % acc , (0.2 , 0.7) , size = 14)
plt.annotate("AUC:%0.2f" %auc , (0.2 , 0.6) , size = 14)
plt.show()

#参数组合
param_values = {
"max_feature": [1000 , 2000 , 3000 , None],
"min_df": [1,2,3],
"nb_alpha": [0.01 , 0.1 , 1.0],
}
result = []
file = open("result.csv" , "w",newline= "")
writer = csv.DictWriter(file , fieldnames=["AUC","max_feature","min_df","nb_alpha"])
writer.writeheader()
d_train , d_test = data_prepare()
for p in product(*param_values.values()):
print(p)
res = create_model_param(d_train , d_test , p[0],p[1],p[2])
result.append(res)
print(res)
writer.writerow(res)

4word2vec + randforest

import pandas as pd
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.naive_bayes import MultinomialNB
from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import accuracy_score , roc_auc_score , roc_curve
import matplotlib.pyplot as plt
from itertools import product
import csv
from gensim.models.word2vec import Word2Vec
import nltk
import numpy as np

#数据处理
def data_prepare():
df = pd.read_excel("window regulator01.xlsx")
split = 0.7
Nodf = df[df.categories == 0]
print("0样本数量 = %d" %len(Nodf))
Yesdf = df[df.categories == 1]
print("1样本数量 = %d" %len(Yesdf))
d_train = Nodf[:int(split * len(Nodf))]
d_train = pd.concat([d_train , Yesdf[:int(split * len(Yesdf))]])
d_test = Nodf[int(split * len(Nodf)):]
d_test = pd.concat([d_test ,Yesdf[int(split * len(Yesdf) ):] ])
print("训练样本：测试样本 = %.1f" %split)
print("训练样本 = %d" %len(d_train))
print("测试样本 = %d" %len(d_test))
return d_train , d_test
def data_process(doc): #word2vec接受的是将每个文档当做一个列表，整个文档是列表里面的元素也是列表
sentences = []
for d in doc:
d = d.lower()
words = nltk.word_tokenize(d)
sentences.append(words)
return sentences
def featurize_w2v(model, sentences): #文档向量化
f= np.zeros((len(sentences),model.vector_size)) #文档特征为文档中所有单词的平均向量
for i , s in enumerate(sentences):
for w in s:
try:
vec = model[w]
except KeyError:
continue
f[i,:] = f[i,:] + vec
f[i,:] = f[i,:] / len(s)
return f
def create_model(d_train , d_test):
sentences = data_process(d_train.title)
model = Word2Vec(sentences , size = 300 , window=1 , min_count=1 , sample= 1e-3 , worke
b3d0
rs=2)
model.init_sims(replace=True)
feature_train = featurize_w2v(model, sentences) #word2vec特征抽取
RFCmodel =RandomForestClassifier(n_estimators= 100 , n_jobs= -1)
RFCmodel.fit(feature_train, d_train.categories)

test_sentences = data_process(d_test.title)
feature_test = featurize_w2v(model , test_sentences)
predict = RFCmodel.predict_proba(feature_test)
return d_test.categories , predict
#模型评估
def performance(y_true , predict ):
acc = accuracy_score(y_true , predict[:,1] > 0.5)
auc = roc_auc_score(y_true, predict[:, 1])
fpr, tpr , thr = roc_curve(y_true , predict[:,1])
# plt.ion() #开启interactive mode
plt.plot(fpr , tpr )
plt.xlabel("False positive rate")
plt.ylabel("True positive rate")
plt.annotate("Acc : %0.2f" % acc , (0.2 , 0.7) , size = 14)
plt.annotate("AUC:%0.2f" %auc , (0.2 , 0.6) , size = 14)
plt.show()

d_train , d_test = data_prepare()
y_true , predict = create_model(d_train , d_test)
performance(y_true , predict )