基于keras的fasttext短文本分类

#!/usr/bin/env python
# coding=utf-8

import codecs
import simplejson as json
import numpy as np
import pandas as pd
from keras.models import Sequential, load_model
from keras.callbacks import EarlyStopping, ModelCheckpoint
from keras.preprocessing import sequence
from keras.utils import to_categorical
from keras.layers import *
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import LabelEncoder
from sklearn.externals import joblib
import logging
import re
import pickle as pkl

logging.basicConfig(level=logging.INFO, format='%(asctime)s %(filename)s: %(message)s', datefmt='%Y-%m-%d %H:%M', filename='log/train_model.log', filemode='a+')

ngram_range = 1
max_features = 6500
maxlen = 120

fw = open('error_line_test.txt', 'wb')

DIRTY_LABEL = re.compile('\W+')
# set([u'业务',u'代销',u'施工',u'策划',u'设计',u'销售',u'除外',u'零售',u'食品'])
STOP_WORDS = pkl.load(open('./data/stopwords.pkl'))

def load_data(fname='data/12315_industry_business_train.csv', nrows=None):
"""
载入训练数据
"""
data, labels = [], []
char2idx = json.load(open('data/char2idx.json'))
used_keys = set(['name', 'business'])
df = pd.read_csv(fname, encoding='utf-8', nrows=nrows)
for idx, item in df.iterrows():
item = item.to_dict()
line = ''
for key, value in item.iteritems():
if key in used_keys:
line += key+value

data.append([char2idx[char] for char in line if char in char2idx])
labels.append(item['label'])

le = LabelEncoder()
logging.info('%d nb_class: %s' % (len(np.unique(labels)), str(np.unique(labels))))
onehot_label = to_categorical(le.fit_transform(labels))
joblib.dump(le, 'model/tgind_labelencoder.h5')
x_train, x_test, y_train, y_test = train_test_split(data, onehot_label, test_size=0.1)
return (x_train, y_train), (x_test, y_test)

def create_ngram_set(input_list, ngram_value=2):
return set(zip(*[input_list[i:] for i in range(ngram_value)]))

def add_ngram(sequences, token_indice, ngram_range=2):
"""
Augment the input list of sequences by appending n-grams values

"""
new_sequences = []
for input_list in sequences:
new_list = input_list[:]
for i in range(len(new_list) - ngram_range + 1):
for ngram_value in range(2, ngram_range+1):
ngram = tuple(new_list[i:i+ngram_value])
if ngram in token_indice:
new_list.append(token_indice[ngram])
new_sequences.append(new_list)

return new_sequences

(x_train, y_train), (x_test, y_test) = load_data()
nb_class = y_train.shape[1]

logging.info('x_train size: %d' % (len(x_train)))
logging.info('x_test size: %d' % (len(x_test)))
logging.info('x_train sent average len: %.2f' % (np.mean(list(map(len, x_train)))))
print 'x_train sent avg length: %.2f' % (np.mean(list(map(len, x_train))))

if ngram_range>1:
print 'add {}-gram features'.format(ngram_range)
ngram_set = set()
for input_list in x_train:
for i in range(2, ngram_range+1):
set_of_ngram = create_ngram_set(input_list, ngram_value=i)
ngram_set.update(set_of_ngram)

start_index = max_features + 1
token_indice = {v: k+start_index for k,v in enumerate(ngram_set)}
indice_token = {token_indice[k]: k for k in token_indice}

max_features = np.max(list(indice_token.keys()))+1

x_train = add_ngram(x_train, token_indice, ngram_range)
x_test = add_ngram(x_test, token_indice, ngram_range)

print 'pad sequences (samples x time)'
x_train = sequence.pad_sequences(x_train, maxlen=maxlen, padding='post', truncating='post')
x_test = sequence.pad_sequences(x_test, maxlen=maxlen, padding='post', truncating='post')

logging.info('x_train.shape: %s' % (str(x_train.shape)))

print 'build model...'

def cal_accuracy(x_test, y_test):
"""
准确率统计
"""
y_test = np.argmax(y_test, axis=1)
y_pred = model.predict_classes(x_test)
correct_cnt = np.sum(y_pred==y_test)
return float(correct_cnt)/len(y_test)

DEBUG = False
if DEBUG:
model = Sequential()
model.add(Embedding(max_features, 200, input_length=maxlen))
model.add(GlobalAveragePooling1D())
model.add(Dropout(0.3))
model.add(Dense(nb_class, activation='softmax'))
else:
model = load_model('./model/tgind_dalei.h5')

#model.compile(loss='categorical_crossentropy', optimizer='rmsprop', metrics=['accuracy'])
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
earlystop = EarlyStopping(monitor='val_loss', patience=8)
checkpoint = ModelCheckpoint(filepath='./model/tgind_dalei.h5', monitor='val_loss', save_best_only=True, save_weights_only=False)

model.fit(x_train, y_train, shuffle=True, batch_size=64, epochs=80, validation_split=0.1, callbacks=[checkpoint, earlystop])

loss, acc = model.evaluate(x_test, y_test)
print '\n\nlast model: loss', loss
print 'acc', acc

model = load_model('model/tgind_dalei.h5')
loss, acc = model.evaluate(x_test, y_test)
print '\n\n cur best model: loss', loss
print 'accuracy', acc
logging.info('loss: %.4f ;accuracy: %.4f' % (loss, acc))

logging.info('\nmodel acc: %.4f' % acc)
logging.info('\nmodel config:\n %s' % model.get_config())

#!/usr/bin/env python
# coding=utf-8

import matplotlib.pyplot as plt
from api_tgind import TgIndustry
import pandas as pd
import codecs
import json
from collections import OrderedDict

###########  根据阈值计算准确率  ###########

def cal_model_acc(model, fname='./data/industry_dalei_test_sample2k.txt', nrows=None):
"""
载入数据, 并计算前5的准确率
"""
res = {}
res['y_pred'] = []
res['y_true'] = []
with codecs.open(fname, encoding='utf-8') as fr:
for idx, line in enumerate(fr):
tokens = line.strip().split()
if len(tokens)>3:
tokens, label = tokens[:-1], tokens[-1].replace('__labe__', '')
tmp = {}
tmp['business'] = ''.join(tokens)
res['y_pred'].append(model.predict(tmp))
res['y_true'].append(label)
if nrows and idx>nrows:
break
json.dump(res, codecs.open('log/total_acc_output.json', 'wb', encoding='utf-8'))
return res

def cal_model_acc2(model, fname='data/test_12315_industry_business_sample100.csv', nrows=None):
"""
直接根据csv预测结果
"""
res = {}
res['y_pred'] = []
res['y_true'] = []
df = pd.read_csv(fname, encoding='utf-8')
for idx, item in df.iterrows():
try:
res['y_pred'].append(model.predict(item.to_dict()))
except Exception as e:
print e
print idx
print item['name']
continue
res['y_true'].append(item['label'])

if nrows and idx>nrows:
break
json.dump(res, codecs.open('log/total_acc_output.json', 'wb', encoding='utf-8'))
return res

def get_model_acc_menlei(res, topk=5, threhold=0.8):
"""
根据阈值计算模型准确率
"""
correct_cnt, total_cnt = 0, 0
for idx, y_pred in enumerate(res['y_pred']):
y_pred_tuple = sorted(y_pred.iteritems(), key=lambda x:float(x[1]), reverse=True)  # 概率排序
y_pred = OrderedDict()
for c, s in y_pred_tuple:
y_pred[c] = float(s)

if y_pred.values()[0] > threhold:    # 最大类别概率大于阈值threhold
if res['y_true'][idx][0] in map(lambda x:x[0], y_pred.keys()[:topk]):
correct_cnt += 1
total_cnt += 1
acc = float(correct_cnt)/total_cnt
recall = float(total_cnt)/len(res['y_true'])
return acc, recall

def get_model_acc_dalei(res, topk=5, threhold=0.8):
"""
根据阈值计算模型准确率
"""
correct_cnt, total_cnt = 0, 0
for idx, y_pred in enumerate(res['y_pred']):
y_pred_tuple = sorted(y_pred.iteritems(), key=lambda x:float(x[1]), reverse=True)  # 概率排序
y_pred = OrderedDict()
for c, s in y_pred_tuple:
y_pred[c] = float(s)

if y_pred.values()[0] >= threhold:    # 最大类别概率大于阈值threhold
if res['y_true'][idx] in y_pred.keys()[:topk]:
correct_cnt += 1
total_cnt += 1

acc = float(correct_cnt)/total_cnt
recall = float(total_cnt)/len(res['y_true'])
return acc, recall

def plot_accuracy(title, df, number):
"""
准确率绘图
"""
for topk in range(1, 5):
tmpdf = df[df.topk==topk]
fig = plt.figure()
ax1 = fig.add_subplot(111)
plt.subplots_adjust(top=0.85)
ax1.plot(tmpdf['threhold'], tmpdf['accuracy'], 'ro-', label='accuracy')
#        ax2 = ax1.twinx()
ax1.plot(tmpdf['threhold'], tmpdf['recall'], 'g^-', label='recall')
ax1.set_ylim(0.3, 1.0)
ax1.legend(loc=3)
ax1.set_xlabel('threhold')
plt.grid(True)
plt.title('%s Industry Classify Result\n topk=%d, number=%d\n' % (title, topk, number))
plt.savefig('log/test_%s_acc_topk%d.png' % (title, topk))
print topk, 'done!'

def gen_plot_data(model_acc, ctype='2nd'):
"""
生成图数据
"""
res = {}
res['accuracy'] = []
res['threhold'] = []
res['topk'] = []
res['recall'] = []
for topk in range(1,5):
for threhold in range(0, 10):
threhold = 0.1*threhold
if ctype == '1st':
acc, recall = get_model_acc_menlei(model_acc, topk, threhold)
else:
acc, recall = get_model_acc_dalei(model_acc, topk, threhold)
res['accuracy'].append(acc)
res['recall'].append(recall)
res['threhold'].append(threhold)
res['topk'].append(topk)
print ctype, topk, acc
json.dump(res, open('log/test_model_threshold_%s.log' % ctype, 'wb'))
df = pd.DataFrame(res)
df.to_csv('log/test_model_result_%s.csv' % ctype, index=False)
plot_accuracy(ctype, df, len(model_acc['y_true']))
return df

if __name__=='__main__':

model = TgIndustry()
# model_acc = cal_model_acc2(model, fname='data/test_12315_industry_business_sample100.csv')
model_acc = json.load(codecs.open('log/total_acc_output_12315.json', encoding='utf-8'))
gen_plot_data(model_acc, '1st')
gen_plot_data(model_acc, '2nd')

#!/usr/bin/env python
# coding=utf-8

import numpy as np
import codecs
import simplejson as json
from keras.models import load_model
from keras.preprocessing import sequence
from sklearn.externals import joblib
from collections import OrderedDict
import pickle as pkl
import re, os
import jieba
import time

"""
行业分类调用Api

__author__: jkmiao
__date__: 2017-07-05

"""

class TgIndustry(object):

def __init__(self, model_path='model/tgind_dalei_acc76.h5'):

base_path = os.path.dirname(__file__)
model_path = os.path.join(base_path, model_path)

# 载入预训练好的模型
self.model = load_model(model_path)
# 载入labelEncoder
self.le = joblib.load(os.path.join(base_path, './model/tgind_labelencoder.h5'))
# 载入字符映射表
self.char2idx = json.load(open(os.path.join(base_path, 'data/char2idx.json')))
# 载入停用词表
# self.stop_words = set([line.strip() for line in codecs.open('./data/stopwords.txt', encoding='utf-8')])
self.stop_words = pkl.load(open(os.path.join(base_path, './data/stopwords.pkl')))
# 载入类别最终的编号和名称映射
self.menlei_label2name = json.load(open(os.path.join(base_path, 'data/menlei_label2name.json')))  # 一级分类
self.dalei_label2name = json.load(open(os.path.join(base_path, 'data/dalei_label2name.json'))) # 二级分类

def predict(self, company_info, topk=2, firstIndustry=False, final_name=False):
"""
:type company_info: 公司相关信息
:rtype business: str: 对应 label
"""
line = ''
for key, value in company_info.iteritems():
if key in ['name', 'business']: # 公司信息, 目前取公司名和经营范围
line +=  company_info[key]

if not isinstance(line, unicode):
line = line.decode('utf-8')

# 去除停用词后的句子
line = ''.join([token for token in jieba.cut(line) if token not in self.stop_words])
data = [self.char2idx[char] for char in line if char in self.char2idx]
data = sequence.pad_sequences([data], maxlen=100, padding='post', truncating='post')
y_pred_proba = self.model.predict(data, verbose=0)
y_pred_idx_list = [c[-topk:][::-1] for c in np.argsort(y_pred_proba, axis=-1)][0]
res = OrderedDict()
for y_pred_idx in y_pred_idx_list:
y_pred_label = self.le.inverse_transform(y_pred_idx)
if final_name:
y_pred_label = self.dalei_label2name[y_pred_label]
if firstIndustry:
res[y_pred_label[0]] = round(y_pred_proba[0, y_pred_idx], 3) # 概率保留3位小数
res[y_pred_label] = round(y_pred_proba[0, y_pred_idx], 3) # 概率保留3位小数
return res

if __name__ == '__main__':

DIRTY_LABEL = re.compile('\W+')
test = TgIndustry()
cnt, total_cnt = 0, 0
start_time = time.time()
fw2 = codecs.open('./output/industry_dalei_test_sample2k_error.txt', 'wb', encoding='utf-8')
with codecs.open('./data/industry_dalei_test_sample2k.txt', encoding='utf-8') as fr:
for idx, line in enumerate(fr):
tokens = line.strip().split()
if len(tokens)>3:
tokens, label = tokens[:-1], tokens[-1].replace('__label__', '')
if len(label) not in [2, 3] or DIRTY_LABEL.search(label):
print 'error line:'
print idx, line, label
continue
tmp = {}
tmp['business'] = ''.join(tokens)
y_pred = test.predict(tmp, topk=1)
if label in y_pred:
cnt += 1
elif y_pred.values()[0] < 0.3:
print 'error: ', ''.join(tokens), y_pred, 'y_true:', label
fw2.write(''.join(tokens))
total_cnt +=1
print label
print json.dumps(y_pred, ensure_ascii=False)
print idx, '=='*20, float(cnt)/total_cnt
if idx>200:
break
print 'avg cost time:', float(time.time()-start_time)/idx