一个简单问答系统搭建代码（附步骤流程）

首先是这个简单问答系统搭建的步骤流程：

因为我这里读文件用的是绝对路径，所以要用的同学需要自己改一下，代码中涉及到的文件有需要的点以下下载：
链接：https://pan.baidu.com/s/14srw2A_RIgTc7ejP3KJQaQ
提取码：n4n5

import json
from collections import Counter
import matplotlib.pyplot as plt
from nltk.stem import PorterStemmer
from nltk.tokenize import word_tokenize
import numpy as np
from gensim.models import KeyedVectors
from gensim.scripts.glove2word2vec import glove2word2vec
import math
from collections import defaultdict
from queue import PriorityQueue
#对文本进行问题和答案分开对应
def read_corpus(filepath):
qlist = []
alist = []
with open(filepath) as f:
file_array = json.load(f)['data']
for file in file_array:
paragraph = file['paragraphs']
for paragraphs in paragraph:
paragraph = paragraphs['qas']
for qa in paragraph:
qlist.append(qa['question'])
try:
alist.append(qa['answers'][0]['text'])
except IndexError:
qlist.pop()
assert len(qlist) == len(alist)  # 确保长度一样
return qlist, alist

def text_preprocessing(text):
# 生成停用词和标准化
stopfile_path = r'C:\Users\Administrator\nltk_data\corpora\stopwords\baidu_stopwords.txt'
with open(stopfile_path, 'r', encoding='UTF-8') as f:
sw = set(f.read())
sw -= {'when', 'who', 'why', 'what', 'how', 'where', 'which'}
ps = PorterStemmer()

seg = list()
#用nltk分词
for word in word_tokenize(text):
#小写化,次干提取
word = ps.stem(word.lower())
#数值归一
word = '#number' if word.isdigit() else word
#去停用词
if len(word)>1 and word not in sw:
seg.append(word)
return seg
#对问题库进行文本处理
def qlist_preprocessing(qlist):
word_cnt = Counter()
qlist_seg = list()
for text in qlist:
seg = text_preprocessing(text)
qlist_seg.append(seg)
word_cnt.update(seg)

value_sort = sorted(word_cnt.values(),reverse=True)
min_tf = value_sort[int(math.exp(0.99*math.log(len(word_cnt))))]
for cur in range(len(qlist_seg)):
qlist_seg[cur] = [word for word in qlist_seg[cur] if word_cnt[word] > min_tf]
return qlist_seg

# word2vec
model = KeyedVectors.load_word2vec_format(r'C:\Users\Administrator\Desktop\glove2word2vec.6B.100d.txt')
#将分词数据转换成句向量
def docvec_get(seg):
vector = np.zeros((1,100))
size = len(seg)
for word in seg:
try:
vector += model.wv[word]
except KeyError:
size -= 1
return vector/size

def top5results_invidx(input_q):
qlist, alist = read_corpus(r'C:\Users\Administrator\Desktop\train-v2.0.json')
alist = np.array(alist)
qlist_seg = qlist_preprocessing(qlist)  #对qlist进行处理
seg = text_preprocessing(input_q)   #对输入的问题进行处理

#将已经处理好的问题库数据词向量化
X = np.zeros((len(qlist_seg),100))
for cur in range(X.shape[0]):
X[cur] = docvec_get(qlist_seg[cur])
Nnorm2 = np.linalg.norm(X,axis=1,keepdims=True)
X = X / Nnorm2

#定义一个简单的倒排表
inversed_idx = defaultdict(set)
for cur in range(len(qlist_seg)):
for word in qlist_seg[cur]:
inversed_idx[word].add(cur)
candiates = set()
for word in seg:
#取所有包含任意一词的文档的并集
candiates = candiates | inversed_idx[word]
candiates = list(candiates)
#将预处理完的输出问题词向量化
input_vec = docvec_get(seg)
#计算问题向量的L2范数
qnorm2 = np.linalg.norm(input_vec,axis=1,keepdims=True)
input_vec = input_vec / qnorm2
sim = (X[candiates] @ input_vec.T)     #计算余弦相似度，@表示矩阵向量乘法

#用优先队列找出相似度最大的5个问题对应的答案的下标
pq = PriorityQueue()
for cur in range(sim.shape[0]):
pq.put((sim[cur][0], candiates[cur]))
if len(pq.queue) > 5:
pq.get()
pq_rank = sorted(pq.queue, reverse=True, key=lambda x: x[0])
print([x[0] for x in pq_rank])
top_idxs = [x[1] for x in pq_rank]
return [alist[i] for i in top_idxs] # 返回相似度最高的问题对应的答案，作为TOP5答案
print(top5results_emb("Which airport was shut down?"))    # 在问题库中存在，经过对比，返回的首结果正确
print(top5results_emb("Which airport is closed?"))
print(top5results_emb("What government blocked aid after Cyclone Nargis?"))    # 在问题库中存在，经过对比，返回的首结果正确
print(top5results_emb("Which government stopped aid after Hurricane Nargis?"))