Tensorflow之Basic word2vec代码详解（上）

Tensorflow上关于Vector Representations of Words里给出了word2vec两个源代码，本文解析基础的代码，代码地址为：https://github.com/tensorflow/tensorflow/blob/r1.3/tensorflow/examples/tutorials/word2vec/word2vec_basic.py

上篇为代码step1-3：讲解数据下载处理，与训练数据的生成。

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import collections
import math
import os
import random
import zipfile

import numpy as np
from six.moves import urllib
from six.moves import xrange  # pylint: disable=redefined-builtin
import tensorflow as tf

# Step 1: 下载数据
url = 'http://mattmahoney.net/dc/'

def maybe_download(filename, expected_bytes):
"""Download a file if not present, and make sure it's the right size."""
if not os.path.exists(filename):
filename, _ = urllib.request.urlretrieve(url + filename, filename)
statinfo = os.stat(filename)
if statinfo.st_size == expected_bytes:
print('Found and verified', filename)
else:
print(statinfo.st_size)
raise Exception(
'Failed to verify ' + filename + '. Can you get to it with a browser?')
return filename
#
filename = maybe_download('text8.zip', 31344016)

# 解压缩并读取数据转化到数组中.
def read_data(filename):
"""Extract the first file enclosed in a zip file as a list of words."""
with zipfile.ZipFile(filename) as f:
data = tf.compat.as_str(f.read(f.namelist()[0])).split() #split(分割成序列)
return data

vocabulary = read_data(filename)
print('Data size', len(vocabulary))

#建立字典
# Step 2: Build the dictionary and replace rare words with UNK token.
vocabulary_size = 50000

def build_dataset(words, n_words):
"""Process raw inputs into a dataset."""
count = [['UNK', -1]]
count.extend(collections.Counter(words).most_common(n_words - 1))#计数，取词频前50000个词，其余的为unk，
dictionary = dict()
for word, _ in count:
dictionary[word] = len(dictionary)#高频词排序给编号
data = list()
unk_count = 0
for word in words:
if word in dictionary:
index = dictionary[word]#给高频词一个索引
else:
index = 0  # 低频词索引为0
unk_count += 1  #统计低频词的个数
data.append(index)
count[0][1] = unk_count
reversed_dictionary = dict(zip(dictionary.values(), dictionary.keys())) #逆词汇，键和值与dictionary相反
return data, count, dictionary, reversed_dictionary

data, count, dictionary, reverse_dictionary = build_dataset(vocabulary,
vocabulary_size)
del vocabulary  # Hint to reduce memory.
print('Most common words (+UNK)', count[:5])
print('Sample data', data[:10], [reverse_dictionary[i] for i in data[:10]])

#如vocabulary（daefbmc……）其中：1.a词频：600；2.b词频：500；3.c词频：400；4.d词频：300；5.e词频：200；6.f词频：100；……unk:4148
#count([UNK,4148],[a,600]，[b,500],[c,400],[d,300],[e,200,[f,100])
#dictionary([a；1]，[b:2],[c:3],[d:4],[e；5],[f；6])
#data:{4,1,5,6,2,0,3}
#reversed_dictionary([1；a]，[2:b],[3:c],[4:d],[5；e],[6；f])

#生成训练数据
#从文本总体的第二次开始，每个单词一次作为输入，输出可以是上下文范围内的单词中的任何一个（一般不是取全部而是随机抽取其中几组，增加随机性）
data_index = 0
# Step 3: Function to generate a training batch for the skip-gram model.
def generate_batch(batch_size, num_skips, skip_window):
#batch_size：每次训练的词长度；num_skips：每个输入词重复的次数（一个输入产生多少个标签数据），skip_window：向左右取多少词
global data_index  #global：全局变量
assert batch_size % num_skips == 0  #assert：断言
assert num_skips <= 2 * skip_window

batch = np.ndarray(shape=(batch_size), dtype=np.int32)  #一维
labels = np.ndarray(shape=(batch_size, 1), dtype=np.int32)  #二维
span = 2 * skip_window + 1  # 左右两边各取skip_window，一共span个
buffer = collections.deque(maxlen=span)  #防止超长，挤出前面的数据，确保span个训练数据
if data_index + span > len(data):  #依次取span个词
data_index = 0
buffer.extend(data[data_index:data_index + span])
data_index += span
for i in range(batch_size // num_skips):
target = skip_window  #butter[skip_window]为输入数据
targets_to_avoid = [skip_window] #去除输入词自己本身
for j in range(num_skips):  #输入词重复num_skips次
while target in targets_to_avoid:
target = random.randint(0, span - 1) #随机生成 (0, span - 1)之间整数
targets_to_avoid.append(target)
batch[i * num_skips + j] = buffer[skip_window] #训练输入的序列
labels[i * num_skips + j, 0] = buffer[target]  #训练输出的序列（标签，对应词频的排序）
if data_index == len(data):  #超长时回到开始
buffer[:] = data[:span]
data_index = span
else:
buffer.append(data[data_index]) #挤掉开始几个，换一组词训练
data_index += 1
# Backtrack a little bit to avoid skipping words in the end of a batch
data_index = (data_index + len(data) - span) % len(data)
return batch, labels

batch, labels = generate_batch(batch_size=8, num_skips=2, skip_window=1)
for i in range(8):
print(batch[i], reverse_dictionary[batch[i]],
'->', labels[i, 0], reverse_dictionary[labels[i, 0]])

#如：vocabulary（m|daefbm|c……）取batch_size=6,num_skips=2,skip_window=1
#batch = [4,4,1,1,5,5]
#labels= [0,1,4,5,1,6]