Tensorflow: recurrent neural network char-level 1

import numpy as np
import tensorflow as tf
import matplotlib.pyplot as plt
from tensorflow.models.rnn import rnn, rnn_cell
from tensorflow.models.rnn import seq2seq

import collections

# @karpathy
data = open('ThreeMusketeers.txt').read()
chars = list(set(data))
data_size , vocab_size = len(data), len(chars)
print 'data has %d characters, %d unique.' %(data_size, vocab_size)
char_to_ix = {ch:i for i, ch in enumerate(chars)}
ix_to_char = {i:ch for i, ch in enumerate(chars)}
counter = collections.Counter(data)
counter = sorted(counter.items(), key=lambda x:-x[1])
for i in xrange(5):
print counter[i]

corpus = [char_to_ix[c] for c in data]

batch_size = 1
seq_length = 1

hidden_size = 128
num_layers = 2
max_grad_norm = 5.0

an_lstm = rnn_cell.BasicLSTMCell(hidden_size)
multi_lstm = rnn_cell.MultiRNNCell([an_lstm] * num_layers)
x = tf.placeholder(tf.int32, [batch_size, seq_length])
y = tf.placeholder(tf.int32, [batch_size, seq_length])
init_state = multi_lstm.zero_state(batch_size, tf.float32)

with tf.variable_scope('rnn'):
softmax_w = tf.get_variable('softmax_w', [hidden_size, vocab_size])
softmax_b = tf.get_variable('softmax_b', [vocab_size])
with tf.device('/cpu:0'):
embedding = tf.get_variable('embedding', [vocab_size, hidden_size])
inputs = tf.nn.embedding_lookup(embedding, x)
inputs = tf.split(1, seq_length, inputs)
inputs = [tf.squeeze(input_, [1]) for input_ in inputs]

#def loop(prev):
#    prev = tf.nn.xw_plus_b(prev, softmax_w, softmax_b)
#    prev_symbol = tf.stop_gradient(tf.arg_max(prev, 1))
#    return tf.nn.embedding_lookup(embedding, prev_symbol)

outputs, last_state = seq2seq.rnn_decoder(inputs, init_state,
multi_lstm,
loop_function=None,
scope='rnn')
# outputs is a list of 2D-Tensor with shape [batch_size , hidden_size]
# the len(outputs)) is seq_length

# first, hiddenlayer outputs belong to same sequence should be concatenated together
out_conca = tf.concat(1, outputs) # [batch_size, hidden_size*seq_length]
# second, to get the softmax prob and add the fc layer, the out_conca's second dim should
# be reshaped to the size: hidden_size
# [batch_size*seq_length, hidden_size]
output = tf.reshape(out_conca, [-1, hidden_size])
# [batch_size*seq_length, vocab_size]
score = tf.nn.xw_plus_b(output, softmax_w, softmax_b)
# [batch_size*seq_length, vocab_size]
probs = tf.nn.softmax(score)

init = tf.initialize_all_variables()
sess = tf.Session()
sess.run(init)

epoch = 20
batch_size = 100
snapshot = 5
save_step = 1
saver = tf.train.Saver()
ckpt = tf.train.get_checkpoint_state('net_snapshot/')
saver.restore(sess, ckpt.model_checkpoint_path)
print ckpt.model_checkpoint_path

def weighted_pick(weights):
t = np.cumsum(weights)
s = np.sum(weights)
return np.searchsorted(t, np.random.rand(1)*s).tolist()[0]

prime = 'The'
state = sess.run(multi_lstm.zero_state(1, tf.float32))
for c in prime[-1]:
ix = np.zeros((1,1))
ix[0,0] = char_to_ix[c]
state = sess.run(last_state, feed_dict={x:ix, init_state: state})

def char_filter(pred):
cache = ['!', '.', '?', '\"', ' ', ',', '%']
if pred >='a' and pred <= 'z':
return pred
if pred >='A' and pred <= 'Z':
return pred
if pred in cache:
return pred
return ''

ret = prime
char = prime[-1]
num = 1000
for n in xrange(num):
ix = np.zeros((1,1))
ix[0,0] = char_to_ix[char]
probsval, state = sess.run([probs, last_state],
feed_dict={x:ix, init_state:state})

if np.random.rand(1) > 0.5:
sample = np.argmax(probsval[0])
else:
sample = weighted_pick(probsval[0])

pred = ix_to_char[sample]

ret += char_filter(pred)
char = ret[-1]

print ret

sampled chars:

The and the caral there turrid seing, acting, and I scapter the cardy and this tow in
the carmant of the cardantg wit. The sesper the carmisted the camplessing unce you will
the cardow shanded the entereled the haved is thuld a conturute of you wnocch forlack
the dal would the cardersty a saiking the contter the man the this lackul not to the
carster which he wat madame you was abver D he womple lead me the conleshs of the
cardinous of the closed there obe in his had past, acking that time the cardinal same
trrang the cant of the cat of the sir thim the deave, and the clome the ca in aftented
of throse one and all be, and proming him are to
aa20
they undersince, she dy hering thought
o man the conter to musked the cardinal be at the porsess of a seet the for the
cardinan and as the omemen exter the cardinal and to the wall take proming this, and
the cands, and then? Tetter. Tno tay will see to see thougle. The chaned them then for
a sra