def ppl(sentence_list):
ppl_list = []
// load data dict
word_to_int = load_word_dict(conf.word_dict_path)
// init params
batch_size = 1
tf.reset_default_graph()
input_data = tf.placeholder(tf.int32, [batch_size, None])
output_targets = tf.placeholder(tf.int32, [batch_size, None])
// init model
end_points = rnn_model(model="lstm",
input_data=input_data,
output_data=output_targets,
vocab_size=len(word_to_int),
rnn_size=128,
num_layers=2,
batch_size=batch_size,
learning_rate=conf.learning_rate)
saver = tf.train.Saver(tf.global_variables())
init_op = tf.group(tf.global_variables_initializer(), tf.local_variables_initializer())
with tf.Session() as sess:
// init op
sess.run(init_op)
checkpoint = tf.train.latest_checkpoint(conf.model_dir)
saver.restore(sess, checkpoint)
print("loading model from the checkpoint {0}".format(checkpoint))
// infer each sentence
for sentence in sentence_list:
ppl = 0
// data idx
x = [word_to_int[c] if c in word_to_int else word_to_int[UNK_TOKEN] for c in sentence]
x = [word_to_int[START_TOKEN]] + x + [word_to_int[END_TOKEN]]
print("x:", x)
// reshape
y = np.array(x[1:]).reshape((-1, batch_size))
x = np.array(x[:-1]).reshape((-1, batch_size))
print(x.shape)
print(y.shape)
// get each word perplexity
word_count = x.shape[0]
for i in range(word_count):
perplexity = sess.run(end_points["perplexity"],
feed_dict={input_data: x[i:i + 1, :],
output_targets: y[i:i + 1, :]})
print("{0} -> {1}, perplexity: {2}".format(x[i:i + 1, :], y[i:i + 1, :], perplexity))
if i == 0 or i == word_count:
continue
ppl += perplexity
ppl /= (word_count - 2)
print("perplexity:" + str(ppl))
ppl_list.append(ppl)
return ppl_list
def infer_generate():
After Change
def ppl(sentence_list):
result = dict()
// load data dict
word_to_idx = load_word_dict(config.word_dict_path)
idx_to_word = {v: k for k, v in word_to_idx.items()}
// init params
batch_size = 1
tf.reset_default_graph()
input_data = tf.placeholder(tf.int32, [batch_size, None])
output_targets = tf.placeholder(tf.int32, [batch_size, None])
// init model
end_points = rnn_model(model="lstm",
input_data=input_data,
output_data=output_targets,
vocab_size=len(word_to_idx),
rnn_size=128,
num_layers=2,
batch_size=batch_size,
learning_rate=config.learning_rate)
saver = tf.train.Saver(tf.global_variables())
init_op = tf.group(tf.global_variables_initializer(), tf.local_variables_initializer())
with tf.Session() as sess:
// init op
sess.run(init_op)
checkpoint = tf.train.latest_checkpoint(config.model_dir)
saver.restore(sess, checkpoint)
print("loading model from the checkpoint {0}".format(checkpoint))
// infer each sentence
for sentence in sentence_list:
ppl = 0
// data idx
x = [word_to_idx[c] if c in word_to_idx else word_to_idx[UNK_TOKEN] for c in sentence]
x = [word_to_idx[START_TOKEN]] + x + [word_to_idx[END_TOKEN]]
// print("x:", x)
// reshape
y = np.array(x[1:]).reshape((-1, batch_size))
x = np.array(x[:-1]).reshape((-1, batch_size))
// get each word perplexity
word_count = x.shape[0]
for i in range(word_count):
perplexity = sess.run(end_points["perplexity"],
feed_dict={input_data: x[i:i + 1, :],
output_targets: y[i:i + 1, :]})
print("{0} -> {1}, perplexity: {2}".format(idx_to_word[x[i:i + 1, :].tolist()[0][0]],
idx_to_word[y[i:i + 1, :].tolist()[0][0]],
perplexity))
if i == 0 or i == word_count:
continue
ppl += perplexity
ppl /= (word_count - 2)
result[sentence] = ppl
return result
def infer_generate():
