ccbaf353bcd6121c50b4de0965177480b8f6eb48,copy_problem_100/rwa_model/train.py,,,#,25

Before Change

// Internal states
//
h = tf.zeros([batch_size, num_cells])
n = tf.zeros([batch_size, num_cells])
d = tf.zeros([batch_size, num_cells])

// Define model
//
error = tf.zeros([batch_size])
h += activation(tf.expand_dims(s, 0))

for i in range(max_steps):

	x_step = x[:,i,:]
	xh_join = tf.concat(1, [x_step, h])	// Combine the features and hidden state into one tensor

	g = tf.matmul(xh_join, W_g)+b_g
	u = tf.matmul(x_step, W_u)+b_u
	q = tf.matmul(xh_join, W_a)+b_a

	q_greater = tf.maximum(q, 0.0)	// Greater of the exponent term or zero
	scale = tf.exp(-q_greater)
	a_scale = tf.exp(q-q_greater)

	n = tf.mul(n, scale)+tf.mul(tf.mul(u, tf.nn.tanh(g)), a_scale)	// Numerically stable update of numerator
	d = tf.mul(d, scale)+a_scale	// Numerically stable update of denominator
	h = activation(tf.div(n, d))

After Change

//
n = tf.zeros([batch_size, num_cells])
d = tf.zeros([batch_size, num_cells])
h = tf.zeros([batch_size, num_cells])
a_max = tf.fill([batch_size, num_cells], -1E38)	// Start off with lowest number possible

// Define model
//
error = tf.zeros([batch_size])
h += activation(tf.expand_dims(s, 0))

for i in range(max_steps):

	x_step = x[:,i,:]
	xh_join = tf.concat(1, [x_step, h])	// Combine the features and hidden state into one tensor

	u = tf.matmul(x_step, W_u)+b_u
	g = tf.matmul(xh_join, W_g)+b_g
	a = tf.matmul(xh_join, W_a)+b_a

	z = tf.mul(u, tf.nn.tanh(g))

	a_newmax = tf.maximum(a_max, a)
	exp_diff = tf.exp(a_max-a_newmax)
	exp_scaled = tf.exp(a-a_newmax)

	n = tf.mul(n, exp_diff)+tf.mul(z, exp_scaled)	// Numerically stable update of numerator
	d = tf.mul(d, exp_diff)+exp_scaled	// Numerically stable update of denominator
	h = activation(tf.div(n, d))
	a_max = a_newmax

	ly = tf.matmul(h, W_o)+b_o

	error_step = tf.nn.softmax_cross_entropy_with_logits(ly, y[:,i,:])	// Cross-entropy cost function

Instances