// Define cost function based on which one was selected via set_loss_function
if self._loss_fn == "softmax cross entropy":
sf_logits = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=xx, labels=tf.argmax(y, 1))
self._graph_ops["cost"] = tf.add(tf.reduce_mean(tf.concat([sf_logits], axis=0)), l2_cost)
// For classification problems, we will compute the training accuracy as well; this is also used
// for Tensorboard
self.__class_predictions = tf.argmax(tf.nn.softmax(xx), 1)
After Change
self._add_layers_to_graph()
// Do the forward pass and training output calcs on possibly multiple GPUs
device_costs = []
device_accuracies = []
device_gradients = []
device_variables = []
for n, d in enumerate(self._get_device_list()): // Build a graph on either a CPU or all of the GPUs
with tf.device(d), tf.name_scope("tower_" + str(n)):
// Run the network operations
if self._has_moderation:
xx = self.forward_pass(x, deterministic=False, moderation_features=mod_w)
else:
xx = self.forward_pass(x, deterministic=False)
// Define regularization cost
self._log("Graph: Calculating loss and gradients...")
if self._reg_coeff is not None:
l2_cost = tf.squeeze(tf.reduce_sum(
[layer.regularization_coefficient * tf.nn.l2_loss(layer.weights) for layer in self._layers
if isinstance(layer, layers.fullyConnectedLayer)]))
else:
l2_cost = 0.0
// Define cost function based on which one was selected via set_loss_function
if self._loss_fn == "softmax cross entropy":
sf_logits = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=xx, labels=tf.argmax(y, 1))
gpu_cost = tf.reduce_mean(tf.concat([sf_logits], axis=0)) + l2_cost
cost_sum = tf.reduce_sum(tf.concat([sf_logits], axis=0))
device_costs.append(cost_sum)
// self._graph_ops["cost"] = tf.add(tf.reduce_mean(tf.concat([sf_logits], axis=0)), l2_cost)
// For classification problems, we will compute the training accuracy as well; this is also used
// for Tensorboard
self.__class_predictions = tf.argmax(tf.nn.softmax(xx), 1)
correct_predictions = tf.equal(self.__class_predictions, tf.argmax(y, 1))
accuracy_sum = tf.reduce_sum(tf.cast(correct_predictions, tf.float32))
device_accuracies.append(accuracy_sum)
// self._graph_ops["accuracy"] = tf.reduce_mean(tf.cast(correct_predictions, tf.float32))
// Set the optimizer and get the gradients from it
gradients, variables, global_grad_norm = self._graph_get_gradients(gpu_cost, optimizer)
device_gradients.append(gradients)
device_variables.append(variables)
// Average the gradients from each GPU and apply them
average_gradients = self._graph_average_gradients(device_gradients)
opt_variables = device_variables[0]
self._graph_ops["optimizer"] = self._graph_apply_gradients(average_gradients, opt_variables, optimizer)
// Average the costs and accuracies from each GPU
self._graph_ops["cost"] = tf.reduce_sum(device_costs) / self._batch_size + l2_cost
self._graph_ops["accuracy"] = tf.reduce_sum(device_accuracies) / self._batch_size
// Calculate test and validation accuracy (on a single device)
