15355d6b1d45c157badd1c21157b529e817b7c3c,tf_agents/agents/reinforce/reinforce_agent.py,ReinforceAgent,_train,#ReinforceAgent#Any#Any#,121

Before Change

                           tf.stop_gradient(returns),
                           weights=weights)

    clip_gradients = None
    if self._gradient_clipping:
      clip_gradients = eager_utils.clip_gradient_norms_fn(
          self._gradient_clipping)

    loss_info = eager_utils.create_train_step(
        loss_info,
        self._optimizer,
        total_loss_fn=lambda loss_info: loss_info.loss,
        global_step=self.train_step_counter,
        transform_grads_fn=clip_gradients,
        summarize_gradients=self._summarize_grads_and_vars,
        variables_to_train=lambda: self._actor_network.trainable_weights,
    )

    if self._summarize_grads_and_vars:
      with tf.name_scope("Variables/"):
        for var in self._actor_network.trainable_weights:
          tf.compat.v2.summary.histogram(
              name=var.name.replace(":", "_"),
              data=var,
              step=self.train_step_counter)

    return loss_info

  @eager_utils.future_in_eager_mode
  def _loss(self, time_steps, actions, returns, weights):
    tf.nest.assert_same_structure(time_steps, self.time_step_spec)

After Change

    self._optimizer.apply_gradients(
        grads_and_vars, global_step=self.train_step_counter)

    return tf.nest.map_structure(tf.identity, loss_info)

  def _loss(self, time_steps, actions, returns, weights):
    tf.nest.assert_same_structure(time_steps, self.time_step_spec)
    actions_distribution = self.collect_policy.distribution(time_steps).action

Instances