79ec8f5009d71a891176f23af20fc077f058713a,algo/ppo.py,PPO,update,#PPO#Any#,32

Before Change

                                           1.0 + self.clip_param) * adv_targ
                action_loss = -torch.min(surr1, surr2).mean()

                value_loss = F.mse_loss(return_batch, values)

                self.optimizer.zero_grad()
                (value_loss * self.value_loss_coef + action_loss -
                 dist_entropy * self.entropy_coef).backward()

After Change

                value_pred_clipped = value_preds_batch + \
                     (values - value_preds_batch).clamp(-self.clip_param, self.clip_param)
                value_losses = (values - return_batch).pow(2)
                value_losses_clipped = (value_pred_clipped - return_batch).pow(2)
                value_loss = .5 * torch.max(value_losses, value_losses_clipped).mean()

                self.optimizer.zero_grad()

Instances