cd1da3fc9c5b68a9e583b8c26e5cdae58560be6d,torch_ac/algos/ppo.py,PPOAlgo,update_parameters,#PPOAlgo#,21

Before Change

                surr2 = torch.clamp(ratio, 1.0 - self.clip_eps, 1.0 + self.clip_eps) * advantage_var
                action_loss = -torch.min(surr1, surr2).mean()

                value_loss = (value - Variable(b.returnn)).pow(2).mean()

                loss = action_loss - self.entropy_coef * entropy + self.value_loss_coef * value_loss

                // Update actor-critic

After Change

                value_clipped = old_value_var + torch.clamp(value - old_value_var, -self.clip_eps, self.clip_eps)
                return_var = Variable(b.returnn)
                surr1 = (value - return_var).pow(2)
                surr2 = (value_clipped - return_var).pow(2)
                value_loss = torch.max(surr1, surr2).mean()

                loss = action_loss - self.entropy_coef * entropy + self.value_loss_coef * value_loss

                // Update actor-critic

Instances