54f30e39c7aee65218cb40e219fe9f4f816f0eee,slm_lab/agent/algorithm/sac.py,SoftActorCritic,calc_policy_loss,#SoftActorCritic#Any#Any#,141

Before Change

            reparam_mus = action_pd.rsample()  // reparametrization for paper eq. 11
            reparam_actions = torch.tanh(reparam_mus)
            // paper Appendix C. Enforcing Action Bounds for continuous actions
            log_probs = action_pd.log_prob(reparam_mus) - torch.log(1 - reparam_actions.pow(2) + 1e-6).sum(1)

        q1_preds = self.calc_q(states, reparam_actions, self.q1_net)
        q2_preds = self.calc_q(states, reparam_actions, self.q2_net)
        q_preds = torch.min(q1_preds, q2_preds)

After Change

    def calc_policy_loss(self, batch, action_pd):
        """policy_loss = alpha * log pi(f(a)|s) - Q1(s, f(a)), where f(a) = reparametrized action"""
        log_probs, reparam_actions = self.sample_log_probs(action_pd, reparam=True)
        states = batch["states"]
        q1_preds = self.calc_q(states, reparam_actions, self.q1_net)
        q2_preds = self.calc_q(states, reparam_actions, self.q2_net)

Instances