86ce5d52134a56806112ff8664e4034338e0e05a,yarll/agents/ppo/ppo.py,PPO,learn,#PPO#,208

Before Change

            states, actions, advs, rs, _ = self.get_processed_trajectories()
            advs = np.array(advs)
            advs = (advs - advs.mean()) / advs.std()
            self.session.run(self.set_old_to_new)

            indices = np.arange(len(states))
            for _ in range(int(self.config["n_epochs"])):
                np.random.shuffle(indices)

After Change

                        batch_actions = np.array(actions)[batch_indices]
                        batch_advs = np.array(advs)[batch_indices]
                        batch_rs = np.array(rs)[batch_indices]
                        train_actor_loss, train_critic_loss, train_loss = self.train(batch_states,
                                                                                     batch_actions,
                                                                                     batch_advs,
                                                                                     batch_rs)
                        tf.summary.scalar("model/loss", train_loss, step=n_updates)
                        tf.summary.scalar("model/actor_loss", train_actor_loss, step=n_updates)
                        tf.summary.scalar("model/critic_loss", train_critic_loss, step=n_updates)
                        n_updates += 1

Instances