1a693073cd01fffa7cb3f018b7221459703119f9,ch09/01_cartpole_dqn.py,,,#,48

Before Change

    batch_states, batch_actions, batch_targets = [], [], []
    total_rewards = []

    for step_idx, exp in enumerate(exp_source):
        selector.epsilon = max(EPSILON_STOP, EPSILON_START - step_idx / EPSILON_STEPS)
        batch_states.append(exp.state)
        batch_actions.append(exp.action)
        batch_targets.append(calc_target(tgt_net.target_model, exp.reward, exp.last_state))
        if len(batch_states) == BATCH_SIZE:
            optimizer.zero_grad()
            states_v = Variable(torch.from_numpy(np.array(batch_states, dtype=np.float32)))
            net_q_v = net(states_v)
            target_q = net_q_v.data.numpy().copy()
            target_q[range(BATCH_SIZE), batch_actions] = batch_targets
            target_q_v = Variable(torch.from_numpy(target_q))
            loss_v = mse_loss(net_q_v, target_q_v)
            loss_v.backward()
            optimizer.step()

            // clear batch
            batch_states.clear()
            batch_actions.clear()
            batch_targets.clear()

        new_rewards = exp_source.pop_total_rewards()
        if new_rewards:
            reward = new_rewards[0]
            total_rewards.append(reward)
            mean_rewards = float(np.mean(total_rewards[-100:]))
            print("%d: reward: %6.2f,\tmean_100: %6.2f,\tepsilon: %.2f" % (step_idx, reward, mean_rewards, selector.epsilon))
            writer.add_scalar("reward", reward, step_idx)
            writer.add_scalar("reward_100", mean_rewards, step_idx)
            writer.add_scalar("epsilon", selector.epsilon, step_idx)
            if mean_rewards > 195:
                print("Solved in %d steps!" % step_idx)
                break

        if step_idx % TARGET_STEPS == 0:
            tgt_net.sync()
    writer.close()
    pass

After Change

            continue

        // sample batch
        batch = replay_buffer.sample(BATCH_SIZE)
        batch_states = [exp.state for exp in batch]
        batch_actions = [exp.action for exp in batch]
        batch_targets = [calc_target(net, exp.reward, exp.last_state)
                         for exp in batch]
        // train
        optimizer.zero_grad()

Instances