b7efdc787a7b9cc78f47e16fbd91e36ab5d12225,variance_reduction/baseline_comparison_prototype.py,,,#,139

Before Change

            // TODO(cathywu) Kill the rest of this chunk of code
            // Get the gradient for this episode, and save it in the gradBuffer
            tBlah0 = sess.run(loglik_new,
                              feed_dict={observations: epx, input_y: epy,
                                         advantages: discounted_epr,
                                         score_old: old_score})
            tBlah1 = sess.run(loglik_old,
                              feed_dict={observations: epx, input_y: epy,
                                         advantages: discounted_epr,
                                         score_old: old_score})
            tBlah2 = sess.run(likratio,
                              feed_dict={observations: epx, input_y: epy,
                                         advantages: discounted_epr,
                                         score_old: old_score})
            // print("Log likelihood ratio:", tBlah0, tBlah1, tBlah2)

            // print(tLoglik_old[0], tLoglik_new[0])
            tGrad = sess.run(newGrads,

After Change

                // Get the gradient for each episode, and save it in the
                // gradBuffer
                // TODO(cathywu) compute gradients here
                for epx, epy, discounted_epr in zip(epxs, epys,
                                                    discounted_eprs):
                    tGrad = sess.run(newGrads,
                                     feed_dict={observations: epx, input_y: epy,
                                                advantages: discounted_epr,
                                                score_old: old_score})
                    for ix, grad in enumerate(tGrad):
                        gradBuffer[ix] += grad

                // TODO(cathywu) compute gradients with V baseline

                // TODO(cathywu) compute gradients with action baseline

                sess.run(updateGrads, feed_dict={W1Grad: gradBuffer[0],
                                                 W2Grad: gradBuffer[1],
                                                 W3Grad: gradBuffer[2]})
                old_score = output.eval(feed_dict={observations: x})

Instances