16d2eb3061f1bed8ade390c5e2a2c1de9daa3509,theanolm/optimizers/adamoptimizer.py,AdamOptimizer,_get_model_updates,#AdamOptimizer#,93

Before Change

        alpha /= 1.0 - (self._gamma_m ** timestep_new)

        result = []
        for name, param in self.network.params.items():
            m_gradient = self.params[name + ".mean_gradient"]
            ms_gradient = self.params[name + ".mean_sqr_gradient"]
            rms_gradient = tensor.sqrt(ms_gradient) + self._epsilon
            param_new = param - (alpha * m_gradient / rms_gradient)
            result.append((param, param_new))
        result.append((timestep, timestep_new))
        return result

    def reset(self):

After Change

        alpha *= tensor.sqrt(1.0 - (self._gamma_ms ** timestep_new))
        alpha /= 1.0 - (self._gamma_m ** timestep_new)

        updates = dict()
        for name, param in self.network.params.items():
            m_gradient = self.params[name + ".mean_gradient"]
            ms_gradient = self.params[name + ".mean_sqr_gradient"]
            rms_gradient = tensor.sqrt(ms_gradient) + self._epsilon
            updates[name] = -m_gradient / rms_gradient
        self._normalize(updates)

        result = []
        for name, param in self.network.params.items():
            update = updates[name]
            result.append((param, param + alpha * update))
        result.append((timestep, timestep_new))
        return result

    def reset(self):

Instances