16d2eb3061f1bed8ade390c5e2a2c1de9daa3509,theanolm/optimizers/basicoptimizer.py,BasicOptimizer,__init__,#BasicOptimizer#Any#Any#Any#,16

Before Change

        gradients = tensor.grad(cost, wrt=list(self.network.params.values()))

        // Normalize the norm of the gradients to given maximum value.
        if "max_gradient_norm" in optimization_options:
            max_norm = optimization_options["max_gradient_norm"]
            epsilon = optimization_options["epsilon"]
            squares = [tensor.sqr(gradient) for gradient in gradients]
            sums = [tensor.sum(square) for square in squares]
            total_sum = sum(sums)  // sum over parameter variables
            norm = tensor.sqrt(total_sum)
            target_norm = tensor.clip(norm, 0.0, max_norm)
            gradients = [gradient * target_norm / (epsilon + norm)
                         for gradient in gradients]

        self._gradient_exprs = gradients
        self.gradient_update_function = \
            theano.function([self.network.minibatch_input,
                             self.network.minibatch_mask],

After Change

                       for name, value in self.param_init_values.items()}

        // numerical stability / smoothing term to prevent divide-by-zero
        if not "epsilon" in optimization_options:
            raise ValueError("Epsilon is not given in optimization options.")
        self._epsilon = optimization_options["epsilon"]

        // maximum norm for parameter updates
        if "max_gradient_norm" in optimization_options:

Instances