ee37cc5236f3a2435c02dfe13fa2d9a743aeef17,theanolm/network/softmaxlayer.py,SoftmaxLayer,create_structure,#SoftmaxLayer#,31

Before Change

        output_size = preact.shape[2]
        preact = preact.reshape([num_time_steps * num_sequences,
                                 output_size])
        self.output_probs = tensor.nnet.softmax(preact)
        self.output_probs = self.output_probs.reshape([num_time_steps,
                                                       num_sequences,
                                                       output_size])
        if self.network.mode.is_distribution():

After Change

            // Generate one word for each training word as a negative sample.
            sample_class_ids = self.random.uniform(self.target_probs.shape)
            sample_class_ids *= num_classes
            sample_class_ids = tensor.cast(sample_class_ids, "int64")
            self.negative_probs = self._get_sigmoid(layer_input,
                                                    sample_class_ids)
            // minibatch_size = self.negative_probs.shape[0] * self.negative_probs.shape[1]
            // minibatch_size = tensor.cast(minibatch_size, theano.config.floatX)
            // cost = -tensor.log(self.target_probs)
            // cost -= minibatch_size * tensor.log(1.0 - self.negative_probs)

Instances