3c5330d8130ec7e03e5df28b199ab0357b559301,official/bert/bert_models.py,BertPretrainLayer,call,#BertPretrainLayer#Any#,111

Before Change

    lm_output = tf.keras.backend.log(lm_output)

    sentence_output = self.next_sentence_dense(pooled_output)
    sentence_output = tf.keras.backend.softmax(sentence_output)
    sentence_output = tf.keras.backend.log(sentence_output)
    return (lm_output, sentence_output)

After Change

    lm_output = tf.nn.log_softmax(lm_output, axis=-1)

    logits = tf.matmul(pooled_output, self.next_seq_weights, transpose_b=True)
    logits = tf.nn.bias_add(logits, self.next_seq_bias)
    sentence_output = tf.nn.log_softmax(logits, axis=-1)
    return (lm_output, sentence_output)


class BertPretrainLossAndMetricLayer(tf.keras.layers.Layer):

Instances