83f0a576015953b30bf8a05fc0033983e0d07354,official/nlp/bert_models.py,,pretrain_model,#Any#Any#Any#Any#,260

Before Change

  pooled_output = bert_submodel.outputs[0]
  sequence_output = bert_submodel.outputs[1]

  pretrain_layer = BertPretrainLayer(
      bert_config,
      bert_submodel.get_layer(bert_submodel_name),
      initializer=initializer,
      name="cls")
  lm_output, sentence_output = pretrain_layer(pooled_output, sequence_output,
                                              masked_lm_positions)

  pretrain_loss_layer = BertPretrainLossAndMetricLayer(bert_config)

After Change

      shape=(1,), name="next_sentence_labels", dtype=tf.int32)

  transformer_encoder = _get_transformer_encoder(bert_config, seq_length)
  if initializer is None:
    initializer = tf.keras.initializers.TruncatedNormal(
        stddev=bert_config.initializer_range)
  pretrainer_model = bert_pretrainer.BertPretrainer(
      network=transformer_encoder,
      num_classes=2,  // The next sentence prediction label has two classes.
      num_token_predictions=max_predictions_per_seq,

Instances