78a7f7de24c34235d0784a5781f46de34d2336eb,python/eight_mile/tf/layers.py,TransformerEncoder,__init__,#TransformerEncoder#Any#Any#Any#Any#Any#Any#Any#,1060

Before Change

        super(TransformerEncoder, self).__init__(name=name)
        if d_ff is None:
            d_ff = 4 * d_model
        self.ln1 = LayerNorm(name="ln_1")
        self.self_attn = MultiHeadedAttention(num_heads, d_model, pdrop, scale)
        self.dropout = tf.keras.layers.Dropout(pdrop)
        self.ln2 = LayerNorm(name="ln_2")
        self.feed_forward = FFN(d_model, pdrop, activation, d_ff, name="ffn")

After Change

        self.d_ff = d_ff if d_ff is not None else 4 * d_model
        self.self_attn = MultiHeadedAttention(num_heads, d_model, pdrop, scale=scale)
        self.ffn = FFN(d_model, pdrop, activation_type, d_ff, name="ffn")
        self.ln1 = tf.keras.layers.LayerNormalization(epsilon=1e-6)
        self.ln2 = tf.keras.layers.LayerNormalization(epsilon=1e-6)
        self.dropout = tf.keras.layers.Dropout(pdrop)

    def forward(self, inputs):

Instances