78a7f7de24c34235d0784a5781f46de34d2336eb,python/eight_mile/tf/layers.py,TransformerEncoder,__init__,#TransformerEncoder#Any#Any#Any#Any#Any#Any#Any#,1060

Before Change

class TransformerEncoder(tf.keras.layers.Layer):

    def __init__(self, d_model, num_heads, pdrop, scale=True, activation="relu", d_ff=None, name=None):
        super(TransformerEncoder, self).__init__(name=name)
        if d_ff is None:
            d_ff = 4 * d_model
        self.ln1 = LayerNorm(name="ln_1")
        self.self_attn = MultiHeadedAttention(num_heads, d_model, pdrop, scale)

After Change

        self.self_attn = MultiHeadedAttention(num_heads, d_model, pdrop, scale=scale)
        self.ffn = FFN(d_model, pdrop, activation_type, d_ff, name="ffn")
        self.ln1 = tf.keras.layers.LayerNormalization(epsilon=1e-6)
        self.ln2 = tf.keras.layers.LayerNormalization(epsilon=1e-6)
        self.dropout = tf.keras.layers.Dropout(pdrop)

    def forward(self, inputs):
        

Instances