78a7f7de24c34235d0784a5781f46de34d2336eb,python/eight_mile/tf/layers.py,TransformerEncoder,__init__,#TransformerEncoder#Any#Any#Any#Any#Any#Any#Any#,1060

Before Change

        super(TransformerEncoder, self).__init__(name=name)
        if d_ff is None:
            d_ff = 4 * d_model
        self.ln1 = LayerNorm(name="ln_1")
        self.self_attn = MultiHeadedAttention(num_heads, d_model, pdrop, scale)
        self.dropout = tf.keras.layers.Dropout(pdrop)
        self.ln2 = LayerNorm(name="ln_2")
        self.feed_forward = FFN(d_model, pdrop, activation, d_ff, name="ffn")

After Change

class TransformerEncoder(tf.keras.layers.Layer):
    def __init__(self, num_heads, d_model, pdrop, scale=True, activation_type="relu", d_ff=None, name=None):
        super().__init__(name=name)
        self.d_model = d_model
        self.d_ff = d_ff if d_ff is not None else 4 * d_model
        self.self_attn = MultiHeadedAttention(num_heads, d_model, pdrop, scale=scale)
        self.ffn = FFN(d_model, pdrop, activation_type, d_ff, name="ffn")

Instances