78a7f7de24c34235d0784a5781f46de34d2336eb,python/eight_mile/tf/layers.py,MultiHeadedAttention,__init__,#MultiHeadedAttention#Any#Any#Any#Any#,1026

Before Change

        :param dropout (``float``): The amount of dropout to use
        :param attn_fn: A function to apply attention, defaults to SDP
        
        super(MultiHeadedAttention, self).__init__()
        assert d_model % h == 0
        self.d_k = d_model // h
        self.h = h
        self.w_Q = TimeDistributedProjection(d_model)

After Change

        self.d_k = d_model // num_heads
        self.h = num_heads
        self.w_Q = tf.keras.layers.Dense(units=d_model)
        self.w_K = tf.keras.layers.Dense(units=d_model)
        self.w_V = tf.keras.layers.Dense(units=d_model)
        self.w_O = tf.keras.layers.Dense(units=d_model)
        self.attn_fn = self._scaled_dot_product_attention if scale else self._dot_product_attention
        self.attn = None

Instances