b7bdc5b940af98a8014e9e2fc63d5e24cc3b013b,layers/eight_mile/tf/layers.py,TransformerEncoder,__init__,#TransformerEncoder#Any#Any#Any#Any#Any#Any#Any#Any#Any#Any#Any#Any#Any#,2510

Before Change

        else:
            self.self_attn = MultiHeadedAttention(num_heads, d_model, pdrop, scale=scale, d_k=d_k)

        self.ffn = FFN(d_model, activation_type, d_ff, ffn_pdrop, name="ffn")
        self.ln1 = tf.keras.layers.LayerNormalization(epsilon=layer_norm_eps)
        self.ln2 = tf.keras.layers.LayerNormalization(epsilon=layer_norm_eps)
        self.dropout = tf.keras.layers.Dropout(pdrop)

After Change

def tie_weight(weight, tie_shape):
    Higher order function to share weights between two layers.

    Tensorflow will take a custom_getter inside of a variable scope.
    This method creates a getter that looks for a match in shapes. If they match,
    The weights are transposed and shared.

    
    def tie_getter(getter, name, *args, **kwargs):
        if kwargs["shape"] == tie_shape:
            return tf.transpose(weight)
        return getter("{}".format(name), *args, **kwargs)
    return tie_getter


def rnn_cell_w_dropout(hsz, pdrop, rnntype, st=None, variational=False, training=False):

    Produce a single RNN cell with dropout
    :param hsz: (``int``) The number of hidden units per LSTM

Instances