6fe998295efee06541295a086d57dc2a10f230b7,opennmt/utils/transformer.py,,multi_head_attention,#Any#Any#Any#Any#Any#Any#Any#Any#,67

Before Change

  input_dim = keys.get_shape().as_list()[-1]
  head_dim = input_dim / num_heads

  with tf.variable_scope("multi_head"):
    for i in range(num_heads):
      with tf.variable_scope("head_" + str(i)):
        // Project queries, keys and values to different and smaller subspaces.
        queries_proj = tf.layers.dense(
          queries,
          head_dim,
          use_bias=False)
        keys_proj = tf.layers.dense(
          keys,
          head_dim,
          use_bias=False)
        values_proj = tf.layers.dense(
          values,
          head_dim,
          use_bias=False)

        head_i = scaled_dot_attention(
          queries_proj,
          keys_proj,
          values_proj,
          mode,
          values_length=values_length,
          mask_future=mask_future,
          dropout=dropout)

        heads.append(head_i)

    // Concatenate all heads output.
    return tf.concat(heads, axis=2)

def feed_forward(x, inner_dim):
  Implements the Transformer"s "Feed Forward" layer.

  ffn(x) = max(0, x*W_1 + b_1)*W_2 + b_2

After Change

  input_dim = keys.get_shape().as_list()[-1]
  head_dim = input_dim / num_heads

  for i in range(num_heads):
    with tf.variable_scope("head_" + str(i)):
      // Project queries, keys and values to different and smaller subspaces.
      queries_proj = tf.layers.dense(
        queries,
        head_dim,
        use_bias=False)
      keys_proj = tf.layers.dense(
        keys,
        head_dim,
        use_bias=False)
      values_proj = tf.layers.dense(
        values,
        head_dim,
        use_bias=False)

      head_i = scaled_dot_attention(
        queries_proj,
        keys_proj,
        values_proj,
        mode,
        values_length=values_length,
        mask_future=mask_future,
        dropout=dropout)

      heads.append(head_i)

  // Concatenate all heads output.
  return tf.concat(heads, axis=2)

def feed_forward(x, inner_dim):
  Implements the Transformer"s "Feed Forward" layer.

Instances