adf9ebd629abc21e0969db2a1c29f389b5301c9d,layers/common_layers.py,GravesAttention,forward,#GravesAttention#Any#Any#Any#,129

Before Change

        c_t = torch.bmm(alpha_t.unsqueeze(1), inputs).squeeze(1)
        self.mu_tm1 = mu_t
        return c_t, mu_t, alpha_t


class Attention(nn.Module):

After Change

        // g_t = torch.softmax(g_t, dim=-1) + self.epsilon  // distribution weight
        // sig_t = torch.exp(b_t) + self.epsilon  // variance
        // mu_t = self.mu_prev + self.attention_alignment * torch.exp(k_t)  // mean
        sig_t = torch.pow(torch.nn.functional.softplus(b_t), 2)
        mu_t = self.mu_prev + torch.nn.functional.softplus(k_t)
        g_t = (torch.softmax(g_t, dim=-1) / sig_t) * self.COEF

        g_t = g_t.unsqueeze(2).expand(g_t.size(0),

Instances