f2ef1ca36afd4eb74bd1ad36a7a84411f69e5435,layers/attention.py,AttentionRNNCell,forward,#AttentionRNNCell#Any#Any#Any#Any#Any#Any#,108

Before Change

            mask = mask.view(memory.size(0), -1)
            alignment.masked_fill_(1 - mask, -float("inf"))
        // Normalize context weight
        alignment = F.softmax(alignment, dim=-1)
        // Attention context vector
        // (batch, 1, dim)
        // c_i = \sum_{j=1}^{T_x} \alpha_{ij} h_j
        context = torch.bmm(alignment.unsqueeze(1), annots)

After Change

            alignment.masked_fill_(1 - mask, -float("inf"))
        // Normalize context weight
        // alignment = F.softmax(alignment, dim=-1)
        alignment  = torch.sigmoid(alignment) / torch.sigmoid(alignment).sum(dim=1).unsqueeze(1)
        // Attention context vector
        // (batch, 1, dim)
        // c_i = \sum_{j=1}^{T_x} \alpha_{ij} h_j
        context = torch.bmm(alignment.unsqueeze(1), annots)

Instances