3128e13109c8766eafb413f2428bba976701e929,beginner_source/transformer_tutorial.py,,,#,295

Before Change

// we"ve seen so far. Adjust the learning rate after each epoch.

best_val_loss = float("inf")
epochs = 3 // The number of epochs
best_model = None

for epoch in range(1, epochs + 1):
    epoch_start_time = time.time()
    train()
    val_loss = evaluate(model, val_data)
    print("-" * 89)
    print("| end of epoch {:3d} | time: {:5.2f}s | valid loss {:5.2f} | "
          "valid ppl {:8.2f}".format(epoch, (time.time() - epoch_start_time),
                                     val_loss, math.exp(val_loss)))
    print("-" * 89)

    if val_loss < best_val_loss:
        best_val_loss = val_loss
        best_model = model

    scheduler.step()


////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// Evaluate the model with the test dataset
// -------------------------------------
//
// Apply the best model to check the result with the test dataset.

test_loss = evaluate(best_model, test_data)
print("=" * 89)
print("| End of training | test loss {:5.2f} | test ppl {:8.2f}".format(
    test_loss, math.exp(test_loss)))

After Change

criterion = nn.CrossEntropyLoss()
lr = 5.0 // learning rate
optimizer = torch.optim.SGD(model.parameters(), lr=lr)
scheduler = torch.optim.lr_scheduler.StepLR(optimizer, 1.0, gamma=0.95)

import time

Instances