bafeed46fb76fa337a771ebb41d65bb95039565a,fairseq/trainer.py,Trainer,train_step,#Trainer#Any#Any#Any#,261

Before Change

                    // printed out if another exception happens.
                    // NB(jerry): added a flush to mitigate this
                    print(msg, file=sys.stderr)
                    if torch.cuda.is_available() and hasattr(torch.cuda, "memory_summary"):
                        for device_idx in range(torch.cuda.device_count()):
                            print(torch.cuda.memory_summary(device=device_idx),
                                  file=sys.stderr)
                    sys.stderr.flush()

                    if raise_oom:
                        raise ValueError(msg)

After Change

            if "out of memory" in str(e):
                self._log_oom(e)
                print("| ERROR: OOM during optimization, irrecoverable")
            raise e

        if self.args.fp16:
            self.meters["loss_scale"].reset()
            self.meters["loss_scale"].update(self.optimizer.scaler.loss_scale)

Instances