34726d5612322b79b621e52f7f6fe47a6716eb65,fairseq/distributed_utils.py,,distributed_init,#Any#,53

Before Change

    print("| distributed init (rank {}): {}".format(
        args.distributed_rank, args.distributed_init_method), flush=True)

    dist.init_process_group(
        backend=args.distributed_backend,
        init_method=args.distributed_init_method,
        world_size=args.distributed_world_size,
        rank=args.distributed_rank,
    )

    suppress_output(is_master(args))

    return args.distributed_rank

After Change

    if args.distributed_world_size == 1:
        raise ValueError("Cannot initialize distributed with distributed_world_size=1")

    if torch.distributed.is_initialized():
        warnings.warn("Distributed is already initialized, cannot initialize twice!")
    else:
        print("| distributed init (rank {}): {}".format(
            args.distributed_rank, args.distributed_init_method), flush=True)

        dist.init_process_group(
            backend=args.distributed_backend,
            init_method=args.distributed_init_method,
            world_size=args.distributed_world_size,
            rank=args.distributed_rank,
        )

        suppress_output(is_master(args))

    args.distributed_rank = torch.distributed.get_rank()
    return args.distributed_rank

Instances