Merge pull request #1336 from xwang233/add-local-rank

Make train.py compatible with torchrun
2 years ago · 2456223052
parent beef62e7ab 11060f84c5
commit 2456223052
1 changed files with 2 additions and 0 deletions
--- a/train.py
+++ b/train.py
@ -355,6 +355,8 @@ def main():
    args.world_size = 1
    args.rank = 0  # global rank
    if args.distributed:
        if 'LOCAL_RANK' in os.environ:
            args.local_rank = int(os.getenv('LOCAL_RANK'))
        args.device = 'cuda:%d' % args.local_rank
        torch.cuda.set_device(args.local_rank)
        torch.distributed.init_process_group(backend='nccl', init_method='env://')