Traceback (most recent call last):


 "/usr/local/lib/python3.6/dist-packages/torch/multiprocessing/spawn.py", line 20, in _wrap

fn(i, *args)


 "/kpoints/HigherHRNet-Human-Pose-Estimation/tools/dist_train.py", line 285, in main_worker

final_output_dir, tb_log_dir, writer_dict, fp16=cfg.FP16.ENABLED)


 "/kpoints/HigherHRNet-Human-Pose-Estimation/tools/../lib/core/trainer.py", line 76, in do_train

loss.backward()


 "/usr/local/lib/python3.6/dist-packages/torch/tensor.py", line 198, in backward

torch.autograd.backward(self, gradient, retain_graph, create_graph)


 "/usr/local/lib/python3.6/dist-packages/torch/autograd/__init__.py", line 100, in backward

allow_unreachable=True)
# allow_unreachable flag
RuntimeError: CUDA out of memory. Tried to allocate 52.00 MiB (GPU 0; 7.80 GiB total capacity; 5.73 GiB already allocated; 27.31 MiB free; 5.86 GiB reserved in total by PyTorch) (malloc at /pytorch/c10/cuda/CUDACachingAllocator.cpp:289)

Traceback (most recent call last):


 "/home/bjfu/anaconda3/envs/hsj-torch-gpu16/lib/python3.6/multiprocessing/process.py", line 258, in _bootstrap

self.run()


 "/home/bjfu/anaconda3/envs/hsj-torch-gpu16/lib/python3.6/multiprocessing/process.py", line 93, in run

self._target(*self._args, **self._kwargs)


 "train.py", line 279, in init_processes

torch.cuda.set_device(args.local_rank)


 "/home/bjfu/anaconda3/envs/hsj-torch-gpu16/lib/python3.6/site-packages/torch/cuda/__init__.py", line 281, in set_device

torch._C._cuda_setDevice(device)
RuntimeError: cuda runtime error (101) : invalid device ordinal at /pytorch/torch/csrc/cuda/Module.cpp:59
