Traceback (most recent call last):


 "evaluation_on_widerface.py", line 24, in <module>

num_output_scales=cfg.param_num_output_scales)


 "/media/danush/b17c9432-d479-4633-aa80-3c70cb68a206/danush/Documents/lffd_conda/face_detection/accuracy_evaluation/predict.py", line 102, in __init__

self.__load_model()


 "/media/danush/b17c9432-d479-4633-aa80-3c70cb68a206/danush/Documents/lffd_conda/face_detection/accuracy_evaluation/predict.py", line 122, in __load_model

for_training=False)


 "/home/danush/anaconda3/envs/lffd/lib/python3.6/site-packages/mxnet/module/module.py", line 429, in bind

state_names=self._state_names)


 "/home/danush/anaconda3/envs/lffd/lib/python3.6/site-packages/mxnet/module/executor_group.py", line 279, in __init__

self.bind_exec(data_shapes, label_shapes, shared_group)


 "/home/danush/anaconda3/envs/lffd/lib/python3.6/site-packages/mxnet/module/executor_group.py", line 375, in bind_exec

shared_group))


 "/home/danush/anaconda3/envs/lffd/lib/python3.6/site-packages/mxnet/module/executor_group.py", line 662, in _bind_ith_exec

shared_buffer=shared_data_arrays, **input_shapes)


 "/home/danush/anaconda3/envs/lffd/lib/python3.6/site-packages/mxnet/symbol/symbol.py", line 1629, in simple_bind

raise RuntimeError(error_msg)
RuntimeError: simple_bind error. Arguments:

Traceback (most recent call last):


 "/home/ml/users/jgonza38/anaconda3/lib/python3.7/site-packages/torch/multiprocessing/spawn.py", line 20, in _wrap

fn(i, *args)


 "/home/ml/users/jgonza38/reproduce_master_thesis/fairseq/fairseq_cli/train.py", line 324, in distributed_main

main(args, init_distributed=True)


 "/home/ml/users/jgonza38/reproduce_master_thesis/fairseq/fairseq_cli/train.py", line 117, in main

valid_losses = train(args, trainer, task, epoch_itr, max_update)


 "/home/ml/users/jgonza38/anaconda3/lib/python3.7/contextlib.py", line 74, in inner

return func(*args, **kwds)


 "/home/ml/users/jgonza38/reproduce_master_thesis/fairseq/fairseq_cli/train.py", line 187, in train

log_output = trainer.train_step(samples)


 "/home/ml/users/jgonza38/anaconda3/lib/python3.7/contextlib.py", line 74, in inner

return func(*args, **kwds)


 "/home/ml/users/jgonza38/reproduce_master_thesis/fairseq/fairseq/trainer.py", line 399, in train_step

raise e


 "/home/ml/users/jgonza38/reproduce_master_thesis/fairseq/fairseq/trainer.py", line 377, in train_step

ignore_grad=is_dummy_batch,


 "/home/ml/users/jgonza38/reproduce_master_thesis/fairseq/fairseq/tasks/fairseq_task.py", line 342, in train_step

optimizer.backward(loss)


 "/home/ml/users/jgonza38/reproduce_master_thesis/fairseq/fairseq/optim/fairseq_optimizer.py", line 81, in backward

loss.backward()


 "/home/ml/users/jgonza38/anaconda3/lib/python3.7/site-packages/torch/tensor.py", line 198, in backward

torch.autograd.backward(self, gradient, retain_graph, create_graph)


 "/home/ml/users/jgonza38/anaconda3/lib/python3.7/site-packages/torch/autograd/__init__.py", line 100, in backward

allow_unreachable=True)
# allow_unreachable flag
RuntimeError: NCCL error in: /opt/conda/conda-bld/pytorch_1591914855613/work/torch/lib/c10d/ProcessGroupNCCL.cpp:32, unhandled cuda error, NCCL version 2.4.8
