Traceback (most recent call last):


 "/home/pi/oprint/lib/python3.7/site-packages/octoprint/logging/handlers.py", line 20, in emit

self._executor.submit(self._emit, record)


 "/usr/lib/python3.7/concurrent/futures/thread.py", line 151, in submit

raise RuntimeError('cannot schedule new futures after shutdown')
RuntimeError: cannot schedule new futures after shutdown

Traceback (most recent call last):


 "./tools/train.py", line 106, in <module>

main()


 "./tools/train.py", line 84, in main

cfg.model, train_cfg=cfg.train_cfg, test_cfg=cfg.test_cfg)


 "/home/wangtao/prj/_mmdetection/mmdetection/mmdet/models/builder.py", line 43, in build_detector

return build(cfg, DETECTORS, dict(train_cfg=train_cfg, test_cfg=test_cfg))


 "/home/wangtao/prj/_mmdetection/mmdetection/mmdet/models/builder.py", line 15, in build

return build_from_cfg(cfg, registry, default_args)


 "/home/wangtao/prj/_mmdetection/mmdetection/mmdet/utils/registry.py", line 74, in build_from_cfg

return obj_type(**args)


 "/home/wangtao/prj/_mmdetection/mmdetection/mmdet/models/detectors/mask_rcnn.py", line 31, in __init__

pretrained=pretrained)


 "/home/wangtao/prj/_mmdetection/mmdetection/mmdet/models/detectors/two_stage.py", line 57, in __init__

self.init_weights(pretrained=pretrained)


 "/home/wangtao/prj/_mmdetection/mmdetection/mmdet/models/detectors/two_stage.py", line 65, in init_weights

self.backbone.init_weights(pretrained=pretrained)


 "/home/wangtao/prj/_mmdetection/mmdetection/mmdet/models/backbones/resnet.py", line 484, in init_weights

load_checkpoint(self, pretrained, strict=False, logger=logger)


 "/home/wangtao/anaconda3/envs/open-mmlab/lib/python3.7/site-packages/mmcv/runner/checkpoint.py", line 137, in load_checkpoint

checkpoint = load_url_dist(model_urls[model_name])


 "/home/wangtao/anaconda3/envs/open-mmlab/lib/python3.7/site-packages/mmcv/runner/checkpoint.py", line 101, in load_url_dist

torch.distributed.barrier()


 "/home/wangtao/anaconda3/envs/open-mmlab/lib/python3.7/site-packages/torch/distributed/distributed_c10d.py", line 1358, in barrier

work = _default_pg.barrier()
RuntimeError: NCCL error in: /opt/conda/conda-bld/pytorch_1556653114079/work/torch/lib/c10d/ProcessGroupNCCL.cpp:272, unhandled cuda error
