Traceback (most recent call last):


 "tools/train.py", line 187, in <module>

main()


 "tools/train.py", line 176, in main

train_detector(


 "/home/ai-root/shenl/mmdetection/mmdet/apis/train.py", line 170, in train_detector

runner.run(data_loaders, cfg.workflow)


 "/home/ai-root/anaconda3/envs/mmlab/lib/python3.8/site-packages/mmcv/runner/epoch_based_runner.py", line 125, in run

epoch_runner(data_loaders[i], **kwargs)


 "/home/ai-root/anaconda3/envs/mmlab/lib/python3.8/site-packages/mmcv/runner/epoch_based_runner.py", line 50, in train

self.run_iter(data_batch, train_mode=True)


 "/home/ai-root/anaconda3/envs/mmlab/lib/python3.8/site-packages/mmcv/runner/epoch_based_runner.py", line 29, in run_iter

outputs = self.model.train_step(data_batch, self.optimizer,


 "/home/ai-root/anaconda3/envs/mmlab/lib/python3.8/site-packages/mmcv/parallel/data_parallel.py", line 67, in train_step

return self.module.train_step(*inputs[0], **kwargs[0])


 "/home/ai-root/shenl/mmdetection/mmdet/models/detectors/base.py", line 247, in train_step

losses = self(**data)


 "/home/ai-root/anaconda3/envs/mmlab/lib/python3.8/site-packages/torch/nn/modules/module.py", line 889, in _call_impl

result = self.forward(*input, **kwargs)


 "/home/ai-root/anaconda3/envs/mmlab/lib/python3.8/site-packages/mmcv/runner/fp16_utils.py", line 84, in new_func

return old_func(*args, **kwargs)


 "/home/ai-root/shenl/mmdetection/mmdet/models/detectors/base.py", line 181, in forward

return self.forward_train(img, img_metas, **kwargs)


 "/home/ai-root/shenl/mmdetection/mmdet/models/detectors/two_stage.py", line 150, in forward_train

rpn_losses, proposal_list = self.rpn_head.forward_train(


 "/home/ai-root/shenl/mmdetection/mmdet/models/dense_heads/base_dense_head.py", line 54, in forward_train

losses = self.loss(*loss_inputs, gt_bboxes_ignore=gt_bboxes_ignore)


 "/home/ai-root/shenl/mmdetection/mmdet/models/dense_heads/rpn_head.py", line 72, in loss

losses = super(RPNHead, self).loss(


 "/home/ai-root/anaconda3/envs/mmlab/lib/python3.8/site-packages/mmcv/runner/fp16_utils.py", line 164, in new_func

return old_func(*args, **kwargs)


 "/home/ai-root/shenl/mmdetection/mmdet/models/dense_heads/anchor_head.py", line 459, in loss

cls_reg_targets = self.get_targets(


 "/home/ai-root/shenl/mmdetection/mmdet/models/dense_heads/anchor_head.py", line 339, in get_targets

results = multi_apply(


 "/home/ai-root/shenl/mmdetection/mmdet/core/utils/misc.py", line 29, in multi_apply

return tuple(map(list, zip(*map_results)))


 "/home/ai-root/shenl/mmdetection/mmdet/models/dense_heads/anchor_head.py", line 224, in _get_targets_single

sampling_result = self.sampler.sample(assign_result, anchors,


 "/home/ai-root/shenl/mmdetection/mmdet/core/bbox/samplers/base_sampler.py", line 95, in sample

neg_inds = self.neg_sampler._sample_neg(


 "/home/ai-root/shenl/mmdetection/mmdet/core/bbox/samplers/random_sampler.py", line 78, in _sample_neg

return self.random_choice(neg_inds, num_expected)


 "/home/ai-root/shenl/mmdetection/mmdet/core/bbox/samplers/random_sampler.py", line 54, in random_choice

perm = torch.randperm(gallery.numel(), device=gallery.device)[:num]
RuntimeError: radix_sort: failed on 1st step: cudaErrorInvalidDevice: invalid device ordinal

Traceback (most recent call last):

 "/var/lib/jenkins/cobra/cobra/__init__.py", line 82, in main
cli.start(args.target, args.format, args.output, args.special_rules, a_sid)

 "/var/lib/jenkins/cobra/cobra/cli.py", line 122, in start
write_to_file(target=target, sid=s_sid, output_format=formatter, filename=output)

 "/var/lib/jenkins/cobra/cobra/export.py", line 140, in write_to_file
os.chdir(export_path)OSError: [Errno 2] No such file or directory: '/tmp/cobra/export'