Traceback (most recent call last):


 "/Users/yonatanalexander/opt/anaconda3/envs/xds/lib/python3.7/site-packages/vaex/dataframe.py", line 3523, in table_part

values[name] = df.evaluate(name)


 "/Users/yonatanalexander/opt/anaconda3/envs/xds/lib/python3.7/site-packages/vaex/dataframe.py", line 5165, in evaluate

self.map_reduce(assign, lambda *_: None, expressions, ignore_filter=False, selection=selection, pre_filter=use_filter, info=True, to_numpy=False)


 "/Users/yonatanalexander/opt/anaconda3/envs/xds/lib/python3.7/site-packages/vaex/dataframe.py", line 306, in map_reduce

return self._delay(delay, task)


 "/Users/yonatanalexander/opt/anaconda3/envs/xds/lib/python3.7/site-packages/vaex/dataframe.py", line 1428, in _delay

self.execute()


 "/Users/yonatanalexander/opt/anaconda3/envs/xds/lib/python3.7/site-packages/vaex/dataframe.py", line 293, in execute

self.executor.execute()


 "/Users/yonatanalexander/opt/anaconda3/envs/xds/lib/python3.7/site-packages/vaex/execution.py", line 127, in execute

raise RuntimeError("nested execute call")
RuntimeError: nested execute call

Traceback (most recent call last):


 "tools/train_net.py", line 186, in <module>

main()


 "tools/train_net.py", line 179, in main

model = train(cfg, args.local_rank, args.distributed)


 "tools/train_net.py", line 85, in train

arguments,


 "/data/ryancheng/maskrcnn-benchmark/maskrcnn_benchmark/engine/trainer.py", line 67, in do_train

loss_dict = model(images, targets)


 "/data/ryancheng/miniconda3/envs/maskrcnn_benchmark/lib/python3.7/site-packages/torch/nn/modules/module.py", line 493, in __call__

result = self.forward(*input, **kwargs)


 "/data/ryancheng/miniconda3/envs/maskrcnn_benchmark/lib/python3.7/site-packages/apex-0.1-py3.7-linux-x86_64.egg/apex/amp/_initialize.py", line 194, in new_fwd

**applier(kwargs, input_caster))


 "/data/ryancheng/maskrcnn-benchmark/maskrcnn_benchmark/modeling/detector/generalized_rcnn.py", line 50, in forward

proposals, proposal_losses = self.rpn(images, features, targets)


 "/data/ryancheng/miniconda3/envs/maskrcnn_benchmark/lib/python3.7/site-packages/torch/nn/modules/module.py", line 493, in __call__

result = self.forward(*input, **kwargs)


 "/data/ryancheng/maskrcnn-benchmark/maskrcnn_benchmark/modeling/rpn/rpn.py", line 159, in forward

return self._forward_train(anchors, objectness, rpn_box_regression, targets)


 "/data/ryancheng/maskrcnn-benchmark/maskrcnn_benchmark/modeling/rpn/rpn.py", line 175, in _forward_train

anchors, objectness, rpn_box_regression, targets


 "/data/ryancheng/miniconda3/envs/maskrcnn_benchmark/lib/python3.7/site-packages/torch/nn/modules/module.py", line 493, in __call__

result = self.forward(*input, **kwargs)


 "/data/ryancheng/maskrcnn-benchmark/maskrcnn_benchmark/modeling/rpn/inference.py", line 140, in forward

sampled_boxes.append(self.forward_for_single_feature_map(a, o, b))


 "/data/ryancheng/maskrcnn-benchmark/maskrcnn_benchmark/modeling/rpn/inference.py", line 115, in forward_for_single_feature_map

boxlist = remove_small_boxes(boxlist, self.min_size)


 "/data/ryancheng/maskrcnn-benchmark/maskrcnn_benchmark/structures/boxlist_ops.py", line 46, in remove_small_boxes

(ws >= min_size) & (hs >= min_size)
RuntimeError: copy_if failed to synchronize: device-side assert triggered
