[
] 0/120, elapsed: 0s, ETA:Traceback (most recent call last):


 "./tools/train.py", line 181, in <module>

main()


 "./tools/train.py", line 177, in main

meta=meta)


 "/opt/data/private/fuxin/mmsegmentation/ccnet_base/mmseg/apis/train.py", line 106, in train_segmentor

runner.run(data_loaders, cfg.workflow, cfg.total_iters)


 "/root/anaconda3/envs/py36tf112/lib/python3.6/site-packages/mmcv/runner/iter_based_runner.py", line 130, in run

iter_runner(iter_loaders[i], **kwargs)


 "/root/anaconda3/envs/py36tf112/lib/python3.6/site-packages/mmcv/runner/iter_based_runner.py", line 66, in train

self.call_hook('after_train_iter')


 "/root/anaconda3/envs/py36tf112/lib/python3.6/site-packages/mmcv/runner/base_runner.py", line 307, in call_hook

getattr(hook, fn_name)(self)


 "/opt/data/private/fuxin/mmsegmentation/ccnet_base/mmseg/core/evaluation/eval_hooks.py", line 78, in after_train_iter

gpu_collect=self.gpu_collect)


 "/opt/data/private/fuxin/mmsegmentation/ccnet_base/mmseg/apis/test.py", line 112, in multi_gpu_test

result = model(return_loss=False, rescale=True, **data)


 "/root/anaconda3/envs/py36tf112/lib/python3.6/site-packages/torch/nn/modules/module.py", line 722, in _call_impl

result = self.forward(*input, **kwargs)


 "/root/anaconda3/envs/py36tf112/lib/python3.6/site-packages/torch/nn/parallel/distributed.py", line 511, in forward

output = self.module(*inputs[0], **kwargs[0])


 "/root/anaconda3/envs/py36tf112/lib/python3.6/site-packages/torch/nn/modules/module.py", line 722, in _call_impl

result = self.forward(*input, **kwargs)


 "/root/anaconda3/envs/py36tf112/lib/python3.6/site-packages/mmcv/runner/fp16_utils.py", line 84, in new_func

return old_func(*args, **kwargs)


 "/opt/data/private/fuxin/mmsegmentation/ccnet_base/mmseg/models/segmentors/base.py", line 124, in forward

return self.forward_test(img, img_metas, **kwargs)


 "/opt/data/private/fuxin/mmsegmentation/ccnet_base/mmseg/models/segmentors/base.py", line 106, in forward_test

return self.simple_test(imgs[0], img_metas[0], **kwargs)
 
 "/opt/data/private/fuxin/mmsegmentation/ccnet_base/mmseg/models/segmentors/encoder_decoder.py", line 265, in simple_test

seg_logit = self.inference(img, img_meta, rescale)


 "/opt/data/private/fuxin/mmsegmentation/ccnet_base/mmseg/models/segmentors/encoder_decoder.py", line 250, in inference

seg_logit = self.whole_inference(img, img_meta, rescale)


 "/opt/data/private/fuxin/mmsegmentation/ccnet_base/mmseg/models/segmentors/encoder_decoder.py", line 224, in whole_inference

warning=False)


 "/opt/data/private/fuxin/mmsegmentation/ccnet_base/mmseg/ops/wrappers.py", line 29, in resize

return F.interpolate(input, size, scale_factor, mode, align_corners)


 "/root/anaconda3/envs/py36tf112/lib/python3.6/site-packages/torch/nn/functional.py", line 3163, in interpolate

return torch._C._nn.upsample_bilinear2d(input, output_size, align_corners, sfl[0], sfl[1])
RuntimeError: CUDA out of memory. Tried to allocate 3.69 GiB (GPU 1; 11.91 GiB total capacity; 1.16 GiB already allocated; 2.84 GiB free; 8.41 GiB reserved in total by PyTorch)

Traceback (most recent call last):


 "/home/suyali/anaconda3/lib/python3.5/site-packages/tensorflow/python/client/session.py", line 1356, in _do_call

return fn(*args)


 "/home/suyali/anaconda3/lib/python3.5/site-packages/tensorflow/python/client/session.py", line 1341, in _run_fn

options, feed_dict, fetch_list, target_list, run_metadata)


 "/home/suyali/anaconda3/lib/python3.5/site-packages/tensorflow/python/client/session.py", line 1429, in _call_tf_sessionrun

run_metadata)
tensorflow.python.framework.errors_impl.InvalidArgumentError: 2 root error(s) found.
