Traceback (most recent call last):


 "/usr/local/anaconda3/envs/cdat81/lib/python3.7/site-packages/xarray/backends/api.py", line 1089, in to_netcdf

dataset, store, writer, encoding=encoding, unlimited_dims=unlimited_dims


 "/usr/local/anaconda3/envs/cdat81/lib/python3.7/site-packages/xarray/backends/api.py", line 1135, in dump_to_store

store.store(variables, attrs, check_encoding, writer, unlimited_dims=unlimited_dims)


 "/usr/local/anaconda3/envs/cdat81/lib/python3.7/site-packages/xarray/backends/common.py", line 298, in store

variables, check_encoding_set, writer, unlimited_dims=unlimited_dims


 "/usr/local/anaconda3/envs/cdat81/lib/python3.7/site-packages/xarray/backends/common.py", line 339, in set_variables

writer.add(source, target)


 "/usr/local/anaconda3/envs/cdat81/lib/python3.7/site-packages/xarray/backends/common.py", line 188, in add

target[...] = source


 "/usr/local/anaconda3/envs/cdat81/lib/python3.7/site-packages/xarray/backends/netCDF4_.py", line 51, in __setitem__

data[key] = value


 "netCDF4/_netCDF4.pyx", line 4950, in netCDF4._netCDF4.Variable.__setitem__


 "netCDF4/_netCDF4.pyx", line 5229, in netCDF4._netCDF4.Variable._put


 "netCDF4/_netCDF4.pyx", line 1887, in netCDF4._netCDF4._ensure_nc_success
RuntimeError: NetCDF: HDF error

Traceback (most recent call last):


 "../../mmdetection/tools/train.py", line 108, in <module>

main()


 "../../mmdetection/tools/train.py", line 104, in main

logger=logger)


 "/home/user/repos/mmdetection1.3/mmdet/apis/train.py", line 61, in train_detector

_non_dist_train(model, dataset, cfg, validate=validate)


 "/home/user/repos/mmdetection1.3/mmdet/apis/train.py", line 225, in _non_dist_train

runner.run(data_loaders, cfg.workflow, cfg.total_epochs)


 "/home/user/anaconda3/envs/mmdet2/lib/python3.7/site-packages/mmcv/runner/runner.py", line 358, in run

epoch_runner(data_loaders[i], **kwargs)


 "/home/user/anaconda3/envs/mmdet2/lib/python3.7/site-packages/mmcv/runner/runner.py", line 264, in train

self.model, data_batch, train_mode=True, **kwargs)


 "/home/user/repos/mmdetection1.3/mmdet/apis/train.py", line 38, in batch_processor

losses = model(**data)


 "/home/user/anaconda3/envs/mmdet2/lib/python3.7/site-packages/torch/nn/modules/module.py", line 541, in __call__

result = self.forward(*input, **kwargs)


 "/home/user/anaconda3/envs/mmdet2/lib/python3.7/site-packages/torch/nn/parallel/data_parallel.py", line 150, in forward

return self.module(*inputs[0], **kwargs[0])


 "/home/user/anaconda3/envs/mmdet2/lib/python3.7/site-packages/torch/nn/modules/module.py", line 541, in __call__

result = self.forward(*input, **kwargs)


 "/home/user/repos/mmdetection1.3/mmdet/core/fp16/decorators.py", line 49, in new_func

return old_func(*args, **kwargs)


 "/home/user/repos/mmdetection1.3/mmdet/models/detectors/base.py", line 92, in forward

return self.forward_train(img, img_meta, **kwargs)


 "/home/user/repos/mmdetection1.3/mmdet/models/detectors/single_stage.py", line 56, in forward_train

x = self.extract_feat(img)


 "/home/user/repos/mmdetection1.3/mmdet/models/detectors/single_stage.py", line 40, in extract_feat

x = self.backbone(img)


 "/home/user/anaconda3/envs/mmdet2/lib/python3.7/site-packages/torch/nn/modules/module.py", line 541, in __call__

result = self.forward(*input, **kwargs)


 "/home/user/repos/mmdetection1.3/mmdet/models/backbones/resnet.py", line 526, in forward

x = res_layer(x)


 "/home/user/anaconda3/envs/mmdet2/lib/python3.7/site-packages/torch/nn/modules/module.py", line 541, in __call__

result = self.forward(*input, **kwargs)


 "/home/user/anaconda3/envs/mmdet2/lib/python3.7/site-packages/torch/nn/modules/container.py", line 92, in forward

input = module(input)


 "/home/user/anaconda3/envs/mmdet2/lib/python3.7/site-packages/torch/nn/modules/module.py", line 541, in __call__

result = self.forward(*input, **kwargs)


 "/home/user/repos/mmdetection1.3/mmdet/models/backbones/resnet.py", line 264, in forward

out = _inner_forward(x)


 "/home/user/repos/mmdetection1.3/mmdet/models/backbones/resnet.py", line 249, in _inner_forward

out = self.norm3(out)


 "/home/user/anaconda3/envs/mmdet2/lib/python3.7/site-packages/torch/nn/modules/module.py", line 541, in __call__

result = self.forward(*input, **kwargs)


 "/home/user/anaconda3/envs/mmdet2/lib/python3.7/site-packages/torch/nn/modules/batchnorm.py", line 81, in forward

exponential_average_factor, self.eps)


 "/home/user/anaconda3/envs/mmdet2/lib/python3.7/site-packages/torch/nn/functional.py", line 1670, in batch_norm

training, momentum, eps, torch.backends.cudnn.enabled
RuntimeError: CUDA out of memory. Tried to allocate 42.00 MiB (GPU 0; 7.80 GiB total capacity; 6.38 GiB already allocated; 23.25 MiB free; 602.65 MiB cached)
