Traceback (most recent call last):


 "test.py", line 61, in <module>

generated = model.inference(data['label'], data['inst'], data['image'])


 "/home/shu/pix2pixHD2/models/pix2pixHD_model.py", line 200, in inference

input_label, inst_map, real_image, _ = self.encode_input(Variable(label), Variable(inst), image, infer=True)


 "/home/shu/pix2pixHD2/models/pix2pixHD_model.py", line 127, in encode_input

inst_map = inst_map.data.cuda()
RuntimeError: cuda runtime error (59) : device-side assert triggered at /opt/conda/conda-bld/pytorch_1525812548180/work/aten/src/THC/generic/THCTensorCopy.c:20

Traceback (most recent call last):


 "pretrain_bert_nccl.py", line 629, in <module>

main()


 "pretrain_bert_nccl.py", line 607, in main

timers, args)


 "pretrain_bert_nccl.py", line 338, in train

args, timers)


 "pretrain_bert_nccl.py", line 297, in train_step

nsp_loss, args)


 "pretrain_bert_nccl.py", line 272, in backward_step

optimizer.update_master_grads()


 "/autofs/nccs-svm1_proj/bif120/deepforce/scripts/deepspeed/DeepSpeedExamples/Megatron-LM/fp16/fp16.py", line 566, in update_master_grads

self._model_grads_to_master_grads()


 "/autofs/nccs-svm1_proj/bif120/deepforce/scripts/deepspeed/DeepSpeedExamples/Megatron-LM/fp16/fp16.py", line 303, in _model_grads_to_master_grads

model_grads_to_master_grads(fp16_group, fp32_from_fp16_group)


 "/autofs/nccs-svm1_proj/bif120/deepforce/scripts/deepspeed/DeepSpeedExamples/Megatron-LM/fp16/fp16util.py", line 167, in model_grads_to_master_grads

master.grad = Variable(master.data.new(*master.data.size()))
RuntimeError: CUDA out of memory. Tried to allocate 24.00 MiB (GPU 0; 15.75 GiB total capacity; 14.04 GiB already allocated; 580.94 MiB free; 200.72 MiB cached; 0 bytes inactive)
