Traceback (most recent call last):


 "train.py", line 330, in <module>

main(params)


 "train.py", line 278, in main

trainer.clm_step(lang1, lang2, params.lambda_clm)


 "/home/test/xlm/src/trainer.py", line 587, in clm_step

_, loss = model('predict', tensor=tensor, pred_mask=pred_mask, y=y, get_scores=False)


 "/home/test/anaconda3/envs/pytorch/lib/python3.6/site-packages/torch/nn/modules/module.py", line 493, in __call__

result = self.forward(*input, **kwargs)


 "/home/test/anaconda3/envs/pytorch/lib/python3.6/site-packages/apex/parallel/distributed.py", line 476, in forward

result = self.module(*inputs, **kwargs)


 "/home/test/anaconda3/envs/pytorch/lib/python3.6/site-packages/torch/nn/modules/module.py", line 493, in __call__

result = self.forward(*input, **kwargs)


 "/home/test/xlm/src/model/transformer.py", line 313, in forward

return self.predict(**kwargs)


 "/home/test/xlm/src/model/transformer.py", line 415, in predict

scores, loss = self.pred_layer(masked_tensor, y, get_scores)


 "/home/test/anaconda3/envs/pytorch/lib/python3.6/site-packages/torch/nn/modules/module.py", line 493, in __call__

result = self.forward(*input, **kwargs)


 "/home/test/xlm/src/model/transformer.py", line 132, in forward

loss = F.cross_entropy(scores, y, reduction='elementwise_mean')


 "/home/test/anaconda3/envs/pytorch/lib/python3.6/site-packages/torch/nn/functional.py", line 2056, in cross_entropy

return nll_loss(log_softmax(input, 1), target, weight, None, ignore_index, None, reduction)


 "/home/test/anaconda3/envs/pytorch/lib/python3.6/site-packages/torch/nn/functional.py", line 1871, in nll_loss

ret = torch._C._nn.nll_loss(input, target, weight, _Reduction.get_enum(reduction), ignore_index)
RuntimeError: cuda runtime error (77) : an illegal memory access was encountered at /pytorch/aten/src/THCUNN/generic/ClassNLLCriterion.cu:110

Traceback (most recent call last):


 "scripts/train_mobile.py", line 372, in <module>

train(train_args, data_path, save_path)


 "scripts/train_mobile.py", line 75, in train

model = torch.nn.DataParallel(model, device_ids=[0, 1]).cuda()


 "/opt/miniconda3/lib/python3.6/site-packages/torch/nn/modules/module.py", line 230, in cuda

return self._apply(lambda t: t.cuda(device))


 "/opt/miniconda3/lib/python3.6/site-packages/torch/nn/modules/module.py", line 152, in _apply

module._apply(fn)


 "/opt/miniconda3/lib/python3.6/site-packages/torch/nn/modules/module.py", line 152, in _apply

module._apply(fn)


 "/opt/miniconda3/lib/python3.6/site-packages/torch/nn/modules/module.py", line 152, in _apply

module._apply(fn)


 "/opt/miniconda3/lib/python3.6/site-packages/torch/nn/modules/module.py", line 158, in _apply

param.data = fn(param.data)


 "/opt/miniconda3/lib/python3.6/site-packages/torch/nn/modules/module.py", line 230, in <lambda>

return self._apply(lambda t: t.cuda(device))
RuntimeError: cuda runtime error (38) : no CUDA-capable device is detected at /home/kenzhang/pytorch/aten/src/THC/THCGeneral.cpp:70