Traceback (most recent call last):


 "/home4/md510/w2020/espnet_20200422/espnet/espnet/bin/asr_train.py", line 622, in <module>

main(sys.argv[1:])


 "/home4/md510/w2020/espnet_20200422/espnet/espnet/bin/asr_train.py", line 608, in main

train(args)


 "/home4/md510/w2020/espnet_20200422/espnet/espnet/asr/pytorch_backend/asr.py", line 813, in train

trainer.run()


 "/home4/md510/w2020/espnet_20200422/espnet/tools/venv/lib/python3.7/site-packages/chainer/training/trainer.py", line 349, in run

six.reraise(*exc_info)


 "/home4/md510/w2020/espnet_20200422/espnet/tools/venv/lib/python3.7/site-packages/six.py", line 703, in reraise

raise value


 "/home4/md510/w2020/espnet_20200422/espnet/tools/venv/lib/python3.7/site-packages/chainer/training/trainer.py", line 316, in run

update()


 "/home4/md510/w2020/espnet_20200422/espnet/espnet/asr/pytorch_backend/asr.py", line 241, in update

self.update_core()


 "/home4/md510/w2020/espnet_20200422/espnet/espnet/asr/pytorch_backend/asr.py", line 205, in update_core

data_parallel(self.model, x, range(self.ngpu)).mean() / self.accum_grad


 "/home4/md510/w2020/espnet_20200422/espnet/tools/venv/lib/python3.7/site-packages/torch/nn/parallel/data_parallel.py", line 204, in data_parallel

return module(*inputs[0], **module_kwargs[0])


 "/home4/md510/w2020/espnet_20200422/espnet/tools/venv/lib/python3.7/site-packages/torch/nn/modules/module.py", line 532, in __call__

result = self.forward(*input, **kwargs)


 "/home4/md510/w2020/espnet_20200422/espnet/espnet/nets/pytorch_backend/e2e_asr_transducer.py", line 413, in forward

loss = self.criterion(pred_pad, target, pred_len, target_len)


 "/home4/md510/w2020/espnet_20200422/espnet/tools/venv/lib/python3.7/site-packages/torch/nn/modules/module.py", line 532, in __call__

result = self.forward(*input, **kwargs)


 "/home4/md510/w2020/espnet_20200422/espnet/espnet/nets/pytorch_backend/transducer/loss.py", line 44, in forward

loss = self.trans_loss(pred_pad, target, pred_len, target_len)


 "/home4/md510/w2020/espnet_20200422/espnet/tools/venv/lib/python3.7/site-packages/torch/nn/modules/module.py", line 532, in __call__

result = self.forward(*input, **kwargs)


 "/home4/md510/w2020/espnet_20200422/espnet/tools/venv/lib/python3.7/site-packages/warprnnt_pytorch-0.1-py3.7-linux-x86_64.egg/warprnnt_pytorch/__init__.py", line 100, in forward

return self.loss(acts, labels, act_lens, label_lens, self.blank, self.reduction)


 "/home4/md510/w2020/espnet_20200422/espnet/tools/venv/lib/python3.7/site-packages/warprnnt_pytorch-0.1-py3.7-linux-x86_64.egg/warprnnt_pytorch/__init__.py", line 40, in forward

grads /= minibatch_size
RuntimeError: CUDA error: an illegal memory access was encountered

Traceback (most recent call last):


 "./tests/test_spatial_narrow_as_op.py", line 59, in test_small_forward_and_gradient

self._run_test(A, B, check_grad=True)


 "./tests/test_spatial_narrow_as_op.py", line 49, in _run_test

res, grad, grad_estimated = gc.CheckSimple(op, [A, B], 0, [0])


 "/home/jyy/anaconda3/envs/caffe/lib/python2.7/site-packages/caffe2/python/gradient_checker.py", line 284, in CheckSimple

outputs_with_grads


 "/home/jyy/anaconda3/envs/caffe/lib/python2.7/site-packages/caffe2/python/gradient_checker.py", line 201, in GetLossAndGrad

workspace.RunOperatorsOnce(grad_ops)


 "/home/jyy/anaconda3/envs/caffe/lib/python2.7/site-packages/caffe2/python/workspace.py", line 184, in RunOperatorsOnce

success = RunOperatorOnce(op)


 "/home/jyy/anaconda3/envs/caffe/lib/python2.7/site-packages/caffe2/python/workspace.py", line 179, in RunOperatorOnce

return C.run_operator_once(StringifyProto(operator))
RuntimeError: [enforce fail at context_gpu.h:171] . Encountered CUDA error: no kernel image is available for execution on the device Error from operator: 
