Traceback (most recent call last):


 "/home/users/ntu/tlvu/espnet_v0.7_202004011/espnet/tools/venv/lib/python3.7/multiprocessing/process.py", line 297, in _bootstrap

self.run()


 "/home/users/ntu/tlvu/espnet_v0.7_202004011/espnet/tools/venv/lib/python3.7/multiprocessing/process.py", line 99, in run

self._target(*self._args, **self._kwargs)


 "/home/users/ntu/tlvu/espnet_v0.7_202004011/espnet/espnet2/tasks/abs_task.py", line 1161, in main_worker

distributed_option=distributed_option,


 "/home/users/ntu/tlvu/espnet_v0.7_202004011/espnet/espnet2/train/trainer.py", line 209, in run

options=trainer_options,


 "/home/users/ntu/tlvu/espnet_v0.7_202004011/espnet/espnet2/train/trainer.py", line 382, in train_one_epoch

loss, stats, weight = model(**batch)


 "/home/users/ntu/tlvu/espnet_v0.7_202004011/espnet/tools/venv/lib/python3.7/site-packages/torch/nn/modules/module.py", line 493, in __call__

result = self.forward(*input, **kwargs)


 "/home/users/ntu/tlvu/espnet_v0.7_202004011/espnet/tools/venv/lib/python3.7/site-packages/torch/nn/parallel/distributed.py", line 376, in forward

output = self.module(*inputs[0], **kwargs[0])


 "/home/users/ntu/tlvu/espnet_v0.7_202004011/espnet/tools/venv/lib/python3.7/site-packages/torch/nn/modules/module.py", line 493, in __call__

result = self.forward(*input, **kwargs)


 "/home/users/ntu/tlvu/espnet_v0.7_202004011/espnet/espnet2/asr/espnet_model.py", line 115, in forward

encoder_out, encoder_out_lens = self.encode(speech, speech_lengths)


 "/home/users/ntu/tlvu/espnet_v0.7_202004011/espnet/espnet2/asr/espnet_model.py", line 191, in encode

encoder_out, encoder_out_lens, _ = self.encoder(feats, feats_lengths)


 "/home/users/ntu/tlvu/espnet_v0.7_202004011/espnet/tools/venv/lib/python3.7/site-packages/torch/nn/modules/module.py", line 493, in __call__

result = self.forward(*input, **kwargs)


 "/home/users/ntu/tlvu/espnet_v0.7_202004011/espnet/espnet2/asr/encoder/transformer_encoder.py", line 160, in forward

xs_pad, masks = self.encoders(xs_pad, masks)


 "/home/users/ntu/tlvu/espnet_v0.7_202004011/espnet/tools/venv/lib/python3.7/site-packages/torch/nn/modules/module.py", line 493, in __call__

result = self.forward(*input, **kwargs)


 "/home/users/ntu/tlvu/espnet_v0.7_202004011/espnet/espnet/nets/pytorch_backend/transformer/repeat.py", line 18, in forward

args = m(*args)


 "/home/users/ntu/tlvu/espnet_v0.7_202004011/espnet/tools/venv/lib/python3.7/site-packages/torch/nn/modules/module.py", line 493, in __call__

result = self.forward(*input, **kwargs)


 "/home/users/ntu/tlvu/espnet_v0.7_202004011/espnet/espnet/nets/pytorch_backend/transformer/encoder_layer.py", line 77, in forward

x = residual + self.dropout(self.feed_forward(x))


 "/home/users/ntu/tlvu/espnet_v0.7_202004011/espnet/tools/venv/lib/python3.7/site-packages/torch/nn/modules/module.py", line 493, in __call__

result = self.forward(*input, **kwargs)


 "/home/users/ntu/tlvu/espnet_v0.7_202004011/espnet/espnet/nets/pytorch_backend/transformer/positionwise_feed_forward.py", line 30, in forward

return self.w_2(self.dropout(torch.relu(self.w_1(x))))


 "/home/users/ntu/tlvu/espnet_v0.7_202004011/espnet/tools/venv/lib/python3.7/site-packages/torch/nn/modules/module.py", line 493, in __call__

result = self.forward(*input, **kwargs)


 "/home/users/ntu/tlvu/espnet_v0.7_202004011/espnet/tools/venv/lib/python3.7/site-packages/torch/nn/modules/dropout.py", line 58, in forward

return F.dropout(input, self.p, self.training, self.inplace)


 "/home/users/ntu/tlvu/espnet_v0.7_202004011/espnet/tools/venv/lib/python3.7/site-packages/torch/nn/functional.py", line 830, in dropout

else _VF.dropout(input, p, training))
RuntimeError: CUDA out of memory. Tried to allocate 102.00 MiB (GPU 1; 15.75 GiB total capacity; 14.54 GiB already allocated; 7.75 MiB free; 191.95 MiB cached)

Traceback (most recent call last):


 "{source}/train_pl.py", line 248, in <module>

main()


 "{source}/train_pl.py", line 235, in main

trainer.fit(model, dataloaders["train"], dataloaders["val"])


 "{venv}/pytorch_lightning/trainer/trainer.py", line 510, in fit

self.pre_dispatch()


 "{venv}/pytorch_lightning/trainer/trainer.py", line 539, in pre_dispatch

self.accelerator.pre_dispatch()


 "{venv}/pytorch_lightning/accelerators/accelerator.py", line 84, in pre_dispatch

self.training_type_plugin.pre_dispatch()


 "{venv}/pytorch_lightning/plugins/training_type/ddp.py", line 254, in pre_dispatch

self.model_to_device()


 "{venv}/pytorch_lightning/plugins/training_type/ddp.py", line 279, in model_to_device

self.model.to(self.root_device)


 "{venv}/pytorch_lightning/core/decorators.py", line 89, in inner_fn

module = fn(self, *args, **kwargs)


 "{venv}/pytorch_lightning/utilities/device_dtype_mixin.py", line 120, in to

return super().to(*args, **kwargs)


 "{venv}/pytorch/torch/nn/modules/module.py", line 612, in to

return self._apply(convert)


 "{venv}/pytorch/torch/nn/modules/module.py", line 359, in _apply

module._apply(fn)


 "{venv}/pytorch/torch/nn/modules/module.py", line 359, in _apply

module._apply(fn)


 "{venv}/pytorch/torch/nn/modules/module.py", line 359, in _apply

module._apply(fn)


 "{venv}/pytorch/torch/nn/modules/module.py", line 381, in _apply

param_applied = fn(param)


 "{venv}/pytorch/torch/nn/modules/module.py", line 610, in convert

return t.to(device, dtype if t.is_floating_point() else None, non_blocking)
RuntimeError: cuda runtime error (711) : peer mapping resources exhausted at /pytorch/aten/src/THC/THCGeneral.cpp:139 
