Traceback (most recent call last):


 "train_parallel.py", line 450, in <module>

main()


 "train_parallel.py", line 184, in main

train(config, train_loader, model, criterion, optimizer, epoch)


 "train_parallel.py", line 308, in train

query_rep = model(query)


 "/opt/conda/lib/python3.6/site-packages/torch/nn/modules/module.py", line 525, in __call__

result = self.forward(*input, **kwargs)


 "/opt/conda/lib/python3.6/site-packages/torch/nn/parallel/distributed.py", line 385, in forward

output = self.module(*inputs[0], **kwargs[0])


 "/opt/conda/lib/python3.6/site-packages/torch/nn/modules/module.py", line 525, in __call__

result = self.forward(*input, **kwargs)


 "/root/c3_workspace/model/model.py", line 45, in forward

x = self.convs(x)


 "/opt/conda/lib/python3.6/site-packages/torch/nn/modules/module.py", line 525, in __call__

result = self.forward(*input, **kwargs)


 "/opt/conda/lib/python3.6/site-packages/torch/nn/modules/container.py", line 92, in forward

input = module(input)


 "/opt/conda/lib/python3.6/site-packages/torch/nn/modules/module.py", line 525, in __call__

result = self.forward(*input, **kwargs)


 "/opt/conda/lib/python3.6/site-packages/torch/nn/modules/container.py", line 92, in forward

input = module(input)


 "/opt/conda/lib/python3.6/site-packages/torch/nn/modules/module.py", line 525, in __call__

result = self.forward(*input, **kwargs)


 "/opt/conda/lib/python3.6/site-packages/torch/nn/modules/conv.py", line 345, in forward

self.padding, self.dilation, self.groups)


 "/opt/conda/lib/python3.6/site-packages/apex/amp/wrap.py", line 28, in wrapper

return orig_fn(*new_args, **kwargs)
RuntimeError: CUDA out of memory. Tried to allocate 160.00 MiB (GPU 0; 23.88 GiB total capacity; 2.94 GiB already allocated; 20.36 GiB free; 24.86 MiB cached)

Traceback (most recent call last):


 "/content/OpenNMT-py/onmt/trainer.py", line 347, in _gradient_accumulation

trunc_size=trunc_size)


 "/content/OpenNMT-py/onmt/utils/loss.py", line 158, in __call__

loss, stats = self._compute_loss(batch, **shard)


 "/content/OpenNMT-py/onmt/utils/loss.py", line 233, in _compute_loss

scores = self.generator(bottled_output)


 "/usr/local/lib/python3.6/dist-packages/torch/nn/modules/module.py", line 489, in __call__

result = self.forward(*input, **kwargs)


 "/usr/local/lib/python3.6/dist-packages/torch/nn/modules/container.py", line 92, in forward

input = module(input)


 "/usr/local/lib/python3.6/dist-packages/torch/nn/modules/module.py", line 489, in __call__

result = self.forward(*input, **kwargs)


 "/usr/local/lib/python3.6/dist-packages/torch/nn/modules/linear.py", line 67, in forward

return F.linear(input, self.weight, self.bias)


 "/usr/local/lib/python3.6/dist-packages/torch/nn/functional.py", line 1352, in linear

ret = torch.addmm(torch.jit._unwrap_optional(bias), input, weight.t())
RuntimeError: Expected object of backend CPU but got backend CUDA for argument #4 'mat1'
