Traceback (most recent call last):


 "examples/bert_pku_seg.py", line 89, in <module>

train()


 "examples/bert_pku_seg.py", line 48, in train

trainer.train(SAVE_DIR)


 "/data/home/liuyang/dlab/dlab/process/trainer.py", line 61, in train

after_batch_iter_hook=train_step_hook)


 "/data/home/liuyang/dlab/dlab/process/common.py", line 49, in data_runner

forward_output = model(batch_sentence)


 "/data/home/liuyang/pyenv/bert-pyt-p3/lib/python3.7/site-packages/torch/nn/modules/module.py", line 477, in __call__

result = self.forward(*input, **kwargs)


 "/data/home/liuyang/dlab/dlab/model/sequence_tagger.py", line 66, in forward

batch_words_present, seq_length = self.embedder.embed(sentences)


 "/data/home/liuyang/dlab/dlab/embedder/stack_embedder.py", line 23, in embed

present, _ = embedder(batch_sentence)


 "/data/home/liuyang/pyenv/bert-pyt-p3/lib/python3.7/site-packages/torch/nn/modules/module.py", line 477, in __call__

result = self.forward(*input, **kwargs)


 "/data/home/liuyang/dlab/dlab/embedder/base_embedder.py", line 28, in forward

return self.embed(*input)


 "/data/home/liuyang/dlab/dlab/embedder/bert_embedder.py", line 141, in embed

output_all_encoded_layers=False)


 "/data/home/liuyang/pyenv/bert-pyt-p3/lib/python3.7/site-packages/torch/nn/modules/module.py", line 477, in __call__

result = self.forward(*input, **kwargs)


 "/data/home/liuyang/pyenv/bert-pyt-p3/lib/python3.7/site-packages/pytorch_pretrained_bert/modeling.py", line 607, in forward

embedding_output = self.embeddings(input_ids, token_type_ids)


 "/data/home/liuyang/pyenv/bert-pyt-p3/lib/python3.7/site-packages/torch/nn/modules/module.py", line 477, in __call__

result = self.forward(*input, **kwargs)


 "/data/home/liuyang/pyenv/bert-pyt-p3/lib/python3.7/site-packages/pytorch_pretrained_bert/modeling.py", line 195, in forward

embeddings = words_embeddings + position_embeddings + token_type_embeddings
RuntimeError: cuda runtime error (59) : device-side assert triggered at /pytorch/aten/src/THC/generated/../generic/THCTensorMathPointwise.cu:266

Traceback (most recent call last):


 "train.py", line 120, in <module>

main(opt)


 "train.py", line 51, in main

single_main(opt, 0)


 "/home/kyungsoo/workspace/NLP/OpenNMT-py/onmt/train_single.py", line 154, in main

trainer.train(train_iter, valid_iter, opt.train_steps, opt.valid_steps)


 "/home/kyungsoo/workspace/NLP/OpenNMT-py/onmt/trainer.py", line 188, in train

valid_stats = self.validate(valid_iter)


 "/home/kyungsoo/workspace/NLP/OpenNMT-py/onmt/trainer.py", line 233, in validate

outputs, attns = self.model(src, tgt, src_lengths)


 "/home/kyungsoo/miniconda3/envs/opennmt/lib/python3.6/site-packages/torch/nn/modules/module.py", line 489, in __call__

result = self.forward(*input, **kwargs)


 "/home/kyungsoo/workspace/NLP/OpenNMT-py/onmt/models/model.py", line 46, in forward

memory_lengths=lengths)


 "/home/kyungsoo/miniconda3/envs/opennmt/lib/python3.6/site-packages/torch/nn/modules/module.py", line 489, in __call__

result = self.forward(*input, **kwargs)


 "/home/kyungsoo/workspace/NLP/OpenNMT-py/onmt/decoders/transformer.py", line 232, in forward

step=step)


 "/home/kyungsoo/miniconda3/envs/opennmt/lib/python3.6/site-packages/torch/nn/modules/module.py", line 489, in __call__

result = self.forward(*input, **kwargs)


 "/home/kyungsoo/workspace/NLP/OpenNMT-py/onmt/decoders/transformer.py", line 92, in forward

output = self.feed_forward(self.drop(mid) + query)


 "/home/kyungsoo/miniconda3/envs/opennmt/lib/python3.6/site-packages/torch/nn/modules/module.py", line 489, in __call__

result = self.forward(*input, **kwargs)


 "/home/kyungsoo/workspace/NLP/OpenNMT-py/onmt/modules/position_ffn.py", line 38, in forward

inter = self.dropout_1(self.relu(self.w_1(self.layer_norm(x))))


 "/home/kyungsoo/miniconda3/envs/opennmt/lib/python3.6/site-packages/torch/nn/modules/module.py", line 489, in __call__

result = self.forward(*input, **kwargs)


 "/home/kyungsoo/miniconda3/envs/opennmt/lib/python3.6/site-packages/torch/nn/modules/linear.py", line 67, in forward

return F.linear(input, self.weight, self.bias)


 "/home/kyungsoo/miniconda3/envs/opennmt/lib/python3.6/site-packages/torch/nn/functional.py", line 1354, in linear

output = input.matmul(weight.t())
RuntimeError: CUDA out of memory. Tried to allocate 39.75 MiB (GPU 0; 10.91 GiB total capacity; 9.17 GiB already allocated; 35.88 MiB free; 865.22 MiB cached)