Traceback (most recent call last):


 "C:/Users/USER/PycharmProjects/Testing/huggingface/run_lm_finetuning.py", line 790, in <module>

main()


 "C:/Users/USER/PycharmProjects/Testing/huggingface/run_lm_finetuning.py", line 740, in main

global_step, tr_loss = train(args, train_dataset, model, tokenizer)


 "C:/Users/USER/PycharmProjects/Testing/huggingface/run_lm_finetuning.py", line 356, in train

loss.backward()


 "C:\Users\USER\Anaconda3\lib\site-packages\torch\tensor.py", line 195, in backward

torch.autograd.backward(self, gradient, retain_graph, create_graph)


 "C:\Users\USER\Anaconda3\lib\site-packages\torch\autograd\__init__.py", line 99, in backward

allow_unreachable=True)
# allow_unreachable flag
RuntimeError: CUDA error: device-side assert triggered

Traceback (most recent call last):


 "./tools/train.py", line 90, in <module>

main()


 "./tools/train.py", line 86, in main

logger=logger)


 "/usr/local/lib/python3.6/site-packages/mmdet-0.5.4+unknown-py3.6.egg/mmdet/apis/train.py", line 59, in train_detector

_non_dist_train(model, dataset, cfg, validate=validate)


 "/usr/local/lib/python3.6/site-packages/mmdet-0.5.4+unknown-py3.6.egg/mmdet/apis/train.py", line 117, in _non_dist_train

runner.run(data_loaders, cfg.workflow, cfg.total_epochs)


 "/usr/local/lib/python3.6/site-packages/mmcv/runner/runner.py", line 349, in run

epoch_runner(data_loaders[i], **kwargs)


 "/usr/local/lib/python3.6/site-packages/mmcv/runner/runner.py", line 265, in train

self.call_hook('after_train_epoch')


 "/usr/local/lib/python3.6/site-packages/mmcv/runner/runner.py", line 222, in call_hook

getattr(hook, fn_name)(self)


 "/usr/local/lib/python3.6/site-packages/mmcv/runner/utils.py", line 31, in wrapper

return func(*args, **kwargs)


 "/usr/local/lib/python3.6/site-packages/mmcv/runner/hooks/checkpoint.py", line 25, in after_train_epoch

self.out_dir, save_optimizer=self.save_optimizer, **self.args)


 "/usr/local/lib/python3.6/site-packages/mmcv/runner/runner.py", line 242, in save_checkpoint

save_checkpoint(self.model, filename, optimizer=optimizer, meta=meta)


 "/usr/local/lib/python3.6/site-packages/mmcv/runner/checkpoint.py", line 156, in save_checkpoint

torch.save(checkpoint, filename)


 "/usr/local/lib/python3.6/site-packages/torch/serialization.py", line 209, in save

return _with_file_like(f, "wb", lambda f: _save(obj, f, pickle_module, pickle_protocol))


 "/usr/local/lib/python3.6/site-packages/torch/serialization.py", line 134, in _with_file_like

return body(f)


 "/usr/local/lib/python3.6/site-packages/torch/serialization.py", line 209, in <lambda>

return _with_file_like(f, "wb", lambda f: _save(obj, f, pickle_module, pickle_protocol))


 "/usr/local/lib/python3.6/site-packages/torch/serialization.py", line 288, in _save

serialized_storages[key]._write_file(f, _should_read_directly(f))
RuntimeError: Unknown error -1
