Traceback (most recent call last):


 "pretrain_bert_nccl.py", line 629, in <module>

main()


 "pretrain_bert_nccl.py", line 607, in main

timers, args)


 "pretrain_bert_nccl.py", line 338, in train

args, timers)


 "pretrain_bert_nccl.py", line 297, in train_step

nsp_loss, args)


 "pretrain_bert_nccl.py", line 272, in backward_step

optimizer.update_master_grads()


 "/autofs/nccs-svm1_proj/bif120/deepforce/scripts/deepspeed/DeepSpeedExamples/Megatron-LM/fp16/fp16.py", line 566, in update_master_grads

self._model_grads_to_master_grads()


 "/autofs/nccs-svm1_proj/bif120/deepforce/scripts/deepspeed/DeepSpeedExamples/Megatron-LM/fp16/fp16.py", line 303, in _model_grads_to_master_grads

model_grads_to_master_grads(fp16_group, fp32_from_fp16_group)


 "/autofs/nccs-svm1_proj/bif120/deepforce/scripts/deepspeed/DeepSpeedExamples/Megatron-LM/fp16/fp16util.py", line 167, in model_grads_to_master_grads

master.grad = Variable(master.data.new(*master.data.size()))
RuntimeError: CUDA out of memory. Tried to allocate 24.00 MiB (GPU 2; 15.75 GiB total capacity; 14.13 GiB already allocated; 586.94 MiB free; 188.72 MiB cached; 0 bytes inactive)

Traceback (most recent call last):


 "testgen.py", line 4, in <module>

ai = aitextgen(model="pytorch/pytorch_model.bin")


 "D:\Anaconda3\envs\gpt2-pytorch\lib\site-packages\aitextgen\aitextgen.py", line 176, in __init__

self.model = GPT2LMHeadModel.from_pretrained(model, config=config)


 "D:\Anaconda3\envs\gpt2-pytorch\lib\site-packages\transformers\modeling_utils.py", line 751, in from_pretrained

raise RuntimeError(
RuntimeError: Error(s) in loading state_dict for GPT2LMHeadModel:
