Traceback (most recent call last):


 "/usr/lib/python3/dist-packages/evdev/device.py", line 159, in __del__

self.close()


 "/usr/lib/python3/dist-packages/evdev/device.py", line 304, in close

super().close()


 "/usr/lib/python3/dist-packages/evdev/eventio_async.py", line 54, in close

loop = asyncio.get_event_loop()


 "/usr/lib/python3.8/asyncio/events.py", line 639, in get_event_loop

raise RuntimeError('There is no current event loop in thread %r.'
RuntimeError: There is no current event loop in thread 'Thread-3'.

Traceback (most recent call last):


 "../pretrain_graph.py", line 98, in <module>

'scaled_upper_triang_masked_softmax_fusion': True})


 "/data/quxiaoru/Code/rev_gen/megatron/training.py", line 129, in pretrain

train_data_iterator, valid_data_iterator)


 "/data/quxiaoru/Code/rev_gen/megatron/training.py", line 808, in train

lr_scheduler)


 "/data/quxiaoru/Code/rev_gen/megatron/training.py", line 594, in train_step

forward_step_func, data_iterator, model, optimizer, timers)


 "/data/quxiaoru/Code/rev_gen/megatron/training.py", line 507, in forward_backward_no_pipelining

loss, loss_reduced = forward_step_func(data_iterator, model, input_tensor=None)


 "../pretrain_graph.py", line 72, in forward_step

labels=labels)


 "/data/quxiaoru/anaconda3/envs/torch18/lib/python3.7/site-packages/torch/nn/modules/module.py", line 889, in _call_impl

result = self.forward(*input, **kwargs)


 "/data/quxiaoru/Code/rev_gen/megatron/model/distributed.py", line 76, in forward

return self.module(*inputs, **kwargs)


 "/data/quxiaoru/anaconda3/envs/torch18/lib/python3.7/site-packages/torch/nn/modules/module.py", line 889, in _call_impl

result = self.forward(*input, **kwargs)


 "/data/quxiaoru/Code/rev_gen/megatron/model/module.py", line 140, in forward

outputs = self.module(*inputs, **kwargs)


 "/data/quxiaoru/anaconda3/envs/torch18/lib/python3.7/site-packages/torch/nn/modules/module.py", line 889, in _call_impl

result = self.forward(*input, **kwargs)


 "/data/quxiaoru/Code/rev_gen/megatron/model/gpt_model.py", line 150, in forward

forward_method_parallel_output=forward_method_parallel_output)


 "/data/quxiaoru/Code/rev_gen/megatron/model/gpt_model.py", line 104, in forward

self.fp16_lm_cross_entropy)


 "/data/quxiaoru/Code/rev_gen/megatron/model/gpt_model.py", line 60, in post_language_model_processing

loss = mpu.vocab_parallel_cross_entropy(output.float(), labels)
RuntimeError: CUDA out of memory. Tried to allocate 9.33 GiB (GPU 0; 10.76 GiB total capacity; 5.17 GiB already allocated; 4.48 GiB free; 5.28 GiB reserved in total by PyTorch)
