Traceback (most recent call last):


 "mitmproxy\net\tcp.py", line 621, in serve_forever


 "threading.py", line 846, in start
RuntimeError: can't start new thread

Traceback (most recent call last):


 "<stdin>", line 1, in <module>


 "ru-gpts/src/xl_wrapper.py", line 281, in __call__

lm_logits = self.model(tokens, position_ids, attention_mask)


 "/home/antoly/3env/lib/python3.6/site-packages/torch/nn/modules/module.py", line 727, in _call_impl

result = self.forward(*input, **kwargs)


 "ru-gpts/src/fp16/fp16.py", line 72, in forward

return fp16_to_fp32(self.module(*(fp32_to_fp16(inputs)), **kwargs))


 "/home/antoly/3env/lib/python3.6/site-packages/torch/nn/modules/module.py", line 727, in _call_impl

result = self.forward(*input, **kwargs)


 "ru-gpts/src/model/gpt3_modeling.py", line 108, in forward

transformer_output = self.transformer(embeddings, attention_mask)


 "/home/antoly/3env/lib/python3.6/site-packages/torch/nn/modules/module.py", line 727, in _call_impl

result = self.forward(*input, **kwargs)


 "ru-gpts/src/mpu/transformer.py", line 449, in forward

hidden_states = layer(hidden_states, attention_mask)


 "/home/antoly/3env/lib/python3.6/site-packages/torch/nn/modules/module.py", line 727, in _call_impl

result = self.forward(*input, **kwargs)


 "ru-gpts/src/mpu/transformer.py", line 301, in forward

attention_output = self.attention(layernorm_output, ltor_mask)


 "/home/antoly/3env/lib/python3.6/site-packages/torch/nn/modules/module.py", line 727, in _call_impl

result = self.forward(*input, **kwargs)


 "ru-gpts/src/mpu/transformer.py", line 131, in forward

attn_mask=ltor_mask)


 "/home/antoly/3env/lib/python3.6/site-packages/torch/nn/modules/module.py", line 727, in _call_impl

result = self.forward(*input, **kwargs)


 "/home/antoly/3env/lib/python3.6/site-packages/deepspeed/ops/sparse_attention/sparse_self_attention.py", line 130, in forward

attn_output_weights = sparse_dot_sdd_nt(query, key)


 "/home/antoly/3env/lib/python3.6/site-packages/deepspeed/ops/sparse_attention/matmul.py", line 746, in __call__

time_db)


 "/home/antoly/3env/lib/python3.6/site-packages/deepspeed/ops/sparse_attention/matmul.py", line 550, in forward

c_time)


 "/home/antoly/3env/lib/python3.6/site-packages/deepspeed/ops/sparse_attention/matmul.py", line 228, in _sdd_matmul

bench=bench)


 "/home/antoly/3env/lib/python3.6/site-packages/triton/kernel.py", line 86, in __call__

torch.ops.triton.launch_kernel(self.op_id, device, params)
RuntimeError: CUDA: Error- invalid ptx
