Traceback (most recent call last):


 "/usr/local/bin/telepresence/telepresence/cli.py", line 136, in crash_reporting

yield


 "/usr/local/bin/telepresence/telepresence/main.py", line 77, in main

runner, remote_info, env, socks_port, ssh, mount_dir, pod_info


 "/usr/local/bin/telepresence/telepresence/outbound/setup.py", line 98, in launch

runner_, remote_info, command, args.also_proxy, env, ssh


 "/usr/local/bin/telepresence/telepresence/outbound/local.py", line 121, in launch_vpn

connect_sshuttle(runner, remote_info, also_proxy, ssh)


 "/usr/local/bin/telepresence/telepresence/outbound/vpn.py", line 312, in connect_sshuttle

raise RuntimeError("vpn-tcp tunnel did not connect")
RuntimeError: vpn-tcp tunnel did not connect

Traceback (most recent call last):


 "/usr/lib/python3.8/multiprocessing/process.py", line 315, in _bootstrap

self.run()


 "/usr/lib/python3.8/multiprocessing/process.py", line 108, in run

self._target(*self._args, **self._kwargs)


 "/home/user/projects/espnet/espnet2/tasks/abs_task.py", line 1304, in main_worker

cls.trainer.run(


 "/home/user/projects/espnet/espnet2/train/trainer.py", line 208, in run

all_steps_are_invalid = cls.train_one_epoch(


 "/home/user/projects/espnet/espnet2/train/trainer.py", line 393, in train_one_epoch

loss, stats, weight = model(**batch)


 "/home/user/projects/espnet/tools/venv/lib/python3.8/site-packages/torch/nn/modules/module.py", line 532, in __call__

result = self.forward(*input, **kwargs)


 "/home/user/projects/espnet/tools/venv/lib/python3.8/site-packages/torch/nn/parallel/distributed.py", line 447, in forward

output = self.module(*inputs[0], **kwargs[0])


 "/home/user/projects/espnet/tools/venv/lib/python3.8/site-packages/torch/nn/modules/module.py", line 532, in __call__

result = self.forward(*input, **kwargs)


 "/home/user/projects/espnet/espnet2/tts/espnet_model.py", line 104, in forward

return self.tts(


 "/home/user/projects/espnet/tools/venv/lib/python3.8/site-packages/torch/nn/modules/module.py", line 532, in __call__

result = self.forward(*input, **kwargs)


 "/home/user/projects/espnet/espnet2/tts/fastspeech2.py", line 408, in forward

before_outs, after_outs, d_outs, p_outs, e_outs = self._forward(


 "/home/user/projects/espnet/espnet2/tts/fastspeech2.py", line 521, in _forward

zs, _ = self.decoder(hs, h_masks)
# (B, Lmax, adim)


 "/home/user/projects/espnet/tools/venv/lib/python3.8/site-packages/torch/nn/modules/module.py", line 532, in __call__

result = self.forward(*input, **kwargs)


 "/home/user/projects/espnet/espnet/nets/pytorch_backend/conformer/encoder.py", line 223, in forward

xs, masks = self.encoders(xs, masks)


 "/home/user/projects/espnet/tools/venv/lib/python3.8/site-packages/torch/nn/modules/module.py", line 532, in __call__

result = self.forward(*input, **kwargs)


 "/home/user/projects/espnet/espnet/nets/pytorch_backend/transformer/repeat.py", line 18, in forward

args = m(*args)


 "/home/user/projects/espnet/tools/venv/lib/python3.8/site-packages/torch/nn/modules/module.py", line 532, in __call__

result = self.forward(*input, **kwargs)


 "/home/user/projects/espnet/espnet/nets/pytorch_backend/conformer/encoder_layer.py", line 119, in forward

x_att = self.self_attn(x_q, x, x, pos_emb, mask)


 "/home/user/projects/espnet/tools/venv/lib/python3.8/site-packages/torch/nn/modules/module.py", line 532, in __call__

result = self.forward(*input, **kwargs)


 "/home/user/projects/espnet/espnet/nets/pytorch_backend/transformer/attention.py", line 199, in forward

matrix_bd = torch.matmul(q_with_bias_v, p.transpose(-2, -1))
RuntimeError: CUDA out of memory. Tried to allocate 502.00 MiB (GPU 0; 23.64 GiB total capacity; 21.13 GiB already allocated; 238.00 MiB free; 22.11 GiB reserved in total by PyTorch)
