Traceback (most recent call last):


 "train.py", line 526, in <module>

train(hyp, opt, device, tb_writer, wandb)


 "train.py", line 292, in train

pred = model(imgs)
# forward


 "/home/ubuntu/anaconda3/envs/yolo5/lib/python3.8/site-packages/torch/nn/modules/module.py", line 727, in _call_impl

result = self.forward(*input, **kwargs)


 "/home/ubuntu/anaconda3/envs/yolo5/lib/python3.8/site-packages/torch/nn/parallel/data_parallel.py", line 160, in forward

replicas = self.replicate(self.module, self.device_ids[:len(inputs)])


 "/home/ubuntu/anaconda3/envs/yolo5/lib/python3.8/site-packages/torch/nn/parallel/data_parallel.py", line 165, in replicate

return replicate(module, device_ids, not torch.is_grad_enabled())


 "/home/ubuntu/anaconda3/envs/yolo5/lib/python3.8/site-packages/torch/nn/parallel/replicate.py", line 88, in replicate

param_copies = _broadcast_coalesced_reshape(params, devices, detach)


 "/home/ubuntu/anaconda3/envs/yolo5/lib/python3.8/site-packages/torch/nn/parallel/replicate.py", line 71, in _broadcast_coalesced_reshape

tensor_copies = Broadcast.apply(devices, *tensors)


 "/home/ubuntu/anaconda3/envs/yolo5/lib/python3.8/site-packages/torch/nn/parallel/_functions.py", line 22, in forward

outputs = comm.broadcast_coalesced(inputs, ctx.target_gpus)


 "/home/ubuntu/anaconda3/envs/yolo5/lib/python3.8/site-packages/torch/nn/parallel/comm.py", line 56, in broadcast_coalesced

return torch._C._broadcast_coalesced(tensors, devices, buffer_size)
RuntimeError: NCCL Error 2: unhandled system error

Traceback (most recent call last):


 "/usr/local/bin/telepresence/telepresence/cli.py", line 135, in crash_reporting

yield


 "/usr/local/bin/telepresence/telepresence/main.py", line 68, in main

socks_port, ssh = do_connect(runner, remote_info)


 "/usr/local/bin/telepresence/telepresence/connect/connect.py", line 119, in do_connect

args.from_pod


 "/usr/local/bin/telepresence/telepresence/connect/connect.py", line 70, in connect

raise RuntimeError("SSH to the cluster failed to start. See logfile.")
RuntimeError: SSH to the cluster failed to start. See logfile.
