Traceback (most recent call last):


 "run/run_training.py", line 114, in <module>

trainer.validate(save_softmax=args.npz, validation_folder_name=val_folder)


 "/home/zlf/nnUNet/nnunet/training/network_training/nnUNetTrainer.py", line 478, in validate

use_gaussian=use_gaussian)


 "/home/zlf/nnUNet/nnunet/training/network_training/nnUNetTrainer.py", line 415, in predict_preprocessed_data_return_softmax

pad_kwargs=self.inference_pad_kwargs)[2]


 "/home/zlf/nnUNet/nnunet/network_architecture/neural_network.py", line 96, in predict_3D

all_in_gpu=all_in_gpu)


 "/home/zlf/nnUNet/nnunet/network_architecture/neural_network.py", line 248, in _internal_predict_3D_3Dconv_tiled

num_repeats, mirror_axes, do_mirroring, add_torch)[0]


 "/home/zlf/nnUNet/nnunet/network_architecture/neural_network.py", line 363, in _internal_maybe_mirror_and_pred_3D

x = to_cuda(maybe_to_torch(x), gpu_id=self.get_device())


 "/home/zlf/nnUNet/nnunet/utilities/to_torch.py", line 30, in to_cuda

data = data.cuda(gpu_id, non_blocking=True)
RuntimeError: Source tensor must be contiguous.

Traceback (most recent call last):


 "/home/sdhar/tudelft-research/pytorch/pytorch_synthetic_benchmark.py", line 127, in <module>

timeit.timeit(benchmark_step, number=args.num_warmup_batches)


 "/cm/shared/package/python/3.5.2/lib/python3.5/timeit.py", line 213, in timeit

return Timer(stmt, setup, timer, globals).timeit(number)


 "/cm/shared/package/python/3.5.2/lib/python3.5/timeit.py", line 178, in timeit

timing = self.inner(it, self.timer)


 "<timeit-src>", line 6, in inner


 "/home/sdhar/tudelft-research/pytorch/pytorch_synthetic_benchmark.py", line 79, in benchmark_step

optimizer.step()


 "/var/scratch/sdhar/venv_pytorch/lib/python3.5/site-packages/horovod/torch/__init__.py", line 193, in step

self.synchronize()


 "/var/scratch/sdhar/venv_pytorch/lib/python3.5/site-packages/horovod/torch/__init__.py", line 157, in synchronize

output = synchronize(handle)


 "/var/scratch/sdhar/venv_pytorch/lib/python3.5/site-packages/horovod/torch/mpi_ops.py", line 443, in synchronize

mpi_lib.horovod_torch_wait_and_clear(handle)
RuntimeError: ncclCommInitRank failed: unhandled cuda error
