Traceback (most recent call last):


 "train.py", line 218, in <module>

train(training_dbs, validation_db, args.start_iter)


 "train.py", line 160, in train

training_loss = nnet.train(**training)


 "/data2/yanmengkai/CornerNet/nnet/py_factory.py", line 82, in train

loss = self.network(xs, ys)


 "/root/anaconda3/envs/CornerNet/lib/python3.6/site-packages/torch/nn/modules/module.py", line 477, in __call__

result = self.forward(*input, **kwargs)


 "/data2/yanmengkai/CornerNet/models/py_utils/data_parallel.py", line 66, in forward

inputs, kwargs = self.scatter(inputs, kwargs, self.device_ids, self.chunk_sizes)


 "/data2/yanmengkai/CornerNet/models/py_utils/data_parallel.py", line 77, in scatter

return scatter_kwargs(inputs, kwargs, device_ids, dim=self.dim, chunk_sizes=self.chunk_sizes)


 "/data2/yanmengkai/CornerNet/models/py_utils/scatter_gather.py", line 30, in scatter_kwargs

inputs = scatter(inputs, target_gpus, dim, chunk_sizes) if inputs else []


 "/data2/yanmengkai/CornerNet/models/py_utils/scatter_gather.py", line 25, in scatter

return scatter_map(inputs)


 "/data2/yanmengkai/CornerNet/models/py_utils/scatter_gather.py", line 18, in scatter_map

return list(zip(*map(scatter_map, obj)))


 "/data2/yanmengkai/CornerNet/models/py_utils/scatter_gather.py", line 20, in scatter_map

return list(map(list, zip(*map(scatter_map, obj))))


 "/data2/yanmengkai/CornerNet/models/py_utils/scatter_gather.py", line 15, in scatter_map

return Scatter.apply(target_gpus, chunk_sizes, dim, obj)


 "/root/anaconda3/envs/CornerNet/lib/python3.6/site-packages/torch/nn/parallel/_functions.py", line 87, in forward

outputs = comm.scatter(input, ctx.target_gpus, ctx.chunk_sizes, ctx.dim, streams)


 "/root/anaconda3/envs/CornerNet/lib/python3.6/site-packages/torch/cuda/comm.py", line 142, in scatter

return tuple(torch._C._scatter(tensor, devices, chunk_sizes, dim, streams))
RuntimeError: CUDA error (10): invalid device ordinal (check_status at /opt/conda/conda-bld/pytorch_1532581333611/work/aten/src/ATen/cuda/detail/CUDAHooks.cpp:36)

Traceback (most recent call last):


 "/home/ubuntu/anaconda3/lib/python3.7/site-packages/ray/new_dashboard/agent.py", line 295, in <module>

raylet_name=args.raylet_name)


 "/home/ubuntu/anaconda3/lib/python3.7/site-packages/ray/new_dashboard/agent.py", line 66, in __init__

f"[::]:{self.dashboard_agent_port}")


 "/home/ubuntu/anaconda3/lib/python3.7/site-packages/grpc/aio/_server.py", line 84, in add_insecure_port

address, self._server.add_insecure_port(_common.encode(address)))


 "/home/ubuntu/anaconda3/lib/python3.7/site-packages/grpc/_common.py", line 166, in validate_port_binding_result

raise RuntimeError(_ERROR_MESSAGE_PORT_BINDING_FAILED % address)
RuntimeError: Failed to bind to address [::]:49324; set GRPC_VERBOSITY=debug environment variable to see detailed error message.
