Traceback (most recent call last):


 "/usr/local/bin/telepresence/telepresence/cli.py", line 130, in crash_reporting

yield


 "/usr/local/bin/telepresence/telepresence/main.py", line 60, in main

remote_info = start_proxy(runner)


 "/usr/local/bin/telepresence/telepresence/proxy/__init__.py", line 104, in start_proxy

run_id=run_id,


 "/usr/local/bin/telepresence/telepresence/proxy/remote.py", line 205, in get_remote_info

wait_for_pod(runner, remote_info)


 "/usr/local/bin/telepresence/telepresence/proxy/remote.py", line 137, in wait_for_pod

"Pod isn't starting or can't be found: {}".format(pod["status"])
RuntimeError: Pod isn't starting or can't be found: {'conditions': [{'lastProbeTime': None, 'lastTransitionTime': '2019-04-29T18:05:03Z', 'status': 'True', 'type': 'Initialized'}, {'lastProbeTime': None, 'lastTransitionTime': '2019-04-29T18:03:58Z', 'message': 'containers with unready status: [matrix-ana-reportjobs-api]', 'reason': 'ContainersNotReady', 'status': 'False', 'type': 'Ready'}, {'lastProbeTime': None, 'lastTransitionTime': '2019-04-29T18:03:58Z', 'message': 'containers with unready status: [matrix-ana-reportjobs-api]', 'reason': 'ContainersNotReady', 'status': 'False', 'type': 'ContainersReady'}, {'lastProbeTime': None, 'lastTransitionTime': '2019-04-29T18:03:58Z', 'status': 'True', 'type': 'PodScheduled'}], 'containerStatuses': [{'image': 'datawire/telepresence-k8s-priv:0.97', 'imageID': '', 'lastState': {}, 'name': 'matrix-ana-reportjobs-api', 'ready': False, 'restartCount': 0, 'state': {'waiting': {'reason': 'PodInitializing'}}}], 'hostIP': '10.128.0.75', 'initContainerStatuses': [{'containerID': 'docker://a890ed540dfedd4ce38c6b40169a973e15cc0bcdcc2ca53c648460bd14a74678', 'image': 'postgres:11.2', 'imageID': 'docker-pullable://postgres@sha256:e002abfaf7b0f1015b7510f66b709e2bacbecc7762c428634c27a3c5e58ffbae', 'lastState': {}, 'name': 'db-ready', 'ready': True, 'restartCount': 0, 'state': {'terminated': {'containerID': 'docker://a890ed540dfedd4ce38c6b40169a973e15cc0bcdcc2ca53c648460bd14a74678', 'exitCode': 0, 'finishedAt': '2019-04-29T18:05:03Z', 'reason': 'Completed', 'startedAt': '2019-04-29T18:05:03Z'}}}], 'phase': 'Pending', 'podIP': '10.76.3.112', 'qosClass': 'Burstable', 'startTime': '2019-04-29T18:03:58Z'}

Traceback (most recent call last):


 "<console>", line 1, in <module>


 "scripts/pipeline/interactive.py", line 81, in process

question, candidates, top_n, n_docs, return_context=True


 "/home/ubuntu/DrQA/drqa/pipeline/drqa.py", line 190, in process

top_n, n_docs, return_context


 "/home/ubuntu/DrQA/drqa/pipeline/drqa.py", line 270, in process_batch

handle = self.reader.predict(batch, async_pool=self.processes)


 "/home/ubuntu/DrQA/drqa/reader/model.py", line 290, in predict

score_s, score_e = self.network(*inputs)


 "/home/ubuntu/anaconda3/envs/pytorch_p36/lib/python3.6/site-packages/torch/nn/modules/module.py", line 477, in __call__

result = self.forward(*input, **kwargs)


 "/home/ubuntu/DrQA/drqa/reader/rnn_reader.py", line 122, in forward

doc_hiddens = self.doc_rnn(torch.cat(drnn_input, 2), x1_mask)


 "/home/ubuntu/anaconda3/envs/pytorch_p36/lib/python3.6/site-packages/torch/nn/modules/module.py", line 477, in __call__

result = self.forward(*input, **kwargs)


 "/home/ubuntu/DrQA/drqa/reader/layers.py", line 60, in forward

output = self._forward_padded(x, x_mask)


 "/home/ubuntu/DrQA/drqa/reader/layers.py", line 141, in _forward_padded

output = torch.cat(outputs[1:], 2)
RuntimeError: CUDA error: out of memory
