Traceback (most recent call last):


 "train.py", line 251, in <module>

_main_(args)


 "train.py", line 210, in _main_

max_queue_size
= 8


 "/usr/local/lib/python3.6/dist-packages/keras/legacy/interfaces.py", line 91, in wrapper

return func(*args, **kwargs)


 "/usr/local/lib/python3.6/dist-packages/keras/engine/training.py", line 2224, in fit_generator

class_weight=class_weight)


 "/usr/local/lib/python3.6/dist-packages/keras/engine/training.py", line 1883, in train_on_batch

outputs = self.train_function(ins)


 "/usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py", line 2478, in __call__

**self.session_kwargs)


 "/usr/local/lib/python3.6/dist-packages/tensorflow/python/client/session.py", line 905, in run

run_metadata_ptr)


 "/usr/local/lib/python3.6/dist-packages/tensorflow/python/client/session.py", line 1137, in _run

feed_dict_tensor, options, run_metadata)


 "/usr/local/lib/python3.6/dist-packages/tensorflow/python/client/session.py", line 1355, in _do_run

options, run_metadata)


 "/usr/local/lib/python3.6/dist-packages/tensorflow/python/client/session.py", line 1374, in _do_call

raise type(e)(node_def, op, message)
tensorflow.python.framework.errors_impl.ResourceExhaustedError: OOM when allocating tensor with shape[8,13,13,1024] and type float on /job:localhost/replica:0/task:0/device:GPU:0 by allocator GPU_0_bfc

Traceback (most recent call last):


 "test/contrib/kubernetes_test.py", line 84, in test_fail_job

self.assertRaises(RuntimeError, fail.run)
AssertionError: Pod fail-20191028221346-eb2a7b33c484458b-6qjz6 ContainerCannotRun (exit code 128). Logs: `kubectl logs pod/fail-20191028221346-eb2a7b33c484458b-6qjz6`
