<p dir="auto"><strong>System information</strong></p>
<ul dir="auto">
<li>Have I written custom code (as opposed to using a stock example script provided in TensorFlow): Yes</li>
<li>OS Platform and Distribution (e.g., Linux Ubuntu 16.04): Linux Fedora 22</li>
<li>Mobile device (e.g. iPhone 8, Pixel 2, Samsung Galaxy) if the issue happens on mobile device: NA</li>
<li>TensorFlow installed from (source or binary): pip</li>
<li>TensorFlow version (use command below): v1.12.0-0-ga6d8ffae09 1.12.0</li>
<li>Python version: 3.6.5</li>
<li>Bazel version (if compiling from source): NA</li>
<li>GCC/Compiler version (if compiling from source): NA</li>
<li>CUDA/cuDNN version: NA</li>
<li>GPU model and memory: NA</li>
</ul>
<p dir="auto"><strong>Describe the current behavior</strong><br>
The test program throws an exception <code class="notranslate">tensorflow.python.framework.errors_impl.InvalidArgumentError: Retval[0] does not have value</code> in session run. However, it completes normally if <code class="notranslate">EPISODE_LENGTH</code> is halved, the <code class="notranslate">SimpleRNN</code> layer is commented out or the <code class="notranslate">batch_normalization</code> layer is commented out. The problem also goes away if I replace the <code class="notranslate">RandomDataset</code>-derived dataset with one built with <code class="notranslate">from_tensor_slices</code> from constant data.</p>
<p dir="auto"><strong>Describe the expected behavior</strong><br>
The test program should complete normally. I don't think my program has a bug that is responsible for the exception but even if it does, the throwing of the exception should be consistent and not depend on <code class="notranslate">EPISODE_LENGTH</code> being big enough.</p>
<p dir="auto"><strong>Code to reproduce the issue</strong></p>
<div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="#!/usr/bin/env python3

import numpy as np
import tensorflow as tf
import tensorflow.keras.layers as kl

NUM_FEATURES = 2
EPISODE_LENGTH = 32
BATCH_SIZE = 2

class GraphBuilder:
    def __init__(self):
        g = tf.Graph()
        with g.as_default():
            tf.set_random_seed(0)

            with tf.variable_scope('input'):
                ds = tf.data.Dataset.from_tensors(
                    (np.zeros(shape=(BATCH_SIZE, EPISODE_LENGTH, NUM_FEATURES), dtype=np.float32),
                     np.zeros(shape=(BATCH_SIZE, EPISODE_LENGTH), dtype=np.float32)))
                it = ds.make_one_shot_iterator()
                batch = it.get_next(name='batch')
                batch_x, batch_y = batch
            with tf.variable_scope('nn'):
                layer = batch_x
                layer = kl.SimpleRNN(1, return_sequences=True)(layer, training=True)
                layer = tf.layers.batch_normalization(layer, axis=2, training=True)
                batch_pred = layer[:, :, 0]

            with tf.variable_scope('loss'):
                num_rows = tf.reduce_prod(tf.shape(batch_y))
                e = tf.subtract(batch_y, batch_pred, name='error')
                sse = tf.reduce_sum(tf.square(e), name='sse')
                mse = tf.divide(sse, tf.cast(num_rows, tf.float32), name='mse')
                rmse = tf.sqrt(mse, name='rmse')

            with tf.variable_scope('train'):
                optimizer = tf.train.AdamOptimizer(0.001)
                update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
                with tf.control_dependencies(update_ops):
                    train_op = optimizer.minimize(mse)

        self.rmse = rmse
        self.train_op = train_op
        self.graph = g


def main():
    gb = GraphBuilder()
    config = tf.ConfigProto(intra_op_parallelism_threads=2, inter_op_parallelism_threads=2)
    config.device_count['GPU'] = 0
    with tf.Session(graph=gb.graph, config=config) as sess:
        sess.run(tf.global_variables_initializer())
        sess.run([gb.rmse, gb.train_op])

main()"><pre class="notranslate"><code class="notranslate">#!/usr/bin/env python3

import numpy as np
import tensorflow as tf
import tensorflow.keras.layers as kl

NUM_FEATURES = 2
EPISODE_LENGTH = 32
BATCH_SIZE = 2

class GraphBuilder:
    def __init__(self):
        g = tf.Graph()
        with g.as_default():
            tf.set_random_seed(0)

            with tf.variable_scope('input'):
                ds = tf.data.Dataset.from_tensors(
                    (np.zeros(shape=(BATCH_SIZE, EPISODE_LENGTH, NUM_FEATURES), dtype=np.float32),
                     np.zeros(shape=(BATCH_SIZE, EPISODE_LENGTH), dtype=np.float32)))
                it = ds.make_one_shot_iterator()
                batch = it.get_next(name='batch')
                batch_x, batch_y = batch
            with tf.variable_scope('nn'):
                layer = batch_x
                layer = kl.SimpleRNN(1, return_sequences=True)(layer, training=True)
                layer = tf.layers.batch_normalization(layer, axis=2, training=True)
                batch_pred = layer[:, :, 0]

            with tf.variable_scope('loss'):
                num_rows = tf.reduce_prod(tf.shape(batch_y))
                e = tf.subtract(batch_y, batch_pred, name='error')
                sse = tf.reduce_sum(tf.square(e), name='sse')
                mse = tf.divide(sse, tf.cast(num_rows, tf.float32), name='mse')
                rmse = tf.sqrt(mse, name='rmse')

            with tf.variable_scope('train'):
                optimizer = tf.train.AdamOptimizer(0.001)
                update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
                with tf.control_dependencies(update_ops):
                    train_op = optimizer.minimize(mse)

        self.rmse = rmse
        self.train_op = train_op
        self.graph = g


def main():
    gb = GraphBuilder()
    config = tf.ConfigProto(intra_op_parallelism_threads=2, inter_op_parallelism_threads=2)
    config.device_count['GPU'] = 0
    with tf.Session(graph=gb.graph, config=config) as sess:
        sess.run(tf.global_variables_initializer())
        sess.run([gb.rmse, gb.train_op])

main()
</code></pre></div>
<p dir="auto"><strong>Other info / logs</strong></p>
<div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="2018-11-20 13:31:56.998664: I tensorflow/core/platform/cpu_feature_guard.cc:141] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2 FMA
Traceback (most recent call last):
  File &quot;/data/jchia/venv/tf/lib/python3.6/site-packages/tensorflow/python/client/session.py&quot;, line 1334, in _do_call
    return fn(*args)
  File &quot;/data/jchia/venv/tf/lib/python3.6/site-packages/tensorflow/python/client/session.py&quot;, line 1319, in _run_fn
    options, feed_dict, fetch_list, target_list, run_metadata)
  File &quot;/data/jchia/venv/tf/lib/python3.6/site-packages/tensorflow/python/client/session.py&quot;, line 1407, in _call_tf_sessionrun
    run_metadata)
tensorflow.python.framework.errors_impl.InvalidArgumentError: Retval[0] does not have value

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File &quot;py3/train2.py&quot;, line 56, in &lt;module&gt;
    main()
  File &quot;py3/train2.py&quot;, line 54, in main
    sess.run([gb.rmse, gb.train_op])
  File &quot;/data/jchia/venv/tf/lib/python3.6/site-packages/tensorflow/python/client/session.py&quot;, line 929, in run
    run_metadata_ptr)
  File &quot;/data/jchia/venv/tf/lib/python3.6/site-packages/tensorflow/python/client/session.py&quot;, line 1152, in _run
    feed_dict_tensor, options, run_metadata)
  File &quot;/data/jchia/venv/tf/lib/python3.6/site-packages/tensorflow/python/client/session.py&quot;, line 1328, in _do_run
    run_metadata)
  File &quot;/data/jchia/venv/tf/lib/python3.6/site-packages/tensorflow/python/client/session.py&quot;, line 1348, in _do_call
    raise type(e)(node_def, op, message)
tensorflow.python.framework.errors_impl.InvalidArgumentError: Retval[0] does not have value"><pre class="notranslate"><code class="notranslate">2018-11-20 13:31:56.998664: I tensorflow/core/platform/cpu_feature_guard.cc:141] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2 FMA
Traceback (most recent call last):
  File "/data/jchia/venv/tf/lib/python3.6/site-packages/tensorflow/python/client/session.py", line 1334, in _do_call
    return fn(*args)
  File "/data/jchia/venv/tf/lib/python3.6/site-packages/tensorflow/python/client/session.py", line 1319, in _run_fn
    options, feed_dict, fetch_list, target_list, run_metadata)
  File "/data/jchia/venv/tf/lib/python3.6/site-packages/tensorflow/python/client/session.py", line 1407, in _call_tf_sessionrun
    run_metadata)
tensorflow.python.framework.errors_impl.InvalidArgumentError: Retval[0] does not have value

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "py3/train2.py", line 56, in &lt;module&gt;
    main()
  File "py3/train2.py", line 54, in main
    sess.run([gb.rmse, gb.train_op])
  File "/data/jchia/venv/tf/lib/python3.6/site-packages/tensorflow/python/client/session.py", line 929, in run
    run_metadata_ptr)
  File "/data/jchia/venv/tf/lib/python3.6/site-packages/tensorflow/python/client/session.py", line 1152, in _run
    feed_dict_tensor, options, run_metadata)
  File "/data/jchia/venv/tf/lib/python3.6/site-packages/tensorflow/python/client/session.py", line 1328, in _do_run
    run_metadata)
  File "/data/jchia/venv/tf/lib/python3.6/site-packages/tensorflow/python/client/session.py", line 1348, in _do_call
    raise type(e)(node_def, op, message)
tensorflow.python.framework.errors_impl.InvalidArgumentError: Retval[0] does not have value
</code></pre></div>
<h3 dir="auto">System information</h3>
<ul dir="auto">
<li><strong>Have I written custom code (as opposed to using a stock example script provided in TensorFlow)</strong>: Yes</li>
<li><strong>OS Platform and Distribution (e.g., Linux Ubuntu 16.04)</strong>: Linux Ubuntu 17.10</li>
<li><strong>TensorFlow installed from (source or binary)</strong>: Binary (pip install tensorflow)</li>
<li><strong>TensorFlow version (use command below)</strong>: v1.4.0-rc1-11-g130a514 1.4.0</li>
<li><strong>Python version</strong>: 3.6.3</li>
<li><strong>CUDA/cuDNN version</strong>: N/A</li>
<li><strong>GPU model and memory</strong>: N/A</li>
<li><strong>Exact command to reproduce</strong>: (see the "Source code" section)</li>
</ul>
<h3 dir="auto">Describe the problem</h3>
<p dir="auto">I was following the instructions for updating <code class="notranslate">moving_mean</code> and <code class="notranslate">moving_variance</code> by using the code snippet provided in <a href="https://www.tensorflow.org/api_docs/python/tf/layers/batch_normalization" rel="nofollow">batch_normalization documentation</a> and it resulted in an "tensorflow.python.framework.errors_impl.InvalidArgumentError: Retval[0] does not have value" exception.</p>
<h3 dir="auto">Source code / logs</h3>
<p dir="auto">Full instructions to reproduce:</p>
<div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="git clone -b control-dependencies-exc https://github.com/naktinis/language-id.git ctrl-dep-exc
cd ctrl-dep-exc/
python3 -m venv venv
. venv/bin/activate
pip install tensorflow==1.4.0 Pillow
python3 main.py --image-dir test_data/ --label-file test_data/labels.csv --model rnn"><pre class="notranslate"><code class="notranslate">git clone -b control-dependencies-exc https://github.com/naktinis/language-id.git ctrl-dep-exc
cd ctrl-dep-exc/
python3 -m venv venv
. venv/bin/activate
pip install tensorflow==1.4.0 Pillow
python3 main.py --image-dir test_data/ --label-file test_data/labels.csv --model rnn
</code></pre></div>
<p dir="auto">The specific code change that was enough to produce the exception (seems to match the snippet in the official <a href="https://www.tensorflow.org/api_docs/python/tf/layers/batch_normalization" rel="nofollow">documentation</a>):<br>
<a class="commit-link" data-hovercard-type="commit" data-hovercard-url="https://github.com/naktinis/language-id/commit/50740f/hovercard" href="https://github.com/naktinis/language-id/commit/50740f">naktinis/language-id@<tt>50740f</tt></a></p>
<p dir="auto">Full exception:</p>
<div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="Traceback (most recent call last):
  File &quot;main.py&quot;, line 173, in &lt;module&gt;
    tf.app.run(main=run_experiment)
  File &quot;/home/naktinis/code/ctrl-dep-exc/venv/lib/python3.6/site-packages/tensorflow/python/platform/app.py&quot;, line 48, in run
    _sys.exit(main(_sys.argv[:1] + flags_passthrough))
  File &quot;main.py&quot;, line 165, in run_experiment
    hparams=params,
  File &quot;/home/naktinis/code/ctrl-dep-exc/venv/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/learn_runner.py&quot;, line 218, in run
    return _execute_schedule(experiment, schedule)
  File &quot;/home/naktinis/code/ctrl-dep-exc/venv/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/learn_runner.py&quot;, line 46, in _execute_schedule
    return task()
  File &quot;/home/naktinis/code/ctrl-dep-exc/venv/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/experiment.py&quot;, line 625, in train_and_evaluate
    self.train(delay_secs=0)
  File &quot;/home/naktinis/code/ctrl-dep-exc/venv/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/experiment.py&quot;, line 367, in train
    hooks=self._train_monitors + extra_hooks)
  File &quot;/home/naktinis/code/ctrl-dep-exc/venv/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/experiment.py&quot;, line 807, in _call_train
    hooks=hooks)
  File &quot;/home/naktinis/code/ctrl-dep-exc/venv/lib/python3.6/site-packages/tensorflow/python/estimator/estimator.py&quot;, line 302, in train
    loss = self._train_model(input_fn, hooks, saving_listeners)
  File &quot;/home/naktinis/code/ctrl-dep-exc/venv/lib/python3.6/site-packages/tensorflow/python/estimator/estimator.py&quot;, line 783, in _train_model
    _, loss = mon_sess.run([estimator_spec.train_op, estimator_spec.loss])
  File &quot;/home/naktinis/code/ctrl-dep-exc/venv/lib/python3.6/site-packages/tensorflow/python/training/monitored_session.py&quot;, line 521, in run
    run_metadata=run_metadata)
  File &quot;/home/naktinis/code/ctrl-dep-exc/venv/lib/python3.6/site-packages/tensorflow/python/training/monitored_session.py&quot;, line 892, in run
    run_metadata=run_metadata)
  File &quot;/home/naktinis/code/ctrl-dep-exc/venv/lib/python3.6/site-packages/tensorflow/python/training/monitored_session.py&quot;, line 967, in run
    raise six.reraise(*original_exc_info)
  File &quot;/home/naktinis/code/ctrl-dep-exc/venv/lib/python3.6/site-packages/six.py&quot;, line 693, in reraise
    raise value
  File &quot;/home/naktinis/code/ctrl-dep-exc/venv/lib/python3.6/site-packages/tensorflow/python/training/monitored_session.py&quot;, line 952, in run
    return self._sess.run(*args, **kwargs)
  File &quot;/home/naktinis/code/ctrl-dep-exc/venv/lib/python3.6/site-packages/tensorflow/python/training/monitored_session.py&quot;, line 1024, in run
    run_metadata=run_metadata)
  File &quot;/home/naktinis/code/ctrl-dep-exc/venv/lib/python3.6/site-packages/tensorflow/python/training/monitored_session.py&quot;, line 827, in run
    return self._sess.run(*args, **kwargs)
  File &quot;/home/naktinis/code/ctrl-dep-exc/venv/lib/python3.6/site-packages/tensorflow/python/client/session.py&quot;, line 889, in run
    run_metadata_ptr)
  File &quot;/home/naktinis/code/ctrl-dep-exc/venv/lib/python3.6/site-packages/tensorflow/python/client/session.py&quot;, line 1120, in _run
    feed_dict_tensor, options, run_metadata)
  File &quot;/home/naktinis/code/ctrl-dep-exc/venv/lib/python3.6/site-packages/tensorflow/python/client/session.py&quot;, line 1317, in _do_run
    options, run_metadata)
  File &quot;/home/naktinis/code/ctrl-dep-exc/venv/lib/python3.6/site-packages/tensorflow/python/client/session.py&quot;, line 1336, in _do_call
    raise type(e)(node_def, op, message)
tensorflow.python.framework.errors_impl.InvalidArgumentError: Retval[0] does not have value
"><pre class="notranslate"><code class="notranslate">Traceback (most recent call last):
  File "main.py", line 173, in &lt;module&gt;
    tf.app.run(main=run_experiment)
  File "/home/naktinis/code/ctrl-dep-exc/venv/lib/python3.6/site-packages/tensorflow/python/platform/app.py", line 48, in run
    _sys.exit(main(_sys.argv[:1] + flags_passthrough))
  File "main.py", line 165, in run_experiment
    hparams=params,
  File "/home/naktinis/code/ctrl-dep-exc/venv/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/learn_runner.py", line 218, in run
    return _execute_schedule(experiment, schedule)
  File "/home/naktinis/code/ctrl-dep-exc/venv/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/learn_runner.py", line 46, in _execute_schedule
    return task()
  File "/home/naktinis/code/ctrl-dep-exc/venv/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/experiment.py", line 625, in train_and_evaluate
    self.train(delay_secs=0)
  File "/home/naktinis/code/ctrl-dep-exc/venv/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/experiment.py", line 367, in train
    hooks=self._train_monitors + extra_hooks)
  File "/home/naktinis/code/ctrl-dep-exc/venv/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/experiment.py", line 807, in _call_train
    hooks=hooks)
  File "/home/naktinis/code/ctrl-dep-exc/venv/lib/python3.6/site-packages/tensorflow/python/estimator/estimator.py", line 302, in train
    loss = self._train_model(input_fn, hooks, saving_listeners)
  File "/home/naktinis/code/ctrl-dep-exc/venv/lib/python3.6/site-packages/tensorflow/python/estimator/estimator.py", line 783, in _train_model
    _, loss = mon_sess.run([estimator_spec.train_op, estimator_spec.loss])
  File "/home/naktinis/code/ctrl-dep-exc/venv/lib/python3.6/site-packages/tensorflow/python/training/monitored_session.py", line 521, in run
    run_metadata=run_metadata)
  File "/home/naktinis/code/ctrl-dep-exc/venv/lib/python3.6/site-packages/tensorflow/python/training/monitored_session.py", line 892, in run
    run_metadata=run_metadata)
  File "/home/naktinis/code/ctrl-dep-exc/venv/lib/python3.6/site-packages/tensorflow/python/training/monitored_session.py", line 967, in run
    raise six.reraise(*original_exc_info)
  File "/home/naktinis/code/ctrl-dep-exc/venv/lib/python3.6/site-packages/six.py", line 693, in reraise
    raise value
  File "/home/naktinis/code/ctrl-dep-exc/venv/lib/python3.6/site-packages/tensorflow/python/training/monitored_session.py", line 952, in run
    return self._sess.run(*args, **kwargs)
  File "/home/naktinis/code/ctrl-dep-exc/venv/lib/python3.6/site-packages/tensorflow/python/training/monitored_session.py", line 1024, in run
    run_metadata=run_metadata)
  File "/home/naktinis/code/ctrl-dep-exc/venv/lib/python3.6/site-packages/tensorflow/python/training/monitored_session.py", line 827, in run
    return self._sess.run(*args, **kwargs)
  File "/home/naktinis/code/ctrl-dep-exc/venv/lib/python3.6/site-packages/tensorflow/python/client/session.py", line 889, in run
    run_metadata_ptr)
  File "/home/naktinis/code/ctrl-dep-exc/venv/lib/python3.6/site-packages/tensorflow/python/client/session.py", line 1120, in _run
    feed_dict_tensor, options, run_metadata)
  File "/home/naktinis/code/ctrl-dep-exc/venv/lib/python3.6/site-packages/tensorflow/python/client/session.py", line 1317, in _do_run
    options, run_metadata)
  File "/home/naktinis/code/ctrl-dep-exc/venv/lib/python3.6/site-packages/tensorflow/python/client/session.py", line 1336, in _do_call
    raise type(e)(node_def, op, message)
tensorflow.python.framework.errors_impl.InvalidArgumentError: Retval[0] does not have value

</code></pre></div>