code: # Copyright 2018 The TensorFlow Authors. All Rights Reserved.
  #
  # Licensed under the Apache License, Version 2.0 (the "License");
  # you may not use this file except in compliance with the License.
  # You may obtain a copy of the License at
  #
  #     http://www.apache.org/licenses/LICENSE-2.0
  #
  # Unless required by applicable law or agreed to in writing, software
  # distributed under the License is distributed on an "AS IS" BASIS,
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  # See the License for the specific language governing permissions and
  # limitations under the License.
  # ==============================================================================
  """Experimental API for controlling threading in `tf.data` pipelines."""
  from __future__ import absolute_import
  from __future__ import division
  from __future__ import print_function
  
  import threading
  
  from tensorflow.python.data.ops import dataset_ops
  from tensorflow.python.eager import context
  from tensorflow.python.ops import gen_experimental_dataset_ops as ged_ops
  from tensorflow.python.ops import resource_variable_ops
  
  _uid_counter = 0
  _uid_lock = threading.Lock()
  
  
  def _generate_shared_name(prefix):
    with _uid_lock:
      global _uid_counter
      uid = _uid_counter
      _uid_counter += 1
    return "{}{}".format(prefix, uid)
  
  
  # TODO(b/73383364): Properly export in the `tf.contrib.data` API when stable
  # or make private / remove.
  class PrivateThreadPool(object):
    """A stateful resource that represents a private thread pool."""
  
    def __init__(self, num_threads, display_name=None,
                 max_intra_op_parallelism=1):
      """Creates a `PrivateThreadPool` with the given number of threads."""
      if context.executing_eagerly():
        shared_name = _generate_shared_name("privatethreadpool")
        self._resource = ged_ops.experimental_thread_pool_handle(
            num_threads=num_threads,
            max_intra_op_parallelism=max_intra_op_parallelism,
            display_name=display_name,
            shared_name=shared_name)
        self._resource_deleter = resource_variable_ops.EagerResourceDeleter(
            handle=self._resource, handle_device=context.context().device_name)
      else:
        self._resource = ged_ops.experimental_thread_pool_handle(
            num_threads=num_threads,
            max_intra_op_parallelism=max_intra_op_parallelism,
            display_name=display_name)
  
  
  class _ThreadPoolDataset(dataset_ops.UnaryDataset):
    """A `Dataset` that acts as an identity, and sets a custom threadpool."""
  
    def __init__(self, input_dataset, thread_pool):
      super(_ThreadPoolDataset, self).__init__(input_dataset)
      self._input_dataset = input_dataset
      self._thread_pool = thread_pool
  
    def _as_variant_tensor(self):
      return ged_ops.experimental_thread_pool_dataset(
          self._input_dataset._as_variant_tensor(),  # pylint: disable=protected-access
          self._thread_pool._resource,  # pylint: disable=protected-access
          **dataset_ops.flat_structure(self))
  
    @property
    def output_shapes(self):
      return self._input_dataset.output_shapes
  
    @property
    def output_types(self):
      return self._input_dataset.output_types
  
    @property
    def output_classes(self):
      return self._input_dataset.output_classes
  
  
  # TODO(b/73383364): Properly export in the `tf.contrib.data` API when stable
  # or make private / remove.
  def override_threadpool(dataset, thread_pool):
    """Returns a new dataset that uses the given thread pool for its operations.
  
    Args:
      dataset: A `tf.data.Dataset` object.
      thread_pool: A `PrivateThreadPool` object.
  
    Returns:
      A dataset containing the same values as `dataset`, but which uses
      `thread_pool` to compute any of its parallel operations (such as
      `tf.data.Dataset.map`).
    """
    return _ThreadPoolDataset(dataset, thread_pool)
   # Copyright 2018 The TensorFlow Authors. All Rights Reserved.
  #
  # Licensed under the Apache License, Version 2.0 (the "License");
  # you may not use this file except in compliance with the License.
  # You may obtain a copy of the License at
  #
  #     http://www.apache.org/licenses/LICENSE-2.0
  #
  # Unless required by applicable law or agreed to in writing, software
  # distributed under the License is distributed on an "AS IS" BASIS,
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  # See the License for the specific language governing permissions and
  # limitations under the License.
  # ==============================================================================
  """Tests for the MapParallelization optimization."""
  from __future__ import absolute_import
  from __future__ import division
  from __future__ import print_function
  
  from absl.testing import parameterized
  
  from tensorflow.contrib.data.python.ops import optimization
  from tensorflow.python.data.kernel_tests import test_base
  from tensorflow.python.data.ops import dataset_ops
  from tensorflow.python.framework import dtypes
  from tensorflow.python.framework import errors
  from tensorflow.python.framework import ops
  from tensorflow.python.ops import control_flow_ops
  from tensorflow.python.ops import math_ops
  from tensorflow.python.ops import random_ops
  from tensorflow.python.platform import test
  
  
  class MapParallelizationTest(test_base.DatasetTestBase, parameterized.TestCase):
  
    @staticmethod
    def map_functions():
      identity = lambda x: x
      increment = lambda x: x + 1
  
      def assert_greater(x):
        assert_op = control_flow_ops.Assert(math_ops.greater(x, -1), [x])
        with ops.control_dependencies([assert_op]):
          return x
  
      def random(_):
        return random_ops.random_uniform([],
                                         minval=0,
                                         maxval=10,
                                         dtype=dtypes.int64,
                                         seed=42)
  
      def assert_with_random(x):
        x = assert_greater(x)
        return random(x)
  
      return (("Identity", identity, True), ("Increment", increment, True),
              ("AssertGreater", assert_greater, True), ("Random", random, False),
              ("AssertWithRandom", assert_with_random, False))
  
    @parameterized.named_parameters(*map_functions.__func__())
    def testMapParallelization(self, function, should_optimize):
      next_nodes = ["ParallelMap"] if should_optimize else ["Map"]
      dataset = dataset_ops.Dataset.range(5).apply(
          optimization.assert_next(next_nodes)).map(function).apply(
              optimization.optimize(["map_parallelization"]))
      iterator = dataset.make_one_shot_iterator()
      get_next = iterator.get_next()
  
      with self.test_session() as sess:
        for x in range(5):
          result = sess.run(get_next)
          # No need to run the pipeline if it was not optimized.  Also the results
          # might be hard to check because of random.
          if not should_optimize:
            return
          r = function(x)
          self.assertAllEqual(r, result)
  
        with self.assertRaises(errors.OutOfRangeError):
          sess.run(get_next)
  
  
  if __name__ == "__main__":
    test.main()
  
apis: tensorflow.python.eager.context.executing_eagerly tensorflow.python.data.ops.dataset_ops.flat_structure tensorflow.python.ops.gen_experimental_dataset_ops.experimental_thread_pool_handle tensorflow.python.eager.context.context
apis: tensorflow.contrib.data.python.ops.optimization.optimize tensorflow.contrib.data.python.ops.optimization.assert_next tensorflow.python.ops.math_ops.greater tensorflow.python.data.ops.dataset_ops.Dataset.range tensorflow.python.platform.test.main tensorflow.python.framework.ops.control_dependencies tensorflow.python.ops.random_ops.random_uniform