code: def image_set(
      filenames,
      batch_size,
      height,
      width,
      training=False,
      distort_color=False,
      num_threads=10,
      nsummary=10,
      deterministic=False,
      use_dali=None,
      idx_filenames=None,
  ):
      if use_dali:
          if idx_filenames is None:
              raise ValueError(
                  "Must provide idx_filenames if Dali is enabled"
              )
  
          preprocessor = DALIPreprocessor(
              filenames,
              idx_filenames,
              height,
              width,
              batch_size,
              num_threads,
              dali_cpu=True if use_dali == "CPU" else False,
              deterministic=deterministic,
              training=training,
          )
          return preprocessor
      else:
          shuffle_buffer_size = 10000
          num_readers = 10
          ds = tf.data.Dataset.from_tensor_slices(filenames)
  
          # AUTOTUNE can give better perf for non-horovod cases
          thread_config = num_threads
  
          # shard should be before any randomizing operations
          if training:
              ds = ds.shard(hvd.size(), hvd.rank())
  
          # read up to num_readers files and interleave their records
          ds = ds.interleave(
              tf.data.TFRecordDataset, cycle_length=num_readers
          )
  
          if training:
              # Improve training performance when training data is in remote storage and
              # can fit into worker memory.
              ds = ds.cache()
  
          if training:
              # shuffle data before repeating to respect epoch boundaries
              ds = ds.shuffle(shuffle_buffer_size)
              ds = ds.repeat()
  
          preproc_func = (
              lambda record: _parse_and_preprocess_image_record(
                  record,
                  height,
                  width,
                  deterministic=deterministic,
                  random_crop=training,
                  distort_color=distort_color,
              )
          )
          ds = ds.map(preproc_func, num_parallel_calls=thread_config)
  
          ds = ds.batch(batch_size, drop_remainder=True)
  
          # prefetching
          ds = ds.prefetch(buffer_size=tf.data.experimental.AUTOTUNE)
  
          options = tf.data.Options()
          options.experimental_slack = True
          ds = ds.with_options(options)
  
          return ds
  
imports: import tensorflow
  import sys
  import os
  import numpy
  from subprocess import call
  import horovod.tensorflow.keras
  from nvidia.dali.pipeline import pipeline_def
  import nvidia.dali.fn
  import nvidia.dali.types
  import nvidia.dali.tfrecord
label_desc: Use image node_id 8 for calling a global function with example usage: image_set(filenames, batch_size, height, width, training, distort_color, num_threads, nsummary, deterministic, use_dali, idx_filenames) and returns: preprocessor, ds