code: def pretokenized_tfrecord_dataset(
      filenames,
      text2self,
      eos_included,
      repeat,
      batch_size,
      sequence_length,
      vocab_shift=0,
  ):
      """Reads tensor2tensor-style data files.
  
      The dataset is defined by sets of TFRecord files of TFExample protos.
      There should be a "targets" feature (a 1d tensor of integers)
      If not text2self, there should also be an "inputs" feature.
      Other features get ignored.
  
      eos_included specifies whether the inputs and targets were written with an
      EOS token, as in tensor2tensor
  
      Args:
        filenames: a list of strings
        text2self: a boolean
        eos_included: a boolean
        repeat: a boolean
        batch_size: an integer, DEPRECATED
        sequence_length: an integer
        vocab_shift: an optional integer - add this value to all ids
      Returns:
        A tf.data.Dataset of batches
      """
      del batch_size
      dataset = tf.data.TFRecordDataset(
          filenames, buffer_size=64 * 1024 * 1024
      )
      if repeat:
          dataset = dataset.repeat()
      keys = ["targets"] if text2self else ["inputs", "targets"]
  
      def decode_example(serialized_example):
          """Return a dict of Tensors from a serialized tensorflow.Example."""
          decoded = tf.io.parse_example(
              serialized=[serialized_example],
              features={k: tf.VarLenFeature(tf.int64) for k in keys},
          )
          decoded = {k: v.values for k, v in decoded.items()}
          if vocab_shift:
              decoded = {k: v + vocab_shift for k, v in decoded.items()}
          if not eos_included:
              decoded = {
                  k: tf.concat([v, [1]], 0) for k, v in decoded.items()
              }
          return decoded
  
      dataset = dataset.map(
          decode_example,
          num_parallel_calls=tf.data.experimental.AUTOTUNE,
      )
      return pack_or_pad(dataset, sequence_length)
  
imports: from __future__ import absolute_import
  from __future__ import division
  from __future__ import print_function
  import collections
  import os
  import gin
  import tensorflow.compat.v1
  import tensorflow_datasets
label_desc: Use image node_id 12 for calling a global function with example usage: pretokenized_tfrecord_dataset(filenames, text2eos_included, repeat, batch_size, sequence_length, vocab_shift) and returns: pack_or_pad, decoded