tensor2tensor/trax/inputs.py
def shuffle_and_batch_data(dataset, target_names, features_info, training):
  """Shuffle and batch the given dataset."""
  def append_targets(example):
    """Append targets to the example dictionary. Needed for Keras."""
    if len(target_names) == 1:
      return (example, example[target_names[0]])
    targets = {}
    for name in target_names:
      targets[name] = example[name]
    return (example, targets)
  dataset = dataset.map(append_targets)
  if training:
    dataset = dataset.repeat()
    # Skip a random fraction at the beginning of the stream.  The skip is
    # essential for synchronous highly-parallel training to avoid multiple
    # replicas reading the same data in lock-step.
    dataset = dataset.skip(random.randint(0, _MAX_SKIP_EXAMPLES))
  shapes = {k: features_info[k].shape for k in features_info}
  shapes = (shapes, shapes[target_names[0]])
  dataset = dataset.shuffle(1024)
  dataset = preprocess_fun(dataset, training)
  dataset = batch_fun(dataset, training, shapes, target_names)
  return dataset.prefetch(2)
Shuffle and batch the given dataset.
Shuffle and batch the given dataset.
Shuffle and batch the given dataset .
def shuffle_and_batch_data(dataset, target_names, features_info, training):
  def append_targets(example):
    if len(target_names) == 1:
      return (example, example[target_names[0]])
    targets = {}
    for name in target_names:
      targets[name] = example[name]
    return (example, targets)
  dataset = dataset.map(append_targets)
  if training:
    dataset = dataset.repeat()
    dataset = dataset.skip(random.randint(0, _MAX_SKIP_EXAMPLES))
  shapes = {k: features_info[k].shape for k in features_info}
  shapes = (shapes, shapes[target_names[0]])
  dataset = dataset.shuffle(1024)
  dataset = preprocess_fun(dataset, training)
  dataset = batch_fun(dataset, training, shapes, target_names)
  return dataset.prefetch(2)
def shuffle_and_batch_data ( dataset , target_names , features_info , training ) : def append_targets ( example ) : """Append targets to the example dictionary. Needed for Keras.""" if len ( target_names ) == 1 : return ( example , example [ target_names [ 0 ] ] ) targets = { } for name in target_names : targets [ name ] = example [ name ] return ( example , targets ) dataset = dataset . map ( append_targets ) if training : dataset = dataset . repeat ( ) dataset = dataset . skip ( random . randint ( 0 , _MAX_SKIP_EXAMPLES ) ) shapes = { k : features_info [ k ] . shape for k in features_info } shapes = ( shapes , shapes [ target_names [ 0 ] ] ) dataset = dataset . shuffle ( 1024 ) dataset = preprocess_fun ( dataset , training ) dataset = batch_fun ( dataset , training , shapes , target_names ) return dataset . prefetch ( 2 )
Shuffle and batch the given dataset.
Shuffle and batch the given dataset .
"""Shuffle and batch the given dataset.""" """Append targets to the example dictionary. Needed for Keras.""" # Skip a random fraction at the beginning of the stream.  The skip is # essential for synchronous highly-parallel training to avoid multiple # replicas reading the same data in lock-step.
dataset
target_names
features_info
training