baselines/cifar/utils.py
def load_input_fn(split,
                  batch_size,
                  name,
                  use_bfloat16,
                  normalize=True,
                  drop_remainder=True,
                  repeat=False,
                  proportion=1.0,
                  data_dir=None):
  """Loads CIFAR dataset for training or testing.

  Args:
    split: tfds.Split.
    batch_size: The global batch size to use.
    name: A string indicates whether it is cifar10 or cifar100.
    use_bfloat16: data type, bfloat16 precision or float32.
    normalize: Whether to apply mean-std normalization on features.
    drop_remainder: bool.
    repeat: bool.
    proportion: float, the proportion of dataset to be used.
    data_dir: Directory where the dataset is stored to be loaded via tfds.load.
      Optional, useful for loading datasets stored on GCS.

  Returns:
    Input function which returns a locally-sharded dataset batch.
  """
  if use_bfloat16:
    dtype = tf.bfloat16
  else:
    dtype = tf.float32
  ds_info = tfds.builder(name).info
  image_shape = ds_info.features['image'].shape
  dataset_size = ds_info.splits['train'].num_examples

  def preprocess(image, label):
    """Image preprocessing function."""
    if split == tfds.Split.TRAIN:
      image = tf.image.resize_with_crop_or_pad(
          image, image_shape[0] + 4, image_shape[1] + 4)
      image = tf.image.random_crop(image, image_shape)
      image = tf.image.random_flip_left_right(image)

    image = tf.image.convert_image_dtype(image, dtype)
    if normalize:
      mean = tf.constant([0.4914, 0.4822, 0.4465], dtype=dtype)
      std = tf.constant([0.2023, 0.1994, 0.2010], dtype=dtype)
      image = (image - mean) / std
    label = tf.cast(label, dtype)
    return image, label

  def input_fn(ctx=None):
    """Returns a locally sharded (i.e., per-core) dataset batch."""
    if proportion == 1.0:
      dataset = tfds.load(
          name, split=split, data_dir=data_dir, as_supervised=True)
    else:
      new_name = '{}:3.*.*'.format(name)
      if split == tfds.Split.TRAIN:
        # use round instead of floor to resolve bug when e.g. using
        # proportion = 1 - 0.8 = 0.19999999
        new_split = 'train[:{}%]'.format(round(100 * proportion))
      elif split == tfds.Split.VALIDATION:
        new_split = 'train[-{}%:]'.format(round(100 * proportion))
      elif split == tfds.Split.TEST:
        new_split = 'test[:{}%]'.format(round(100 * proportion))
      else:
        raise ValueError('Provide valid split.')
      dataset = tfds.load(new_name, split=new_split, as_supervised=True)
    if split == tfds.Split.TRAIN or repeat:
      dataset = dataset.shuffle(buffer_size=dataset_size).repeat()

    dataset = dataset.map(preprocess,
                          num_parallel_calls=tf.data.experimental.AUTOTUNE)
    dataset = dataset.batch(batch_size, drop_remainder=drop_remainder)
    dataset = dataset.prefetch(tf.data.experimental.AUTOTUNE)
    if ctx and ctx.num_input_pipelines > 1:
      dataset = dataset.shard(ctx.num_input_pipelines, ctx.input_pipeline_id)
    return dataset
  return input_fn
Loads CIFAR dataset for training or testing.

  Args:
    split: tfds.Split.
    batch_size: The global batch size to use.
    name: A string indicates whether it is cifar10 or cifar100.
    use_bfloat16: data type, bfloat16 precision or float32.
    normalize: Whether to apply mean-std normalization on features.
    drop_remainder: bool.
    repeat: bool.
    proportion: float, the proportion of dataset to be used.
    data_dir: Directory where the dataset is stored to be loaded via tfds.load.
      Optional, useful for loading datasets stored on GCS.

  Returns:
    Input function which returns a locally-sharded dataset batch.
  
Loads CIFAR dataset for training or testing.
Loads CIFAR dataset for training or testing .
def load_input_fn(split,
                  batch_size,
                  name,
                  use_bfloat16,
                  normalize=True,
                  drop_remainder=True,
                  repeat=False,
                  proportion=1.0,
                  data_dir=None):
  if use_bfloat16:
    dtype = tf.bfloat16
  else:
    dtype = tf.float32
  ds_info = tfds.builder(name).info
  image_shape = ds_info.features['image'].shape
  dataset_size = ds_info.splits['train'].num_examples
  def preprocess(image, label):
    if split == tfds.Split.TRAIN:
      image = tf.image.resize_with_crop_or_pad(
          image, image_shape[0] + 4, image_shape[1] + 4)
      image = tf.image.random_crop(image, image_shape)
      image = tf.image.random_flip_left_right(image)
    image = tf.image.convert_image_dtype(image, dtype)
    if normalize:
      mean = tf.constant([0.4914, 0.4822, 0.4465], dtype=dtype)
      std = tf.constant([0.2023, 0.1994, 0.2010], dtype=dtype)
      image = (image - mean) / std
    label = tf.cast(label, dtype)
    return image, label
  def input_fn(ctx=None):
    if proportion == 1.0:
      dataset = tfds.load(
          name, split=split, data_dir=data_dir, as_supervised=True)
    else:
      new_name = '{}:3.*.*'.format(name)
      if split == tfds.Split.TRAIN:
        new_split = 'train[:{}%]'.format(round(100 * proportion))
      elif split == tfds.Split.VALIDATION:
        new_split = 'train[-{}%:]'.format(round(100 * proportion))
      elif split == tfds.Split.TEST:
        new_split = 'test[:{}%]'.format(round(100 * proportion))
      else:
        raise ValueError('Provide valid split.')
      dataset = tfds.load(new_name, split=new_split, as_supervised=True)
    if split == tfds.Split.TRAIN or repeat:
      dataset = dataset.shuffle(buffer_size=dataset_size).repeat()
    dataset = dataset.map(preprocess,
                          num_parallel_calls=tf.data.experimental.AUTOTUNE)
    dataset = dataset.batch(batch_size, drop_remainder=drop_remainder)
    dataset = dataset.prefetch(tf.data.experimental.AUTOTUNE)
    if ctx and ctx.num_input_pipelines > 1:
      dataset = dataset.shard(ctx.num_input_pipelines, ctx.input_pipeline_id)
    return dataset
  return input_fn
def load_input_fn ( split , batch_size , name , use_bfloat16 , normalize = True , drop_remainder = True , repeat = False , proportion = 1.0 , data_dir = None ) : if use_bfloat16 : dtype = tf . bfloat16 else : dtype = tf . float32 ds_info = tfds . builder ( name ) . info image_shape = ds_info . features [ 'image' ] . shape dataset_size = ds_info . splits [ 'train' ] . num_examples def preprocess ( image , label ) : """Image preprocessing function.""" if split == tfds . Split . TRAIN : image = tf . image . resize_with_crop_or_pad ( image , image_shape [ 0 ] + 4 , image_shape [ 1 ] + 4 ) image = tf . image . random_crop ( image , image_shape ) image = tf . image . random_flip_left_right ( image ) image = tf . image . convert_image_dtype ( image , dtype ) if normalize : mean = tf . constant ( [ 0.4914 , 0.4822 , 0.4465 ] , dtype = dtype ) std = tf . constant ( [ 0.2023 , 0.1994 , 0.2010 ] , dtype = dtype ) image = ( image - mean ) / std label = tf . cast ( label , dtype ) return image , label def input_fn ( ctx = None ) : """Returns a locally sharded (i.e., per-core) dataset batch.""" if proportion == 1.0 : dataset = tfds . load ( name , split = split , data_dir = data_dir , as_supervised = True ) else : new_name = '{}:3.*.*' . format ( name ) if split == tfds . Split . TRAIN : new_split = 'train[:{}%]' . format ( round ( 100 * proportion ) ) elif split == tfds . Split . VALIDATION : new_split = 'train[-{}%:]' . format ( round ( 100 * proportion ) ) elif split == tfds . Split . TEST : new_split = 'test[:{}%]' . format ( round ( 100 * proportion ) ) else : raise ValueError ( 'Provide valid split.' ) dataset = tfds . load ( new_name , split = new_split , as_supervised = True ) if split == tfds . Split . TRAIN or repeat : dataset = dataset . shuffle ( buffer_size = dataset_size ) . repeat ( ) dataset = dataset . map ( preprocess , num_parallel_calls = tf . data . experimental . AUTOTUNE ) dataset = dataset . batch ( batch_size , drop_remainder = drop_remainder ) dataset = dataset . prefetch ( tf . data . experimental . AUTOTUNE ) if ctx and ctx . num_input_pipelines > 1 : dataset = dataset . shard ( ctx . num_input_pipelines , ctx . input_pipeline_id ) return dataset return input_fn
Loads CIFAR dataset for training or testing.
Loads CIFAR dataset for training or testing .
"""Loads CIFAR dataset for training or testing.

  Args:
    split: tfds.Split.
    batch_size: The global batch size to use.
    name: A string indicates whether it is cifar10 or cifar100.
    use_bfloat16: data type, bfloat16 precision or float32.
    normalize: Whether to apply mean-std normalization on features.
    drop_remainder: bool.
    repeat: bool.
    proportion: float, the proportion of dataset to be used.
    data_dir: Directory where the dataset is stored to be loaded via tfds.load.
      Optional, useful for loading datasets stored on GCS.

  Returns:
    Input function which returns a locally-sharded dataset batch.
  """ """Image preprocessing function.""" """Returns a locally sharded (i.e., per-core) dataset batch.""" # use round instead of floor to resolve bug when e.g. using # proportion = 1 - 0.8 = 0.19999999
Input function which returns a locally-sharded dataset batch.
Input function which returns a locally - sharded dataset batch .
split
None
batch_size
The global batch size to use.
The global batch size to use .
name
A string indicates whether it is cifar10 or cifar100.
A string indicates whether it is cifar10 or cifar100 .
use_bfloat16
data type, bfloat16 precision or float32.
data type bfloat16 precision or float32 .
normalize
Whether to apply mean-std normalization on features.
Whether to apply mean - std normalization on features .
drop_remainder
None
repeat
None
proportion
float, the proportion of dataset to be used.
float the proportion of dataset to be used .
data_dir
Directory where the dataset is stored to be loaded via tfds.load.
Optional, useful for loading datasets stored on GCS.
Directory where the dataset is stored to be loaded via tfds . load . Optional useful for loading datasets stored on GCS .