opennmt/data/dataset.py
def shuffle_dataset(buffer_size, shuffle_shards=True, dataset_size=None):
    """Transformation that shuffles the dataset based on its size.

    Example:

      >>> dataset = tf.data.Dataset.range(6)
      >>> dataset = dataset.apply(opennmt.data.shuffle_dataset(3))
      >>> list(dataset.as_numpy_iterator())
      [2, 3, 1, 0, 4, 5]

    Args:
      buffer_size: The number of elements from which to sample.
      shuffle_shards: When :obj:`buffer_size` is smaller than the dataset size,
        the dataset is first sharded in a random order to add another level of
        shuffling.
      dataset_size: If the dataset size is already known, it can be passed here to
        avoid a slower generic computation of the dataset size later.

    Returns:
      A ``tf.data.Dataset`` transformation.
    """

    def _shuffle(dataset):
        sample_size = buffer_size
        if sample_size < 0 or shuffle_shards:
            total_size = dataset_size
            if total_size is None:
                total_size = get_dataset_size(dataset)
            tf.get_logger().info("Training on %d examples", total_size)
            if sample_size < 0:
                sample_size = total_size
            elif sample_size < total_size:
                dataset = dataset.apply(random_shard(sample_size, total_size))
        dataset = dataset.shuffle(sample_size)
        return dataset

    return _shuffle
Transformation that shuffles the dataset based on its size.

    Example:

      >>> dataset = tf.data.Dataset.range(6)
      >>> dataset = dataset.apply(opennmt.data.shuffle_dataset(3))
      >>> list(dataset.as_numpy_iterator())
      [2, 3, 1, 0, 4, 5]

    Args:
      buffer_size: The number of elements from which to sample.
      shuffle_shards: When :obj:`buffer_size` is smaller than the dataset size,
        the dataset is first sharded in a random order to add another level of
        shuffling.
      dataset_size: If the dataset size is already known, it can be passed here to
        avoid a slower generic computation of the dataset size later.

    Returns:
      A ``tf.data.Dataset`` transformation.
    
Transformation that shuffles the dataset based on its size.
Transformation that shuffles the dataset based on its size .
def shuffle_dataset(buffer_size, shuffle_shards=True, dataset_size=None):
    def _shuffle(dataset):
        sample_size = buffer_size
        if sample_size < 0 or shuffle_shards:
            total_size = dataset_size
            if total_size is None:
                total_size = get_dataset_size(dataset)
            tf.get_logger().info("Training on %d examples", total_size)
            if sample_size < 0:
                sample_size = total_size
            elif sample_size < total_size:
                dataset = dataset.apply(random_shard(sample_size, total_size))
        dataset = dataset.shuffle(sample_size)
        return dataset
    return _shuffle
def shuffle_dataset ( buffer_size , shuffle_shards = True , dataset_size = None ) : def _shuffle ( dataset ) : sample_size = buffer_size if sample_size < 0 or shuffle_shards : total_size = dataset_size if total_size is None : total_size = get_dataset_size ( dataset ) tf . get_logger ( ) . info ( "Training on %d examples" , total_size ) if sample_size < 0 : sample_size = total_size elif sample_size < total_size : dataset = dataset . apply ( random_shard ( sample_size , total_size ) ) dataset = dataset . shuffle ( sample_size ) return dataset return _shuffle
Transformation that shuffles the dataset based on its size.
Transformation that shuffles the dataset based on its size .
"""Transformation that shuffles the dataset based on its size.

    Example:

      >>> dataset = tf.data.Dataset.range(6)
      >>> dataset = dataset.apply(opennmt.data.shuffle_dataset(3))
      >>> list(dataset.as_numpy_iterator())
      [2, 3, 1, 0, 4, 5]

    Args:
      buffer_size: The number of elements from which to sample.
      shuffle_shards: When :obj:`buffer_size` is smaller than the dataset size,
        the dataset is first sharded in a random order to add another level of
        shuffling.
      dataset_size: If the dataset size is already known, it can be passed here to
        avoid a slower generic computation of the dataset size later.

    Returns:
      A ``tf.data.Dataset`` transformation.
    """
A ``tf.data.Dataset`` transformation.
A ` ` tf . data . Dataset ` ` transformation .
buffer_size
The number of elements from which to sample.
The number of elements from which to sample .
shuffle_shards
When :obj:`buffer_size` is smaller than the dataset size,
the dataset is first sharded in a random order to add another level of
shuffling.
When : obj : ` buffer_size ` is smaller than the dataset size the dataset is first sharded in a random order to add another level of shuffling .
dataset_size
If the dataset size is already known, it can be passed here to
avoid a slower generic computation of the dataset size later.
If the dataset size is already known it can be passed here to avoid a slower generic computation of the dataset size later .
examples
>>> dataset = tf.data.Dataset.range(6)
>>> dataset = dataset.apply(opennmt.data.shuffle_dataset(3))
>>> list(dataset.as_numpy_iterator())
[2, 3, 1, 0, 4, 5]
>>> dataset = tf . data . Dataset . range ( 6 ) >>> dataset = dataset . apply ( opennmt . data . shuffle_dataset ( 3 )) >>> list ( dataset . as_numpy_iterator () ) [ 2 3 1 0 4 5 ]