opennmt/data/dataset.py
def random_shard(shard_size, dataset_size):
    """Transformation that shards the dataset in a random order.

    Example:

      >>> dataset = tf.data.Dataset.range(6)
      >>> dataset = dataset.apply(opennmt.data.random_shard(2, 6))
      >>> list(dataset.as_numpy_iterator())
      [0, 1, 4, 5, 2, 3]

    Args:
      shard_size: The number of examples in each shard.
      dataset_size: The total number of examples in the dataset.

    Returns:
      A ``tf.data.Dataset`` transformation.
    """
    num_shards = -(-dataset_size // shard_size)  # Ceil division.
    offsets = np.linspace(
        0, dataset_size, num=num_shards, endpoint=False, dtype=np.int64
    )

    def _random_shard(dataset):
        sharded_dataset = tf.data.Dataset.from_tensor_slices(offsets)
        sharded_dataset = sharded_dataset.shuffle(num_shards)
        sharded_dataset = sharded_dataset.flat_map(
            lambda offset: dataset.skip(offset).take(shard_size)
        )
        return sharded_dataset

    return _random_shard
Transformation that shards the dataset in a random order.

    Example:

      >>> dataset = tf.data.Dataset.range(6)
      >>> dataset = dataset.apply(opennmt.data.random_shard(2, 6))
      >>> list(dataset.as_numpy_iterator())
      [0, 1, 4, 5, 2, 3]

    Args:
      shard_size: The number of examples in each shard.
      dataset_size: The total number of examples in the dataset.

    Returns:
      A ``tf.data.Dataset`` transformation.
    
Transformation that shards the dataset in a random order.
Transformation that shards the dataset in a random order .
def random_shard(shard_size, dataset_size):
    num_shards = -(-dataset_size // shard_size)  
    offsets = np.linspace(
        0, dataset_size, num=num_shards, endpoint=False, dtype=np.int64
    )
    def _random_shard(dataset):
        sharded_dataset = tf.data.Dataset.from_tensor_slices(offsets)
        sharded_dataset = sharded_dataset.shuffle(num_shards)
        sharded_dataset = sharded_dataset.flat_map(
            lambda offset: dataset.skip(offset).take(shard_size)
        )
        return sharded_dataset
    return _random_shard
def random_shard ( shard_size , dataset_size ) : num_shards = - ( - dataset_size // shard_size ) offsets = np . linspace ( 0 , dataset_size , num = num_shards , endpoint = False , dtype = np . int64 ) def _random_shard ( dataset ) : sharded_dataset = tf . data . Dataset . from_tensor_slices ( offsets ) sharded_dataset = sharded_dataset . shuffle ( num_shards ) sharded_dataset = sharded_dataset . flat_map ( lambda offset : dataset . skip ( offset ) . take ( shard_size ) ) return sharded_dataset return _random_shard
Transformation that shards the dataset in a random order.
Transformation that shards the dataset in a random order .
"""Transformation that shards the dataset in a random order.

    Example:

      >>> dataset = tf.data.Dataset.range(6)
      >>> dataset = dataset.apply(opennmt.data.random_shard(2, 6))
      >>> list(dataset.as_numpy_iterator())
      [0, 1, 4, 5, 2, 3]

    Args:
      shard_size: The number of examples in each shard.
      dataset_size: The total number of examples in the dataset.

    Returns:
      A ``tf.data.Dataset`` transformation.
    """ # Ceil division.
A ``tf.data.Dataset`` transformation.
A ` ` tf . data . Dataset ` ` transformation .
shard_size
The number of examples in each shard.
The number of examples in each shard .
dataset_size
The total number of examples in the dataset.
The total number of examples in the dataset .
examples
>>> dataset = tf.data.Dataset.range(6)
>>> dataset = dataset.apply(opennmt.data.random_shard(2, 6))
>>> list(dataset.as_numpy_iterator())
[0, 1, 4, 5, 2, 3]
>>> dataset = tf . data . Dataset . range ( 6 ) >>> dataset = dataset . apply ( opennmt . data . random_shard ( 2 6 )) >>> list ( dataset . as_numpy_iterator () ) [ 0 1 4 5 2 3 ]