opennmt/data/dataset.py
def batch_sequence_dataset(
    batch_size,
    batch_type="examples",
    batch_multiplier=1,
    batch_size_multiple=1,
    length_bucket_width=None,
    length_fn=None,
    padded_shapes=None,
):
    """Transformation that batches a dataset of sequences.

    This implements an example-based and a token-based batching strategy
    with optional bucketing of sequences.

    Bucketing makes the batches contain sequences of similar lengths to optimize
    the training efficiency. For example, if :obj:`length_bucket_width` is 5,
    sequences will be organized by the following length buckets:

    1 - 5 | 6 - 10 | 11 - 15 | ...

    Then when building the next batch, sequences will be selected from the same
    length bucket.

    If the dataset has parallel elements (e.g. a parallel source and target
    dataset), the element is assigned to the bucket corresponding to the maximum
    length of all parallel elements.

    Example:

      >>> dataset = dataset.apply(opennmt.data.batch_sequence_dataset(...))

    Args:
      batch_size: The batch size.
      batch_type: The training batching strategy to use: can be "examples" or
        "tokens".
      batch_multiplier: The batch size multiplier.
      batch_size_multiple: When :obj:`batch_type` is "tokens", ensure that the
        resulting batch size is a multiple of this value.
      length_bucket_width: The width of the length buckets to select batch
        candidates from. ``None`` to not constrain batch formation.
      length_fn: A function or list of functions (in case of a parallel dataset)
        that take features as argument and return the associated sequence length.
      padded_shapes: The padded shapes for this dataset. If ``None``, the shapes
        are automatically inferred from the dataset output shapes.

    Returns:
      A ``tf.data.Dataset`` transformation.

    Raises:
      ValueError: if :obj:`batch_type` is not one of "examples" or "tokens".
      ValueError: if :obj:`batch_type` is "tokens" but :obj:`length_bucket_width`
        is not set.
      ValueError: if the number of length functions in :obj:`length_fn` does not
        match the number of parallel elements.

    See Also:
      :func:`opennmt.data.batch_dataset`
    """
    batch_size = batch_size * batch_multiplier

    def _get_bucket_id(features, length_fn):
        default_id = tf.constant(0, dtype=tf.int32)
        if length_fn is None:
            return default_id
        lengths = length_fn(features)
        if lengths is None:
            return default_id
        if not isinstance(lengths, list):
            lengths = [lengths]  # Fallback to the general case of parallel inputs.
        lengths = [length // length_bucket_width for length in lengths]
        return tf.reduce_max(lengths)

    def _key_func(*args):
        length_fns = length_fn
        if length_fns is None:
            length_fns = [None for _ in args]
        elif not isinstance(length_fns, (list, tuple)):
            length_fns = [length_fns]
        if len(length_fns) != len(args):
            raise ValueError(
                "%d length functions were passed but this dataset contains "
                "%d parallel elements" % (len(length_fns), len(args))
            )
        # Take the highest bucket id.
        bucket_id = tf.reduce_max(
            [
                _get_bucket_id(features, length_fn)
                for features, length_fn in zip(args, length_fns)
            ]
        )
        return tf.cast(bucket_id, tf.int64)

    def _reduce_func(unused_key, dataset):
        return dataset.apply(batch_dataset(batch_size, padded_shapes=padded_shapes))

    def _window_size_func(key):
        if length_bucket_width > 1:
            key += 1  # For length_bucket_width == 1, key 0 is unassigned.
        size = batch_size // (key * length_bucket_width)
        required_multiple = batch_multiplier * batch_size_multiple
        if required_multiple > 1:
            size = size + required_multiple - size % required_multiple
        return tf.cast(tf.maximum(size, required_multiple), tf.int64)

    def _group_by_window(*args, **kwargs):
        # TODO: clean this API when TensorFlow requirement is updated to >=2.6.
        if compat.tf_supports("data.Dataset.group_by_window"):
            return lambda dataset: dataset.group_by_window(*args, **kwargs)
        else:
            return tf.data.experimental.group_by_window(*args, **kwargs)

    if length_bucket_width is None:
        if batch_type == "tokens":
            raise ValueError(
                "Batch type 'tokens' requires length bucketing (the parameter "
                "length_bucket_width should be non null)"
            )
        return batch_dataset(batch_size, padded_shapes=padded_shapes)

    if batch_type == "examples":
        return _group_by_window(_key_func, _reduce_func, window_size=batch_size)
    elif batch_type == "tokens":
        return _group_by_window(
            _key_func, _reduce_func, window_size_func=_window_size_func
        )
    else:
        raise ValueError(
            "Invalid batch type: '{}'; should be 'examples' or 'tokens'".format(
                batch_type
            )
        )
Transformation that batches a dataset of sequences.

    This implements an example-based and a token-based batching strategy
    with optional bucketing of sequences.

    Bucketing makes the batches contain sequences of similar lengths to optimize
    the training efficiency. For example, if :obj:`length_bucket_width` is 5,
    sequences will be organized by the following length buckets:

    1 - 5 | 6 - 10 | 11 - 15 | ...

    Then when building the next batch, sequences will be selected from the same
    length bucket.

    If the dataset has parallel elements (e.g. a parallel source and target
    dataset), the element is assigned to the bucket corresponding to the maximum
    length of all parallel elements.

    Example:

      >>> dataset = dataset.apply(opennmt.data.batch_sequence_dataset(...))

    Args:
      batch_size: The batch size.
      batch_type: The training batching strategy to use: can be "examples" or
        "tokens".
      batch_multiplier: The batch size multiplier.
      batch_size_multiple: When :obj:`batch_type` is "tokens", ensure that the
        resulting batch size is a multiple of this value.
      length_bucket_width: The width of the length buckets to select batch
        candidates from. ``None`` to not constrain batch formation.
      length_fn: A function or list of functions (in case of a parallel dataset)
        that take features as argument and return the associated sequence length.
      padded_shapes: The padded shapes for this dataset. If ``None``, the shapes
        are automatically inferred from the dataset output shapes.

    Returns:
      A ``tf.data.Dataset`` transformation.

    Raises:
      ValueError: if :obj:`batch_type` is not one of "examples" or "tokens".
      ValueError: if :obj:`batch_type` is "tokens" but :obj:`length_bucket_width`
        is not set.
      ValueError: if the number of length functions in :obj:`length_fn` does not
        match the number of parallel elements.

    See Also:
      :func:`opennmt.data.batch_dataset`
    
Transformation that batches a dataset of sequences.
This implements an example-based and a token-based batching strategy
with optional bucketing of sequences.

Bucketing makes the batches contain sequences of similar lengths to optimize
the training efficiency. For example, if :obj:`length_bucket_width` is 5,
sequences will be organized by the following length buckets.



Then when building the next batch, sequences will be selected from the same
length bucket.

If the dataset has parallel elements , the element is assigned to the bucket corresponding to the maximum
length of all parallel elements.
Transformation that batches a dataset of sequences . This implements an example - based and a token - based batching strategy with optional bucketing of sequences . Bucketing makes the batches contain sequences of similar lengths to optimize the training efficiency . For example if : obj : ` length_bucket_width ` is 5 sequences will be organized by the following length buckets . Then when building the next batch sequences will be selected from the same length bucket . If the dataset has parallel elements the element is assigned to the bucket corresponding to the maximum length of all parallel elements .
def batch_sequence_dataset(
    batch_size,
    batch_type="examples",
    batch_multiplier=1,
    batch_size_multiple=1,
    length_bucket_width=None,
    length_fn=None,
    padded_shapes=None,
):
    batch_size = batch_size * batch_multiplier
    def _get_bucket_id(features, length_fn):
        default_id = tf.constant(0, dtype=tf.int32)
        if length_fn is None:
            return default_id
        lengths = length_fn(features)
        if lengths is None:
            return default_id
        if not isinstance(lengths, list):
            lengths = [lengths]  
        lengths = [length // length_bucket_width for length in lengths]
        return tf.reduce_max(lengths)
    def _key_func(*args):
        length_fns = length_fn
        if length_fns is None:
            length_fns = [None for _ in args]
        elif not isinstance(length_fns, (list, tuple)):
            length_fns = [length_fns]
        if len(length_fns) != len(args):
            raise ValueError(
                "%d length functions were passed but this dataset contains "
                "%d parallel elements" % (len(length_fns), len(args))
            )
        bucket_id = tf.reduce_max(
            [
                _get_bucket_id(features, length_fn)
                for features, length_fn in zip(args, length_fns)
            ]
        )
        return tf.cast(bucket_id, tf.int64)
    def _reduce_func(unused_key, dataset):
        return dataset.apply(batch_dataset(batch_size, padded_shapes=padded_shapes))
    def _window_size_func(key):
        if length_bucket_width > 1:
            key += 1  
        size = batch_size // (key * length_bucket_width)
        required_multiple = batch_multiplier * batch_size_multiple
        if required_multiple > 1:
            size = size + required_multiple - size % required_multiple
        return tf.cast(tf.maximum(size, required_multiple), tf.int64)
    def _group_by_window(*args, **kwargs):
        if compat.tf_supports("data.Dataset.group_by_window"):
            return lambda dataset: dataset.group_by_window(*args, **kwargs)
        else:
            return tf.data.experimental.group_by_window(*args, **kwargs)
    if length_bucket_width is None:
        if batch_type == "tokens":
            raise ValueError(
                "Batch type 'tokens' requires length bucketing (the parameter "
                "length_bucket_width should be non null)"
            )
        return batch_dataset(batch_size, padded_shapes=padded_shapes)
    if batch_type == "examples":
        return _group_by_window(_key_func, _reduce_func, window_size=batch_size)
    elif batch_type == "tokens":
        return _group_by_window(
            _key_func, _reduce_func, window_size_func=_window_size_func
        )
    else:
        raise ValueError(
            "Invalid batch type: '{}'; should be 'examples' or 'tokens'".format(
                batch_type
            )
        )
def batch_sequence_dataset ( batch_size , batch_type = "examples" , batch_multiplier = 1 , batch_size_multiple = 1 , length_bucket_width = None , length_fn = None , padded_shapes = None , ) : batch_size = batch_size * batch_multiplier def _get_bucket_id ( features , length_fn ) : default_id = tf . constant ( 0 , dtype = tf . int32 ) if length_fn is None : return default_id lengths = length_fn ( features ) if lengths is None : return default_id if not isinstance ( lengths , list ) : lengths = [ lengths ] lengths = [ length // length_bucket_width for length in lengths ] return tf . reduce_max ( lengths ) def _key_func ( * args ) : length_fns = length_fn if length_fns is None : length_fns = [ None for _ in args ] elif not isinstance ( length_fns , ( list , tuple ) ) : length_fns = [ length_fns ] if len ( length_fns ) != len ( args ) : raise ValueError ( "%d length functions were passed but this dataset contains " "%d parallel elements" % ( len ( length_fns ) , len ( args ) ) ) bucket_id = tf . reduce_max ( [ _get_bucket_id ( features , length_fn ) for features , length_fn in zip ( args , length_fns ) ] ) return tf . cast ( bucket_id , tf . int64 ) def _reduce_func ( unused_key , dataset ) : return dataset . apply ( batch_dataset ( batch_size , padded_shapes = padded_shapes ) ) def _window_size_func ( key ) : if length_bucket_width > 1 : key += 1 size = batch_size // ( key * length_bucket_width ) required_multiple = batch_multiplier * batch_size_multiple if required_multiple > 1 : size = size + required_multiple - size % required_multiple return tf . cast ( tf . maximum ( size , required_multiple ) , tf . int64 ) def _group_by_window ( * args , ** kwargs ) : if compat . tf_supports ( "data.Dataset.group_by_window" ) : return lambda dataset : dataset . group_by_window ( * args , ** kwargs ) else : return tf . data . experimental . group_by_window ( * args , ** kwargs ) if length_bucket_width is None : if batch_type == "tokens" : raise ValueError ( "Batch type 'tokens' requires length bucketing (the parameter " "length_bucket_width should be non null)" ) return batch_dataset ( batch_size , padded_shapes = padded_shapes ) if batch_type == "examples" : return _group_by_window ( _key_func , _reduce_func , window_size = batch_size ) elif batch_type == "tokens" : return _group_by_window ( _key_func , _reduce_func , window_size_func = _window_size_func ) else : raise ValueError ( "Invalid batch type: '{}'; should be 'examples' or 'tokens'" . format ( batch_type ) )
Transformation that batches a dataset of sequences.
Transformation that batches a dataset of sequences .
"""Transformation that batches a dataset of sequences.

    This implements an example-based and a token-based batching strategy
    with optional bucketing of sequences.

    Bucketing makes the batches contain sequences of similar lengths to optimize
    the training efficiency. For example, if :obj:`length_bucket_width` is 5,
    sequences will be organized by the following length buckets:

    1 - 5 | 6 - 10 | 11 - 15 | ...

    Then when building the next batch, sequences will be selected from the same
    length bucket.

    If the dataset has parallel elements (e.g. a parallel source and target
    dataset), the element is assigned to the bucket corresponding to the maximum
    length of all parallel elements.

    Example:

      >>> dataset = dataset.apply(opennmt.data.batch_sequence_dataset(...))

    Args:
      batch_size: The batch size.
      batch_type: The training batching strategy to use: can be "examples" or
        "tokens".
      batch_multiplier: The batch size multiplier.
      batch_size_multiple: When :obj:`batch_type` is "tokens", ensure that the
        resulting batch size is a multiple of this value.
      length_bucket_width: The width of the length buckets to select batch
        candidates from. ``None`` to not constrain batch formation.
      length_fn: A function or list of functions (in case of a parallel dataset)
        that take features as argument and return the associated sequence length.
      padded_shapes: The padded shapes for this dataset. If ``None``, the shapes
        are automatically inferred from the dataset output shapes.

    Returns:
      A ``tf.data.Dataset`` transformation.

    Raises:
      ValueError: if :obj:`batch_type` is not one of "examples" or "tokens".
      ValueError: if :obj:`batch_type` is "tokens" but :obj:`length_bucket_width`
        is not set.
      ValueError: if the number of length functions in :obj:`length_fn` does not
        match the number of parallel elements.

    See Also:
      :func:`opennmt.data.batch_dataset`
    """ # Fallback to the general case of parallel inputs. # Take the highest bucket id. # For length_bucket_width == 1, key 0 is unassigned. # TODO: clean this API when TensorFlow requirement is updated to >=2.6.
A ``tf.data.Dataset`` transformation.
A ` ` tf . data . Dataset ` ` transformation .
if :obj:`batch_type` is not one of "examples" or "tokens".
if : obj : ` batch_type ` is not one of " examples " or " tokens " .
ValueError
if :obj:`batch_type` is "tokens" but :obj:`length_bucket_width`
is not set.
if : obj : ` batch_type ` is " tokens " but : obj : ` length_bucket_width ` is not set .
ValueError
if the number of length functions in :obj:`length_fn` does not
match the number of parallel elements.
if the number of length functions in : obj : ` length_fn ` does not match the number of parallel elements .
ValueError
batch_size
The batch size.
The batch size .
batch_type
The training batching strategy to use: can be "examples" or
"tokens".
The training batching strategy to use : can be " examples " or " tokens " .
batch_multiplier
The batch size multiplier.
The batch size multiplier .
batch_size_multiple
When :obj:`batch_type` is "tokens", ensure that the
resulting batch size is a multiple of this value.
When : obj : ` batch_type ` is " tokens " ensure that the resulting batch size is a multiple of this value .
length_bucket_width
The width of the length buckets to select batch
candidates from. ``None`` to not constrain batch formation.
The width of the length buckets to select batch candidates from . ` ` None ` ` to not constrain batch formation .
length_fn
A function or list of functions (in case of a parallel dataset)
that take features as argument and return the associated sequence length.
A function or list of functions ( in case of a parallel dataset ) that take features as argument and return the associated sequence length .
padded_shapes
The padded shapes for this dataset. If ``None``, the shapes
are automatically inferred from the dataset output shapes.
The padded shapes for this dataset . If ` ` None ` ` the shapes are automatically inferred from the dataset output shapes .
examples
>>> dataset = dataset.apply(opennmt.data.batch_sequence_dataset(...))
>>> dataset = dataset . apply ( opennmt . data . batch_sequence_dataset ( ... ))