opennmt/data/dataset.py
def filter_examples_by_length(
    maximum_features_length=None,
    maximum_labels_length=None,
    features_length_fn=None,
    labels_length_fn=None,
):
    """Transformation that filters out examples with zero length or length that is
    greater than the configured maximum.

    Example:

      >>> dataset = dataset.apply(opennmt.data.filter_examples_by_length(...))

    Args:
      maximum_features_length: The maximum length or list of maximum lengths of
        the features sequence(s). ``None`` to not constrain the length.
      maximum_labels_length: The maximum length or list of maximum lengths of
        the labels sequence(s). ``None`` to not constrain the length.
      features_length_fn: A function mapping features to a sequence length.
      labels_length_fn: A function mapping labels to a sequence length.

    Returns:
      A ``tf.data.Dataset`` transformation.
    """
    if features_length_fn is None and labels_length_fn is None:
        return lambda dataset: dataset

    def _length_constraints(lengths, maximum_lengths):
        # Work with lists of lengths which correspond to the general multi source case.
        if not isinstance(lengths, list):
            lengths = [lengths]
        if not isinstance(maximum_lengths, list):
            maximum_lengths = [maximum_lengths]
        # Unset maximum lengths are set to None (i.e. no constraint).
        maximum_lengths += [None] * (len(lengths) - len(maximum_lengths))
        constraints = []
        for length, maximum_length in zip(lengths, maximum_lengths):
            constraints.append(tf.greater(length, 0))
            if maximum_length is not None:
                constraints.append(tf.less_equal(length, maximum_length))
        return constraints

    def _predicate(features, labels):
        cond = []
        features_length = (
            features_length_fn(features) if features_length_fn is not None else None
        )
        labels_length = (
            labels_length_fn(labels) if labels_length_fn is not None else None
        )
        if features_length is not None:
            cond.extend(_length_constraints(features_length, maximum_features_length))
        if labels_length is not None:
            cond.extend(_length_constraints(labels_length, maximum_labels_length))
        return tf.reduce_all(cond)

    return lambda dataset: dataset.filter(_predicate)
Transformation that filters out examples with zero length or length that is
    greater than the configured maximum.

    Example:

      >>> dataset = dataset.apply(opennmt.data.filter_examples_by_length(...))

    Args:
      maximum_features_length: The maximum length or list of maximum lengths of
        the features sequence(s). ``None`` to not constrain the length.
      maximum_labels_length: The maximum length or list of maximum lengths of
        the labels sequence(s). ``None`` to not constrain the length.
      features_length_fn: A function mapping features to a sequence length.
      labels_length_fn: A function mapping labels to a sequence length.

    Returns:
      A ``tf.data.Dataset`` transformation.
    
Transformation that filters out examples with zero length or length that is
greater than the configured maximum.
Transformation that filters out examples with zero length or length that is greater than the configured maximum .
def filter_examples_by_length(
    maximum_features_length=None,
    maximum_labels_length=None,
    features_length_fn=None,
    labels_length_fn=None,
):
    if features_length_fn is None and labels_length_fn is None:
        return lambda dataset: dataset
    def _length_constraints(lengths, maximum_lengths):
        if not isinstance(lengths, list):
            lengths = [lengths]
        if not isinstance(maximum_lengths, list):
            maximum_lengths = [maximum_lengths]
        maximum_lengths += [None] * (len(lengths) - len(maximum_lengths))
        constraints = []
        for length, maximum_length in zip(lengths, maximum_lengths):
            constraints.append(tf.greater(length, 0))
            if maximum_length is not None:
                constraints.append(tf.less_equal(length, maximum_length))
        return constraints
    def _predicate(features, labels):
        cond = []
        features_length = (
            features_length_fn(features) if features_length_fn is not None else None
        )
        labels_length = (
            labels_length_fn(labels) if labels_length_fn is not None else None
        )
        if features_length is not None:
            cond.extend(_length_constraints(features_length, maximum_features_length))
        if labels_length is not None:
            cond.extend(_length_constraints(labels_length, maximum_labels_length))
        return tf.reduce_all(cond)
    return lambda dataset: dataset.filter(_predicate)
def filter_examples_by_length ( maximum_features_length = None , maximum_labels_length = None , features_length_fn = None , labels_length_fn = None , ) : if features_length_fn is None and labels_length_fn is None : return lambda dataset : dataset def _length_constraints ( lengths , maximum_lengths ) : if not isinstance ( lengths , list ) : lengths = [ lengths ] if not isinstance ( maximum_lengths , list ) : maximum_lengths = [ maximum_lengths ] maximum_lengths += [ None ] * ( len ( lengths ) - len ( maximum_lengths ) ) constraints = [ ] for length , maximum_length in zip ( lengths , maximum_lengths ) : constraints . append ( tf . greater ( length , 0 ) ) if maximum_length is not None : constraints . append ( tf . less_equal ( length , maximum_length ) ) return constraints def _predicate ( features , labels ) : cond = [ ] features_length = ( features_length_fn ( features ) if features_length_fn is not None else None ) labels_length = ( labels_length_fn ( labels ) if labels_length_fn is not None else None ) if features_length is not None : cond . extend ( _length_constraints ( features_length , maximum_features_length ) ) if labels_length is not None : cond . extend ( _length_constraints ( labels_length , maximum_labels_length ) ) return tf . reduce_all ( cond ) return lambda dataset : dataset . filter ( _predicate )
Transformation that filters out examples with zero length or length that is
greater than the configured maximum.
Transformation that filters out examples with zero length or length that is greater than the configured maximum .
"""Transformation that filters out examples with zero length or length that is
    greater than the configured maximum.

    Example:

      >>> dataset = dataset.apply(opennmt.data.filter_examples_by_length(...))

    Args:
      maximum_features_length: The maximum length or list of maximum lengths of
        the features sequence(s). ``None`` to not constrain the length.
      maximum_labels_length: The maximum length or list of maximum lengths of
        the labels sequence(s). ``None`` to not constrain the length.
      features_length_fn: A function mapping features to a sequence length.
      labels_length_fn: A function mapping labels to a sequence length.

    Returns:
      A ``tf.data.Dataset`` transformation.
    """ # Work with lists of lengths which correspond to the general multi source case. # Unset maximum lengths are set to None (i.e. no constraint).
A ``tf.data.Dataset`` transformation.
A ` ` tf . data . Dataset ` ` transformation .
maximum_features_length
The maximum length or list of maximum lengths of
the features sequence(s). ``None`` to not constrain the length.
The maximum length or list of maximum lengths of the features sequence ( s ) . ` ` None ` ` to not constrain the length .
maximum_labels_length
The maximum length or list of maximum lengths of
the labels sequence(s). ``None`` to not constrain the length.
The maximum length or list of maximum lengths of the labels sequence ( s ) . ` ` None ` ` to not constrain the length .
features_length_fn
A function mapping features to a sequence length.
A function mapping features to a sequence length .
labels_length_fn
A function mapping labels to a sequence length.
A function mapping labels to a sequence length .
examples
>>> dataset = dataset.apply(opennmt.data.filter_examples_by_length(...))
>>> dataset = dataset . apply ( opennmt . data . filter_examples_by_length ( ... ))