opennmt/data/dataset.py
def inference_pipeline(
    batch_size,
    batch_type="examples",
    process_fn=None,
    transform_fns=None,
    length_bucket_width=None,
    length_fn=None,
    num_threads=None,
    prefetch_buffer_size=None,
):
    """Transformation that applies dataset operations for inference.

    Example:

      >>> dataset = dataset.apply(opennmt.data.inference_pipeline(...))

    Args:
      batch_size: The batch size to use.
      batch_type: The batching strategy to use: can be "examples" or "tokens".
      process_fn: The processing function to apply on each element.
      transform_fns: List of dataset transformation functions (applied after
        :obj:`process_fn` if defined).
      length_bucket_width: The width of the length buckets to select batch
        candidates from. If set, this means the inference pipeline will be
        reordered based on the examples length, the application is then
        responsible to restore the predictions in order. An "index" key will be
        inserted in the examples dictionary.
      length_fn: A function mapping features to a sequence length.
      num_threads: The number of elements processed in parallel.
      prefetch_buffer_size: The number of batches to prefetch asynchronously. If
        ``None``, use an automatically tuned value.

    Returns:
      A ``tf.data.Dataset`` transformation.

    Raises:
      ValueError: if :obj:`length_bucket_width` is set but not :obj:`length_fn`.
      ValueError: if :obj:`length_bucket_width` is set but the dataset does not
        output a dictionary structure.
    """

    def _inject_index(index, x):
        if isinstance(x, tuple):
            features = x[0]
        else:
            features = x
        features["index"] = index
        return x

    def _pipeline(dataset):
        if process_fn is not None:
            dataset = dataset.map(process_fn, num_parallel_calls=num_threads)
        if transform_fns is not None:
            for transform_fn in transform_fns:
                dataset = dataset.apply(transform_fn)
        if length_bucket_width is not None and length_bucket_width > 0:
            if length_fn is None:
                raise ValueError("length_fn is required when reordering by length")
            output_shapes = _get_output_shapes(dataset)
            if isinstance(output_shapes, tuple):
                num_length_fn = (
                    len(length_fn) if isinstance(length_fn, (list, tuple)) else 1
                )
                if len(output_shapes) != num_length_fn:
                    raise ValueError(
                        "The dataset outputs %d parallel features, but got %d "
                        "length functions" % (len(output_shapes), num_length_fn)
                    )
                output_shapes = output_shapes[0]
            if not isinstance(output_shapes, dict):
                raise ValueError(
                    "Reordering by length expects dataset elements to be Python dicts"
                )
            dataset = dataset.enumerate()
            dataset = dataset.map(_inject_index)
            dataset = dataset.apply(
                batch_sequence_dataset(
                    batch_size,
                    batch_type=batch_type,
                    length_bucket_width=length_bucket_width,
                    length_fn=length_fn,
                )
            )
        else:
            dataset = dataset.apply(batch_dataset(batch_size))
        dataset = dataset.prefetch(prefetch_buffer_size)
        return dataset

    return _pipeline
Transformation that applies dataset operations for inference.

    Example:

      >>> dataset = dataset.apply(opennmt.data.inference_pipeline(...))

    Args:
      batch_size: The batch size to use.
      batch_type: The batching strategy to use: can be "examples" or "tokens".
      process_fn: The processing function to apply on each element.
      transform_fns: List of dataset transformation functions (applied after
        :obj:`process_fn` if defined).
      length_bucket_width: The width of the length buckets to select batch
        candidates from. If set, this means the inference pipeline will be
        reordered based on the examples length, the application is then
        responsible to restore the predictions in order. An "index" key will be
        inserted in the examples dictionary.
      length_fn: A function mapping features to a sequence length.
      num_threads: The number of elements processed in parallel.
      prefetch_buffer_size: The number of batches to prefetch asynchronously. If
        ``None``, use an automatically tuned value.

    Returns:
      A ``tf.data.Dataset`` transformation.

    Raises:
      ValueError: if :obj:`length_bucket_width` is set but not :obj:`length_fn`.
      ValueError: if :obj:`length_bucket_width` is set but the dataset does not
        output a dictionary structure.
    
Transformation that applies dataset operations for inference.
Transformation that applies dataset operations for inference .
def inference_pipeline(
    batch_size,
    batch_type="examples",
    process_fn=None,
    transform_fns=None,
    length_bucket_width=None,
    length_fn=None,
    num_threads=None,
    prefetch_buffer_size=None,
):
    def _inject_index(index, x):
        if isinstance(x, tuple):
            features = x[0]
        else:
            features = x
        features["index"] = index
        return x
    def _pipeline(dataset):
        if process_fn is not None:
            dataset = dataset.map(process_fn, num_parallel_calls=num_threads)
        if transform_fns is not None:
            for transform_fn in transform_fns:
                dataset = dataset.apply(transform_fn)
        if length_bucket_width is not None and length_bucket_width > 0:
            if length_fn is None:
                raise ValueError("length_fn is required when reordering by length")
            output_shapes = _get_output_shapes(dataset)
            if isinstance(output_shapes, tuple):
                num_length_fn = (
                    len(length_fn) if isinstance(length_fn, (list, tuple)) else 1
                )
                if len(output_shapes) != num_length_fn:
                    raise ValueError(
                        "The dataset outputs %d parallel features, but got %d "
                        "length functions" % (len(output_shapes), num_length_fn)
                    )
                output_shapes = output_shapes[0]
            if not isinstance(output_shapes, dict):
                raise ValueError(
                    "Reordering by length expects dataset elements to be Python dicts"
                )
            dataset = dataset.enumerate()
            dataset = dataset.map(_inject_index)
            dataset = dataset.apply(
                batch_sequence_dataset(
                    batch_size,
                    batch_type=batch_type,
                    length_bucket_width=length_bucket_width,
                    length_fn=length_fn,
                )
            )
        else:
            dataset = dataset.apply(batch_dataset(batch_size))
        dataset = dataset.prefetch(prefetch_buffer_size)
        return dataset
    return _pipeline
def inference_pipeline ( batch_size , batch_type = "examples" , process_fn = None , transform_fns = None , length_bucket_width = None , length_fn = None , num_threads = None , prefetch_buffer_size = None , ) : def _inject_index ( index , x ) : if isinstance ( x , tuple ) : features = x [ 0 ] else : features = x features [ "index" ] = index return x def _pipeline ( dataset ) : if process_fn is not None : dataset = dataset . map ( process_fn , num_parallel_calls = num_threads ) if transform_fns is not None : for transform_fn in transform_fns : dataset = dataset . apply ( transform_fn ) if length_bucket_width is not None and length_bucket_width > 0 : if length_fn is None : raise ValueError ( "length_fn is required when reordering by length" ) output_shapes = _get_output_shapes ( dataset ) if isinstance ( output_shapes , tuple ) : num_length_fn = ( len ( length_fn ) if isinstance ( length_fn , ( list , tuple ) ) else 1 ) if len ( output_shapes ) != num_length_fn : raise ValueError ( "The dataset outputs %d parallel features, but got %d " "length functions" % ( len ( output_shapes ) , num_length_fn ) ) output_shapes = output_shapes [ 0 ] if not isinstance ( output_shapes , dict ) : raise ValueError ( "Reordering by length expects dataset elements to be Python dicts" ) dataset = dataset . enumerate ( ) dataset = dataset . map ( _inject_index ) dataset = dataset . apply ( batch_sequence_dataset ( batch_size , batch_type = batch_type , length_bucket_width = length_bucket_width , length_fn = length_fn , ) ) else : dataset = dataset . apply ( batch_dataset ( batch_size ) ) dataset = dataset . prefetch ( prefetch_buffer_size ) return dataset return _pipeline
Transformation that applies dataset operations for inference.
Transformation that applies dataset operations for inference .
"""Transformation that applies dataset operations for inference.

    Example:

      >>> dataset = dataset.apply(opennmt.data.inference_pipeline(...))

    Args:
      batch_size: The batch size to use.
      batch_type: The batching strategy to use: can be "examples" or "tokens".
      process_fn: The processing function to apply on each element.
      transform_fns: List of dataset transformation functions (applied after
        :obj:`process_fn` if defined).
      length_bucket_width: The width of the length buckets to select batch
        candidates from. If set, this means the inference pipeline will be
        reordered based on the examples length, the application is then
        responsible to restore the predictions in order. An "index" key will be
        inserted in the examples dictionary.
      length_fn: A function mapping features to a sequence length.
      num_threads: The number of elements processed in parallel.
      prefetch_buffer_size: The number of batches to prefetch asynchronously. If
        ``None``, use an automatically tuned value.

    Returns:
      A ``tf.data.Dataset`` transformation.

    Raises:
      ValueError: if :obj:`length_bucket_width` is set but not :obj:`length_fn`.
      ValueError: if :obj:`length_bucket_width` is set but the dataset does not
        output a dictionary structure.
    """
A ``tf.data.Dataset`` transformation.
A ` ` tf . data . Dataset ` ` transformation .
if :obj:`length_bucket_width` is set but not :obj:`length_fn`.
if : obj : ` length_bucket_width ` is set but not : obj : ` length_fn ` .
ValueError
if :obj:`length_bucket_width` is set but the dataset does not
output a dictionary structure.
if : obj : ` length_bucket_width ` is set but the dataset does not output a dictionary structure .
ValueError
batch_size
The batch size to use.
The batch size to use .
batch_type
The batching strategy to use: can be "examples" or "tokens".
The batching strategy to use : can be " examples " or " tokens " .
process_fn
The processing function to apply on each element.
The processing function to apply on each element .
transform_fns
List of dataset transformation functions (applied after
:obj:`process_fn` if defined).
List of dataset transformation functions ( applied after : obj : ` process_fn ` if defined ) .
length_bucket_width
The width of the length buckets to select batch
candidates from. If set, this means the inference pipeline will be
reordered based on the examples length, the application is then
responsible to restore the predictions in order. An "index" key will be
inserted in the examples dictionary.
The width of the length buckets to select batch candidates from . If set this means the inference pipeline will be reordered based on the examples length the application is then responsible to restore the predictions in order . An " index " key will be inserted in the examples dictionary .
length_fn
A function mapping features to a sequence length.
A function mapping features to a sequence length .
num_threads
The number of elements processed in parallel.
The number of elements processed in parallel .
prefetch_buffer_size
The number of batches to prefetch asynchronously. If
``None``, use an automatically tuned value.
The number of batches to prefetch asynchronously . If ` ` None ` ` use an automatically tuned value .
examples
>>> dataset = dataset.apply(opennmt.data.inference_pipeline(...))
>>> dataset = dataset . apply ( opennmt . data . inference_pipeline ( ... ))