opennmt/data/dataset.py
def normalize_weights(datasets, weights=None, sizes=None):
    """Returns normalized dataset weights based on datasets sizes.

    Args:
      datasets: A list of ``tf.data.Dataset`` instances.
      weights: An optional list of dataset weights.
      sizes: The size of each dataset, if known.

    Returns:
      A normalized list of weights that can be used as sampling probabilities.

    Raises:
      ValueError: if the length of :obj:`weights` or :obj:`sizes` does not match
        the length of :obj:`datasets`.
    """
    if not datasets:
        return []
    if len(datasets) == 1:
        return [1.0]

    if weights is None:
        weights = [1 / len(datasets)] * len(datasets)
    elif len(weights) != len(datasets):
        raise ValueError(
            "Got %d datasets but %d weights" % (len(datasets), len(weights))
        )

    if sizes is None:
        sizes = [int(get_dataset_size(dataset)) for dataset in datasets]
    elif len(sizes) != len(datasets):
        raise ValueError("Got %d datasets but %d sizes" % (len(datasets), len(sizes)))

    # Weights should be normalized by the dataset size relative to the total size.
    total_size = sum(sizes)
    weights = [weight * (size / total_size) for weight, size in zip(weights, sizes)]

    # Convert weights to probabilities.
    total_weights = sum(weights)
    return [weight / total_weights for weight in weights]
Returns normalized dataset weights based on datasets sizes.

    Args:
      datasets: A list of ``tf.data.Dataset`` instances.
      weights: An optional list of dataset weights.
      sizes: The size of each dataset, if known.

    Returns:
      A normalized list of weights that can be used as sampling probabilities.

    Raises:
      ValueError: if the length of :obj:`weights` or :obj:`sizes` does not match
        the length of :obj:`datasets`.
    
Returns normalized dataset weights based on datasets sizes.
Returns normalized dataset weights based on datasets sizes .
def normalize_weights(datasets, weights=None, sizes=None):
    if not datasets:
        return []
    if len(datasets) == 1:
        return [1.0]
    if weights is None:
        weights = [1 / len(datasets)] * len(datasets)
    elif len(weights) != len(datasets):
        raise ValueError(
            "Got %d datasets but %d weights" % (len(datasets), len(weights))
        )
    if sizes is None:
        sizes = [int(get_dataset_size(dataset)) for dataset in datasets]
    elif len(sizes) != len(datasets):
        raise ValueError("Got %d datasets but %d sizes" % (len(datasets), len(sizes)))
    total_size = sum(sizes)
    weights = [weight * (size / total_size) for weight, size in zip(weights, sizes)]
    total_weights = sum(weights)
    return [weight / total_weights for weight in weights]
def normalize_weights ( datasets , weights = None , sizes = None ) : if not datasets : return [ ] if len ( datasets ) == 1 : return [ 1.0 ] if weights is None : weights = [ 1 / len ( datasets ) ] * len ( datasets ) elif len ( weights ) != len ( datasets ) : raise ValueError ( "Got %d datasets but %d weights" % ( len ( datasets ) , len ( weights ) ) ) if sizes is None : sizes = [ int ( get_dataset_size ( dataset ) ) for dataset in datasets ] elif len ( sizes ) != len ( datasets ) : raise ValueError ( "Got %d datasets but %d sizes" % ( len ( datasets ) , len ( sizes ) ) ) total_size = sum ( sizes ) weights = [ weight * ( size / total_size ) for weight , size in zip ( weights , sizes ) ] total_weights = sum ( weights ) return [ weight / total_weights for weight in weights ]
Returns normalized dataset weights based on datasets sizes.
Returns normalized dataset weights based on datasets sizes .
"""Returns normalized dataset weights based on datasets sizes.

    Args:
      datasets: A list of ``tf.data.Dataset`` instances.
      weights: An optional list of dataset weights.
      sizes: The size of each dataset, if known.

    Returns:
      A normalized list of weights that can be used as sampling probabilities.

    Raises:
      ValueError: if the length of :obj:`weights` or :obj:`sizes` does not match
        the length of :obj:`datasets`.
    """ # Weights should be normalized by the dataset size relative to the total size. # Convert weights to probabilities.
A normalized list of weights that can be used as sampling probabilities.
A normalized list of weights that can be used as sampling probabilities .
if the length of :obj:`weights` or :obj:`sizes` does not match
the length of :obj:`datasets`.
if the length of : obj : ` weights ` or : obj : ` sizes ` does not match the length of : obj : ` datasets ` .
ValueError
datasets
A list of ``tf.data.Dataset`` instances.
A list of ` ` tf . data . Dataset ` ` instances .
weights
An optional list of dataset weights.
An optional list of dataset weights .
sizes
The size of each dataset, if known.
The size of each dataset if known .