third_party/tensorflow-text/src/tensorflow_text/python/ops/trimmer_ops.py
def generate_mask(self, segments):
    """Calculates a truncation mask given a per-batch budget.

    Calculate a truncation mask given a budget of the max number of items for
    each or all batch row. The allocation of the budget is done using a
    'waterfall' algorithm. This algorithm allocates quota in a left-to-right
    manner and fill up the buckets until we run out of budget.

    For example if the budget of [5] and we have segments of size
    [3, 4, 2], the truncate budget will be allocated as [3, 2, 0].

    The budget can be a scalar, in which case the same budget is broadcasted
    and applied to all batch rows. It can also be a 1D `Tensor` of size
    `batch_size`, in which each batch row i will have a budget corresponding to
    `per_batch_quota[i]`.

    Args:
      segments: A list of `RaggedTensor` each w/ a shape of [num_batch,
        (num_items)].
    Returns:
      a list with len(segments) of `RaggedTensor`s, see superclass for details.
    """
    with ops.name_scope("WaterfallTrimmer/generate_mask"):
      segment_row_lengths = [_get_row_lengths(s, self._axis) for s in segments]
      segment_row_lengths = array_ops.stack(segment_row_lengths, axis=-1)

      # Broadcast budget to match the rank of segments[0]
      budget = ops.convert_to_tensor(self._max_seq_length)
      for _ in range(segments[0].shape.ndims - budget.shape.ndims):
        budget = array_ops.expand_dims(budget, -1)

      # Compute the allocation for each segment using a `waterfall` algorithm
      segment_lengths = math_ops.cast(segment_row_lengths, dtypes.int32)
      budget = math_ops.cast(budget, dtypes.int32)
      leftover_budget = math_ops.cumsum(
          -1 * segment_lengths, exclusive=False, axis=-1) + budget
      leftover_budget = segment_lengths + math_ops.minimum(leftover_budget, 0)
      results = math_ops.maximum(leftover_budget, 0)

      # Translate the results into boolean masks that match the shape of each
      # segment
      results = array_ops.unstack(results, axis=-1)
      item_selectors = [
          item_selector_ops.FirstNItemSelector(i) for i in results
      ]
      return [
          i.get_selectable(s, self._axis)
          for s, i in zip(segments, item_selectors)
      ]
Calculates a truncation mask given a per-batch budget.

    Calculate a truncation mask given a budget of the max number of items for
    each or all batch row. The allocation of the budget is done using a
    'waterfall' algorithm. This algorithm allocates quota in a left-to-right
    manner and fill up the buckets until we run out of budget.

    For example if the budget of [5] and we have segments of size
    [3, 4, 2], the truncate budget will be allocated as [3, 2, 0].

    The budget can be a scalar, in which case the same budget is broadcasted
    and applied to all batch rows. It can also be a 1D `Tensor` of size
    `batch_size`, in which each batch row i will have a budget corresponding to
    `per_batch_quota[i]`.

    Args:
      segments: A list of `RaggedTensor` each w/ a shape of [num_batch,
        (num_items)].
    Returns:
      a list with len(segments) of `RaggedTensor`s, see superclass for details.
    
Calculates a truncation mask given a per-batch budget.
Calculate a truncation mask given a budget of the max number of items for
each or all batch row. The allocation of the budget is done using a
'waterfall' algorithm. This algorithm allocates quota in a left-to-right
manner and fill up the buckets until we run out of budget.

For example if the budget of [5] and we have segments of size
[3, 4, 2], the truncate budget will be allocated as [3, 2, 0].

The budget can be a scalar, in which case the same budget is broadcasted
and applied to all batch rows.
Calculates a truncation mask given a per - batch budget . Calculate a truncation mask given a budget of the max number of items for each or all batch row . The allocation of the budget is done using a ' waterfall ' algorithm . This algorithm allocates quota in a left - to - right manner and fill up the buckets until we run out of budget . For example if the budget of [ 5 ] and we have segments of size [ 3 4 2 ] the truncate budget will be allocated as [ 3 2 0 ] . The budget can be a scalar in which case the same budget is broadcasted and applied to all batch rows .
def generate_mask(self, segments):
    with ops.name_scope("WaterfallTrimmer/generate_mask"):
      segment_row_lengths = [_get_row_lengths(s, self._axis) for s in segments]
      segment_row_lengths = array_ops.stack(segment_row_lengths, axis=-1)
      budget = ops.convert_to_tensor(self._max_seq_length)
      for _ in range(segments[0].shape.ndims - budget.shape.ndims):
        budget = array_ops.expand_dims(budget, -1)
      segment_lengths = math_ops.cast(segment_row_lengths, dtypes.int32)
      budget = math_ops.cast(budget, dtypes.int32)
      leftover_budget = math_ops.cumsum(
          -1 * segment_lengths, exclusive=False, axis=-1) + budget
      leftover_budget = segment_lengths + math_ops.minimum(leftover_budget, 0)
      results = math_ops.maximum(leftover_budget, 0)
      results = array_ops.unstack(results, axis=-1)
      item_selectors = [
          item_selector_ops.FirstNItemSelector(i) for i in results
      ]
      return [
          i.get_selectable(s, self._axis)
          for s, i in zip(segments, item_selectors)
      ]
def generate_mask ( self , segments ) : with ops . name_scope ( "WaterfallTrimmer/generate_mask" ) : segment_row_lengths = [ _get_row_lengths ( s , self . _axis ) for s in segments ] segment_row_lengths = array_ops . stack ( segment_row_lengths , axis = - 1 ) budget = ops . convert_to_tensor ( self . _max_seq_length ) for _ in range ( segments [ 0 ] . shape . ndims - budget . shape . ndims ) : budget = array_ops . expand_dims ( budget , - 1 ) segment_lengths = math_ops . cast ( segment_row_lengths , dtypes . int32 ) budget = math_ops . cast ( budget , dtypes . int32 ) leftover_budget = math_ops . cumsum ( - 1 * segment_lengths , exclusive = False , axis = - 1 ) + budget leftover_budget = segment_lengths + math_ops . minimum ( leftover_budget , 0 ) results = math_ops . maximum ( leftover_budget , 0 ) results = array_ops . unstack ( results , axis = - 1 ) item_selectors = [ item_selector_ops . FirstNItemSelector ( i ) for i in results ] return [ i . get_selectable ( s , self . _axis ) for s , i in zip ( segments , item_selectors ) ]
Calculates a truncation mask given a per-batch budget.
Calculates a truncation mask given a per - batch budget .
"""Calculates a truncation mask given a per-batch budget.

    Calculate a truncation mask given a budget of the max number of items for
    each or all batch row. The allocation of the budget is done using a
    'waterfall' algorithm. This algorithm allocates quota in a left-to-right
    manner and fill up the buckets until we run out of budget.

    For example if the budget of [5] and we have segments of size
    [3, 4, 2], the truncate budget will be allocated as [3, 2, 0].

    The budget can be a scalar, in which case the same budget is broadcasted
    and applied to all batch rows. It can also be a 1D `Tensor` of size
    `batch_size`, in which each batch row i will have a budget corresponding to
    `per_batch_quota[i]`.

    Args:
      segments: A list of `RaggedTensor` each w/ a shape of [num_batch,
        (num_items)].
    Returns:
      a list with len(segments) of `RaggedTensor`s, see superclass for details.
    """ # Broadcast budget to match the rank of segments[0] # Compute the allocation for each segment using a `waterfall` algorithm # Translate the results into boolean masks that match the shape of each # segment
a list with len(segments) of `RaggedTensor`s, see superclass for details.
a list with len ( segments ) of ` RaggedTensor ` s see superclass for details .
self
segments
A list of `RaggedTensor` each w/ a shape of [num_batch,
(num_items)].
A list of ` RaggedTensor ` each w / a shape of [ num_batch ( num_items ) ] .