third_party/tensorflow-text/src/tensorflow_text/python/ops/whitespace_tokenizer.py
def tokenize_with_offsets(self, input):  # pylint: disable=redefined-builtin
    """Tokenizes a tensor of UTF-8 strings on whitespaces.

    The strings are split on ICU defined whitespace characters. These
    whitespace characters are dropped.

    Args:
      input: A `RaggedTensor`or `Tensor` of UTF-8 strings with any shape.

    Returns:
      A tuple `(tokens, start_offsets, end_offsets)` where:

        * `tokens`: A `RaggedTensor` of tokenized text.
        * `start_offsets`: A `RaggedTensor` of the tokens' starting byte offset.
        * `end_offsets`: A `RaggedTensor` of the tokens' ending byte offset.
    """
    name = None
    with ops.name_scope(name, "WhitespaceTokenize", [input]):
      input_tensor = ragged_tensor.convert_to_tensor_or_ragged_tensor(input)
      if input_tensor.shape.ndims is None:
        raise ValueError("Rank of input_tensor must be statically known.")
      if ragged_tensor.is_ragged(input_tensor):
        if input_tensor.flat_values.shape.ndims > 1:
          # If the flat_values of our ragged tensor is multi-dimensional, we can
          # process it separately and our output will have the same nested
          # splits as our input.
          (tokens, starts,
           ends) = self.tokenize_with_offsets(input_tensor.flat_values)
          return (input_tensor.with_flat_values(tokens),
                  input_tensor.with_flat_values(starts),
                  input_tensor.with_flat_values(ends))
        else:
          # Recursively process the values of the ragged tensor.
          (tokens, starts,
           ends) = self.tokenize_with_offsets(input_tensor.values)
          return (input_tensor.with_values(tokens),
                  input_tensor.with_values(starts),
                  input_tensor.with_values(ends))
      else:
        if input_tensor.shape.ndims > 1:
          # Convert the input tensor to ragged and process it.
          return self.tokenize_with_offsets(
              ragged_conversion_ops.from_tensor(input_tensor))
        elif input_tensor.shape.ndims == 0:
          (tokens, starts, ends) = self.tokenize_with_offsets(
              array_ops.stack([input_tensor]))
          return tokens.values, starts.values, ends.values
        else:
          # Our rank 1 tensor is the correct shape, so we can process it as
          # normal.
          return self._whitespace_tokenize_with_offsets_encode_decode_wrapper(
              input_tensor)
Tokenizes a tensor of UTF-8 strings on whitespaces.

    The strings are split on ICU defined whitespace characters. These
    whitespace characters are dropped.

    Args:
      input: A `RaggedTensor`or `Tensor` of UTF-8 strings with any shape.

    Returns:
      A tuple `(tokens, start_offsets, end_offsets)` where:

        * `tokens`: A `RaggedTensor` of tokenized text.
        * `start_offsets`: A `RaggedTensor` of the tokens' starting byte offset.
        * `end_offsets`: A `RaggedTensor` of the tokens' ending byte offset.
    
Tokenizes a tensor of UTF-8 strings on whitespaces.
The strings are split on ICU defined whitespace characters. These
whitespace characters are dropped.
Tokenizes a tensor of UTF - 8 strings on whitespaces . The strings are split on ICU defined whitespace characters . These whitespace characters are dropped .
def tokenize_with_offsets(self, input):  
    name = None
    with ops.name_scope(name, "WhitespaceTokenize", [input]):
      input_tensor = ragged_tensor.convert_to_tensor_or_ragged_tensor(input)
      if input_tensor.shape.ndims is None:
        raise ValueError("Rank of input_tensor must be statically known.")
      if ragged_tensor.is_ragged(input_tensor):
        if input_tensor.flat_values.shape.ndims > 1:
          (tokens, starts,
           ends) = self.tokenize_with_offsets(input_tensor.flat_values)
          return (input_tensor.with_flat_values(tokens),
                  input_tensor.with_flat_values(starts),
                  input_tensor.with_flat_values(ends))
        else:
          (tokens, starts,
           ends) = self.tokenize_with_offsets(input_tensor.values)
          return (input_tensor.with_values(tokens),
                  input_tensor.with_values(starts),
                  input_tensor.with_values(ends))
      else:
        if input_tensor.shape.ndims > 1:
          return self.tokenize_with_offsets(
              ragged_conversion_ops.from_tensor(input_tensor))
        elif input_tensor.shape.ndims == 0:
          (tokens, starts, ends) = self.tokenize_with_offsets(
              array_ops.stack([input_tensor]))
          return tokens.values, starts.values, ends.values
        else:
          return self._whitespace_tokenize_with_offsets_encode_decode_wrapper(
              input_tensor)
def tokenize_with_offsets ( self , input ) : name = None with ops . name_scope ( name , "WhitespaceTokenize" , [ input ] ) : input_tensor = ragged_tensor . convert_to_tensor_or_ragged_tensor ( input ) if input_tensor . shape . ndims is None : raise ValueError ( "Rank of input_tensor must be statically known." ) if ragged_tensor . is_ragged ( input_tensor ) : if input_tensor . flat_values . shape . ndims > 1 : ( tokens , starts , ends ) = self . tokenize_with_offsets ( input_tensor . flat_values ) return ( input_tensor . with_flat_values ( tokens ) , input_tensor . with_flat_values ( starts ) , input_tensor . with_flat_values ( ends ) ) else : ( tokens , starts , ends ) = self . tokenize_with_offsets ( input_tensor . values ) return ( input_tensor . with_values ( tokens ) , input_tensor . with_values ( starts ) , input_tensor . with_values ( ends ) ) else : if input_tensor . shape . ndims > 1 : return self . tokenize_with_offsets ( ragged_conversion_ops . from_tensor ( input_tensor ) ) elif input_tensor . shape . ndims == 0 : ( tokens , starts , ends ) = self . tokenize_with_offsets ( array_ops . stack ( [ input_tensor ] ) ) return tokens . values , starts . values , ends . values else : return self . _whitespace_tokenize_with_offsets_encode_decode_wrapper ( input_tensor )
Tokenizes a tensor of UTF-8 strings on whitespaces.
Tokenizes a tensor of UTF - 8 strings on whitespaces .
# pylint: disable=redefined-builtin """Tokenizes a tensor of UTF-8 strings on whitespaces.

    The strings are split on ICU defined whitespace characters. These
    whitespace characters are dropped.

    Args:
      input: A `RaggedTensor`or `Tensor` of UTF-8 strings with any shape.

    Returns:
      A tuple `(tokens, start_offsets, end_offsets)` where:

        * `tokens`: A `RaggedTensor` of tokenized text.
        * `start_offsets`: A `RaggedTensor` of the tokens' starting byte offset.
        * `end_offsets`: A `RaggedTensor` of the tokens' ending byte offset.
    """ # If the flat_values of our ragged tensor is multi-dimensional, we can # process it separately and our output will have the same nested # splits as our input. # Recursively process the values of the ragged tensor. # Convert the input tensor to ragged and process it. # Our rank 1 tensor is the correct shape, so we can process it as # normal.
A tuple `(tokens, start_offsets, end_offsets)` where.

`tokens`: A `RaggedTensor` of tokenized text.
`start_offsets`: A `RaggedTensor` of the tokens' starting byte offset.
`end_offsets`: A `RaggedTensor` of the tokens' ending byte offset.
A tuple ` ( tokens start_offsets end_offsets ) ` where . ` tokens ` : A ` RaggedTensor ` of tokenized text . ` start_offsets ` : A ` RaggedTensor ` of the tokens ' starting byte offset . ` end_offsets ` : A ` RaggedTensor ` of the tokens ' ending byte offset .
self
input
A `RaggedTensor`or `Tensor` of UTF-8 strings with any shape.
A ` RaggedTensor ` or ` Tensor ` of UTF - 8 strings with any shape .