third_party/tensorflow-text/src/tensorflow_text/python/ops/unicode_script_tokenizer.py
def tokenize_with_offsets(self, input):  # pylint: disable=redefined-builtin
    """Tokenizes a tensor of UTF-8 strings on Unicode script boundaries.

    The strings are split when a change in the Unicode script is detected
    between sequential tokens. The script codes used correspond to International
    Components for Unicode (ICU) UScriptCode values. See:
    http://icu-project.org/apiref/icu4c/uscript_8h.html

    ICU defined whitespace characters are dropped, unless the keep_whitespace
    option was specified at construction time.

    Args:
      input: A `RaggedTensor`or `Tensor` of UTF-8 strings with any shape.

    Returns:
      A tuple `(tokens, start_offsets, end_offsets)` where:

        * `tokens`: A `RaggedTensor` of tokenized text.
        * `start_offsets`: A `RaggedTensor` of the tokens' starting byte offset.
        * `end_offsets`: A `RaggedTensor` of the tokens' ending byte offset.
    """
    name = None
    with ops.name_scope(name, "UnicodeScriptTokenize", [input]):
      input_tensor = ragged_tensor.convert_to_tensor_or_ragged_tensor(input)
      if input_tensor.shape.ndims is None:
        raise ValueError("Rank of input_tensor must be statically known.")
      if ragged_tensor.is_ragged(input_tensor):
        if input_tensor.flat_values.shape.ndims > 1:
          # If the flat_values of our ragged tensor is multi-dimensional, we can
          # process it separately and our output will have the same nested
          # splits as our input.
          (tokens, starts, ends) = self.tokenize_with_offsets(
              input_tensor.flat_values)
          return (input_tensor.with_flat_values(tokens),
                  input_tensor.with_flat_values(starts),
                  input_tensor.with_flat_values(ends))
        else:
          # Recursively process the values of the ragged tensor.
          (tokens, starts, ends) = self.tokenize_with_offsets(
              input_tensor.values)
          return (input_tensor.with_values(tokens),
                  input_tensor.with_values(starts),
                  input_tensor.with_values(ends))
      else:
        if input_tensor.shape.ndims > 1:
          # Convert the input tensor to ragged and process it.
          return self.tokenize_with_offsets(
              ragged_conversion_ops.from_tensor(input_tensor))
        elif input_tensor.shape.ndims == 0:
          (tokens, starts, ends) = self.tokenize_with_offsets(
              array_ops.stack([input_tensor]))
          return tokens.values, starts.values, ends.values
        else:
          # Our rank 1 tensor is the correct shape, so we can process it as
          # normal
          return self._tokenize_with_offsets_encode_decode_wrapper(input_tensor)
Tokenizes a tensor of UTF-8 strings on Unicode script boundaries.

    The strings are split when a change in the Unicode script is detected
    between sequential tokens. The script codes used correspond to International
    Components for Unicode (ICU) UScriptCode values. See:
    http://icu-project.org/apiref/icu4c/uscript_8h.html

    ICU defined whitespace characters are dropped, unless the keep_whitespace
    option was specified at construction time.

    Args:
      input: A `RaggedTensor`or `Tensor` of UTF-8 strings with any shape.

    Returns:
      A tuple `(tokens, start_offsets, end_offsets)` where:

        * `tokens`: A `RaggedTensor` of tokenized text.
        * `start_offsets`: A `RaggedTensor` of the tokens' starting byte offset.
        * `end_offsets`: A `RaggedTensor` of the tokens' ending byte offset.
    
Tokenizes a tensor of UTF-8 strings on Unicode script boundaries.
The strings are split when a change in the Unicode script is detected
between sequential tokens. The script codes used correspond to International
Components for Unicode (ICU) UScriptCode values.

ICU defined whitespace characters are dropped, unless the keep_whitespace
option was specified at construction time.
Tokenizes a tensor of UTF - 8 strings on Unicode script boundaries . The strings are split when a change in the Unicode script is detected between sequential tokens . The script codes used correspond to International Components for Unicode ( ICU ) UScriptCode values . ICU defined whitespace characters are dropped unless the keep_whitespace option was specified at construction time .
def tokenize_with_offsets(self, input):  
    name = None
    with ops.name_scope(name, "UnicodeScriptTokenize", [input]):
      input_tensor = ragged_tensor.convert_to_tensor_or_ragged_tensor(input)
      if input_tensor.shape.ndims is None:
        raise ValueError("Rank of input_tensor must be statically known.")
      if ragged_tensor.is_ragged(input_tensor):
        if input_tensor.flat_values.shape.ndims > 1:
          (tokens, starts, ends) = self.tokenize_with_offsets(
              input_tensor.flat_values)
          return (input_tensor.with_flat_values(tokens),
                  input_tensor.with_flat_values(starts),
                  input_tensor.with_flat_values(ends))
        else:
          (tokens, starts, ends) = self.tokenize_with_offsets(
              input_tensor.values)
          return (input_tensor.with_values(tokens),
                  input_tensor.with_values(starts),
                  input_tensor.with_values(ends))
      else:
        if input_tensor.shape.ndims > 1:
          return self.tokenize_with_offsets(
              ragged_conversion_ops.from_tensor(input_tensor))
        elif input_tensor.shape.ndims == 0:
          (tokens, starts, ends) = self.tokenize_with_offsets(
              array_ops.stack([input_tensor]))
          return tokens.values, starts.values, ends.values
        else:
          return self._tokenize_with_offsets_encode_decode_wrapper(input_tensor)
def tokenize_with_offsets ( self , input ) : name = None with ops . name_scope ( name , "UnicodeScriptTokenize" , [ input ] ) : input_tensor = ragged_tensor . convert_to_tensor_or_ragged_tensor ( input ) if input_tensor . shape . ndims is None : raise ValueError ( "Rank of input_tensor must be statically known." ) if ragged_tensor . is_ragged ( input_tensor ) : if input_tensor . flat_values . shape . ndims > 1 : ( tokens , starts , ends ) = self . tokenize_with_offsets ( input_tensor . flat_values ) return ( input_tensor . with_flat_values ( tokens ) , input_tensor . with_flat_values ( starts ) , input_tensor . with_flat_values ( ends ) ) else : ( tokens , starts , ends ) = self . tokenize_with_offsets ( input_tensor . values ) return ( input_tensor . with_values ( tokens ) , input_tensor . with_values ( starts ) , input_tensor . with_values ( ends ) ) else : if input_tensor . shape . ndims > 1 : return self . tokenize_with_offsets ( ragged_conversion_ops . from_tensor ( input_tensor ) ) elif input_tensor . shape . ndims == 0 : ( tokens , starts , ends ) = self . tokenize_with_offsets ( array_ops . stack ( [ input_tensor ] ) ) return tokens . values , starts . values , ends . values else : return self . _tokenize_with_offsets_encode_decode_wrapper ( input_tensor )
Tokenizes a tensor of UTF-8 strings on Unicode script boundaries.
Tokenizes a tensor of UTF - 8 strings on Unicode script boundaries .
# pylint: disable=redefined-builtin """Tokenizes a tensor of UTF-8 strings on Unicode script boundaries.

    The strings are split when a change in the Unicode script is detected
    between sequential tokens. The script codes used correspond to International
    Components for Unicode (ICU) UScriptCode values. See:
    http://icu-project.org/apiref/icu4c/uscript_8h.html

    ICU defined whitespace characters are dropped, unless the keep_whitespace
    option was specified at construction time.

    Args:
      input: A `RaggedTensor`or `Tensor` of UTF-8 strings with any shape.

    Returns:
      A tuple `(tokens, start_offsets, end_offsets)` where:

        * `tokens`: A `RaggedTensor` of tokenized text.
        * `start_offsets`: A `RaggedTensor` of the tokens' starting byte offset.
        * `end_offsets`: A `RaggedTensor` of the tokens' ending byte offset.
    """ # If the flat_values of our ragged tensor is multi-dimensional, we can # process it separately and our output will have the same nested # splits as our input. # Recursively process the values of the ragged tensor. # Convert the input tensor to ragged and process it. # Our rank 1 tensor is the correct shape, so we can process it as # normal
A tuple `(tokens, start_offsets, end_offsets)` where.

`tokens`: A `RaggedTensor` of tokenized text.
`start_offsets`: A `RaggedTensor` of the tokens' starting byte offset.
`end_offsets`: A `RaggedTensor` of the tokens' ending byte offset.
A tuple ` ( tokens start_offsets end_offsets ) ` where . ` tokens ` : A ` RaggedTensor ` of tokenized text . ` start_offsets ` : A ` RaggedTensor ` of the tokens ' starting byte offset . ` end_offsets ` : A ` RaggedTensor ` of the tokens ' ending byte offset .
self
input
A `RaggedTensor`or `Tensor` of UTF-8 strings with any shape.
A ` RaggedTensor ` or ` Tensor ` of UTF - 8 strings with any shape .