third_party/tensorflow-text/src/tensorflow_text/python/ops/unicode_char_tokenizer.py
def tokenize_with_offsets(self, input):  # pylint: disable=redefined-builtin
    """Tokenizes a tensor of UTF-8 strings to Unicode characters.

    Returned token tensors are of integer type.

    Args:
      input: A `RaggedTensor`or `Tensor` of UTF-8 strings with any shape.

    Returns:
      A tuple `(tokens, start_offsets, end_offsets)` where:

        * `tokens`: A `RaggedTensor` of codepoints (integer type).
        * `start_offsets`: A `RaggedTensor` of the tokens' starting byte offset.
        * `end_offsets`: A `RaggedTensor` of the tokens' ending byte offset.
    """
    name = None
    with ops.name_scope(name, "UnicodeCharTokenize", [input]):
      input_tensor = ragged_tensor.convert_to_tensor_or_ragged_tensor(input)
      (codepoints, byte_start_offsets) = (
          ragged_string_ops.unicode_decode_with_offsets(input_tensor, "UTF-8"))
      strlens = math_ops.cast(
          array_ops.expand_dims(string_ops.string_length(input_tensor), -1),
          dtypes.int64)
      # Adjust strlens to set 0-length strings to empty array (there will be no
      # tokens in this case).
      final_ends = ragged_array_ops.boolean_mask(strlens, strlens > 0)
      byte_end_offsets = array_ops.concat(
          [byte_start_offsets[..., 1:], final_ends], -1)
      return codepoints, byte_start_offsets, byte_end_offsets
Tokenizes a tensor of UTF-8 strings to Unicode characters.

    Returned token tensors are of integer type.

    Args:
      input: A `RaggedTensor`or `Tensor` of UTF-8 strings with any shape.

    Returns:
      A tuple `(tokens, start_offsets, end_offsets)` where:

        * `tokens`: A `RaggedTensor` of codepoints (integer type).
        * `start_offsets`: A `RaggedTensor` of the tokens' starting byte offset.
        * `end_offsets`: A `RaggedTensor` of the tokens' ending byte offset.
    
Tokenizes a tensor of UTF-8 strings to Unicode characters.
Returned token tensors are of integer type.
Tokenizes a tensor of UTF - 8 strings to Unicode characters . Returned token tensors are of integer type .
def tokenize_with_offsets(self, input):  
    name = None
    with ops.name_scope(name, "UnicodeCharTokenize", [input]):
      input_tensor = ragged_tensor.convert_to_tensor_or_ragged_tensor(input)
      (codepoints, byte_start_offsets) = (
          ragged_string_ops.unicode_decode_with_offsets(input_tensor, "UTF-8"))
      strlens = math_ops.cast(
          array_ops.expand_dims(string_ops.string_length(input_tensor), -1),
          dtypes.int64)
      final_ends = ragged_array_ops.boolean_mask(strlens, strlens > 0)
      byte_end_offsets = array_ops.concat(
          [byte_start_offsets[..., 1:], final_ends], -1)
      return codepoints, byte_start_offsets, byte_end_offsets
def tokenize_with_offsets ( self , input ) : name = None with ops . name_scope ( name , "UnicodeCharTokenize" , [ input ] ) : input_tensor = ragged_tensor . convert_to_tensor_or_ragged_tensor ( input ) ( codepoints , byte_start_offsets ) = ( ragged_string_ops . unicode_decode_with_offsets ( input_tensor , "UTF-8" ) ) strlens = math_ops . cast ( array_ops . expand_dims ( string_ops . string_length ( input_tensor ) , - 1 ) , dtypes . int64 ) final_ends = ragged_array_ops . boolean_mask ( strlens , strlens > 0 ) byte_end_offsets = array_ops . concat ( [ byte_start_offsets [ ... , 1 : ] , final_ends ] , - 1 ) return codepoints , byte_start_offsets , byte_end_offsets
Tokenizes a tensor of UTF-8 strings to Unicode characters.
Tokenizes a tensor of UTF - 8 strings to Unicode characters .
# pylint: disable=redefined-builtin """Tokenizes a tensor of UTF-8 strings to Unicode characters.

    Returned token tensors are of integer type.

    Args:
      input: A `RaggedTensor`or `Tensor` of UTF-8 strings with any shape.

    Returns:
      A tuple `(tokens, start_offsets, end_offsets)` where:

        * `tokens`: A `RaggedTensor` of codepoints (integer type).
        * `start_offsets`: A `RaggedTensor` of the tokens' starting byte offset.
        * `end_offsets`: A `RaggedTensor` of the tokens' ending byte offset.
    """ # Adjust strlens to set 0-length strings to empty array (there will be no # tokens in this case).
A tuple `(tokens, start_offsets, end_offsets)` where.

`tokens`: A `RaggedTensor` of codepoints (integer type).
`start_offsets`: A `RaggedTensor` of the tokens' starting byte offset.
`end_offsets`: A `RaggedTensor` of the tokens' ending byte offset.
A tuple ` ( tokens start_offsets end_offsets ) ` where . ` tokens ` : A ` RaggedTensor ` of codepoints ( integer type ) . ` start_offsets ` : A ` RaggedTensor ` of the tokens ' starting byte offset . ` end_offsets ` : A ` RaggedTensor ` of the tokens ' ending byte offset .
self
input
A `RaggedTensor`or `Tensor` of UTF-8 strings with any shape.
A ` RaggedTensor ` or ` Tensor ` of UTF - 8 strings with any shape .