third_party/tensorflow-text/src/tensorflow_text/python/ops/whitespace_tokenizer.py
def _whitespace_tokenize_with_offsets_encode_decode_wrapper(
      self, input_tensor):
    """Tokenizes a tensor of UTF-8 strings with rank of 1.

    Args:
      input_tensor: The single dimensional Tensor to tokenize.

    Returns:
      Tuple of RaggedTensors of tokenized text and byte offsets, with shapes
      [num_strings, (num_tokens or num_offsets)].
    """
    # Decode the strings and get byte offsets
    (codepoints, byte_start_offsets) = (
        ragged_string_ops.unicode_decode_with_offsets(input_tensor, "UTF-8"))
    byte_end_offsets = array_ops.concat([
        byte_start_offsets[:, 1:],
        math_ops.cast(
            array_ops.expand_dims(string_ops.string_length(input_tensor), 1),
            dtypes.int64)
    ], 1)

    # Tokenize
    (codepoint_tokens, codepoint_start_offsets, codepoint_end_offsets) = (
        self._whitespace_tokenize_codepoints_with_offsets(codepoints))

    # Encode the codepoints and translate the codepoint offsets to byte offsets.
    return (ragged_string_ops.unicode_encode(codepoint_tokens, "UTF-8"),
            array_ops.batch_gather(byte_start_offsets, codepoint_start_offsets),
            array_ops.batch_gather(
                byte_end_offsets,
                math_ops.subtract(codepoint_end_offsets, [1])))
Tokenizes a tensor of UTF-8 strings with rank of 1.

    Args:
      input_tensor: The single dimensional Tensor to tokenize.

    Returns:
      Tuple of RaggedTensors of tokenized text and byte offsets, with shapes
      [num_strings, (num_tokens or num_offsets)].
    
Tokenizes a tensor of UTF-8 strings with rank of 1.
Tokenizes a tensor of UTF - 8 strings with rank of 1 .
def _whitespace_tokenize_with_offsets_encode_decode_wrapper(
      self, input_tensor):
    (codepoints, byte_start_offsets) = (
        ragged_string_ops.unicode_decode_with_offsets(input_tensor, "UTF-8"))
    byte_end_offsets = array_ops.concat([
        byte_start_offsets[:, 1:],
        math_ops.cast(
            array_ops.expand_dims(string_ops.string_length(input_tensor), 1),
            dtypes.int64)
    ], 1)
    (codepoint_tokens, codepoint_start_offsets, codepoint_end_offsets) = (
        self._whitespace_tokenize_codepoints_with_offsets(codepoints))
    return (ragged_string_ops.unicode_encode(codepoint_tokens, "UTF-8"),
            array_ops.batch_gather(byte_start_offsets, codepoint_start_offsets),
            array_ops.batch_gather(
                byte_end_offsets,
                math_ops.subtract(codepoint_end_offsets, [1])))
def _whitespace_tokenize_with_offsets_encode_decode_wrapper ( self , input_tensor ) : ( codepoints , byte_start_offsets ) = ( ragged_string_ops . unicode_decode_with_offsets ( input_tensor , "UTF-8" ) ) byte_end_offsets = array_ops . concat ( [ byte_start_offsets [ : , 1 : ] , math_ops . cast ( array_ops . expand_dims ( string_ops . string_length ( input_tensor ) , 1 ) , dtypes . int64 ) ] , 1 ) ( codepoint_tokens , codepoint_start_offsets , codepoint_end_offsets ) = ( self . _whitespace_tokenize_codepoints_with_offsets ( codepoints ) ) return ( ragged_string_ops . unicode_encode ( codepoint_tokens , "UTF-8" ) , array_ops . batch_gather ( byte_start_offsets , codepoint_start_offsets ) , array_ops . batch_gather ( byte_end_offsets , math_ops . subtract ( codepoint_end_offsets , [ 1 ] ) ) )
Tokenizes a tensor of UTF-8 strings with rank of 1.
Tokenizes a tensor of UTF - 8 strings with rank of 1 .
"""Tokenizes a tensor of UTF-8 strings with rank of 1.

    Args:
      input_tensor: The single dimensional Tensor to tokenize.

    Returns:
      Tuple of RaggedTensors of tokenized text and byte offsets, with shapes
      [num_strings, (num_tokens or num_offsets)].
    """ # Decode the strings and get byte offsets # Tokenize # Encode the codepoints and translate the codepoint offsets to byte offsets.
Tuple of RaggedTensors of tokenized text and byte offsets, with shapes
[num_strings, (num_tokens or num_offsets)].
Tuple of RaggedTensors of tokenized text and byte offsets with shapes [ num_strings ( num_tokens or num_offsets ) ] .
self
input_tensor
The single dimensional Tensor to tokenize.
The single dimensional Tensor to tokenize .