class_docstr: Tokenizes UTF-8 by splitting when there is a change in Unicode script.
  
  By default, this tokenizer leaves out scripts matching the whitespace unicode
  property (use the `keep_whitespace` argument to keep it), so in this case the
  results are similar to the `WhitespaceTokenizer`. Any punctuation
  will get its own token (since it is in a different script), and any script
  change in the input string will be the location of a split.
  
  Example:
  >>> tokenizer = tf_text.UnicodeScriptTokenizer()
  >>> tokens = tokenizer.tokenize(["xy.,z de", "fg?h", "abαβ"])
  >>> print(tokens.to_list())
  [[b'xy', b'.,', b'z', b'de'], [b'fg', b'?', b'h'],
   [b'ab', b'\xce\xb1\xce\xb2']]
  
  >>> tokens = tokenizer.tokenize(u"累計7239人")
  >>> print(tokens)
  tf.Tensor([b'\xe7\xb4\xaf\xe8\xa8\x88' b'7239' b'\xe4\xba\xba'], shape=(3,),
            dtype=string)
  
  Both the punctuation and the whitespace in the first string have been split,
  but the punctuation run is present as a token while the whitespace isn't
  emitted (by default). The third example shows the case of a script change
  without any whitespace. This results in a split at that boundary point.
class_docstr_tok: ["Tokenizes","UTF-8","by","splitting","when","there","is","a","change","in","Unicode","script",".","By","default",",","this","tokenizer","leaves","out","scripts","matching","the","whitespace","unicode","property","(","use","the","`","keep_whitespace","`","argument","to","keep","it",")",",","so","in","this","case","the","results","are","similar","to","the","`","WhitespaceTokenizer","`",".","Any","punctuation","will","get","its","own","token","(","since","it","is","in","a","different","script",")",",","and","any","script","change","in","the","input","string","will","be","the","location","of","a","split",".","Example",":",">",">",">","tokenizer","=","tf_text.UnicodeScriptTokenizer","(",")",">",">",">","tokens","=","tokenizer.tokenize","(","[","``","xy.",",","z","de","''",",","``","fg","?","h","''",",","``","ab\u03b1\u03b2","''","]",")",">",">",">","print","(","tokens.to_list","(",")",")","[","[","b'xy","'",",","b'.",",","'",",","b","'","z","'",",","b'de","'","]",",","[","b'fg","'",",","b","'","?","'",",","b","'","h","'","]",",","[","b'ab","'",",","b'\\xce\\xb1\\xce\\xb2","'","]","]",">",">",">","tokens","=","tokenizer.tokenize","(","u","''","\u7d2f\u8a087239\u4eba","''",")",">",">",">","print","(","tokens",")","tf.Tensor","(","[","b'\\xe7\\xb4\\xaf\\xe8\\xa8\\x88","'","b'7239","'","b'\\xe4\\xba\\xba","'","]",",","shape=","(","3",",",")",",","dtype=string",")","Both","the","punctuation","and","the","whitespace","in","the","first","string","have","been","split",",","but","the","punctuation","run","is","present","as","a","token","while","the","whitespace","isn't","emitted","(","by","default",")",".","The","third","example","shows","the","case","of","a","script","change","without","any","whitespace",".","This","results","in","a","split","at","that","boundary","point","."]
docstr: Tokenizes a tensor of UTF-8 strings with rank of 1.
  
  Args:
    input_tensor: The single dimensional Tensor to tokenize.
  
  Returns:
    Tuple of RaggedTensors of tokenized text and byte offsets, with shapes
    [num_strings, (num_tokens or num_offsets)].
docstr_tok: ["Tokenizes","a","tensor","of","UTF-8","strings","with","rank","of","1",".","Args",":","input_tensor",":","The","single","dimensional","Tensor","to","tokenize",".","Returns",":","Tuple","of","RaggedTensors","of","tokenized","text","and","byte","offsets",",","with","shapes","[","num_strings",",","(","num_tokens","or","num_offsets",")","]","."]
code: def _tokenize_with_offsets_encode_decode_wrapper(self, input_tensor):
      """Tokenizes a tensor of UTF-8 strings with rank of 1.
  
      Args:
        input_tensor: The single dimensional Tensor to tokenize.
  
      Returns:
        Tuple of RaggedTensors of tokenized text and byte offsets, with shapes
        [num_strings, (num_tokens or num_offsets)].
      """
      # Decode the strings and get byte offsets
      (
          codepoints,
          byte_start_offsets,
      ) = ragged_string_ops.unicode_decode_with_offsets(
          input_tensor, "UTF-8"
      )
      byte_end_offsets = array_ops.concat(
          [
              byte_start_offsets[:, 1:],
              math_ops.cast(
                  array_ops.expand_dims(
                      string_ops.string_length(input_tensor), 1
                  ),
                  dtypes.int64,
              ),
          ],
          1,
      )
  
      # Tokenize
      (
          codepoint_tokens,
          codepoint_start_offsets,
          codepoint_end_offsets,
      ) = self._tokenize_codepoints_with_offsets(codepoints)
  
      # Encode the codepoints and translate the codepoint offsets to byte offsets.
      return (
          ragged_string_ops.unicode_encode(codepoint_tokens, "UTF-8"),
          array_ops.batch_gather(
              byte_start_offsets, codepoint_start_offsets
          ),
          array_ops.batch_gather(
              byte_end_offsets,
              math_ops.subtract(codepoint_end_offsets, [1]),
          ),
      )
  
imports: from __future__ import absolute_import
  from __future__ import division
  from __future__ import print_function
  from tensorflow.python.eager import monitoring
  from tensorflow.python.framework import dtypes
  from tensorflow.python.framework import ops
  from tensorflow.python.ops import array_ops
  from tensorflow.python.ops import array_ops_stack
  from tensorflow.python.ops import math_ops
  from tensorflow.python.ops import string_ops
  from tensorflow.python.ops.ragged import ragged_conversion_ops
  from tensorflow.python.ops.ragged import ragged_string_ops
  from tensorflow.python.ops.ragged import ragged_tensor
  from tensorflow.python.ops.ragged.ragged_tensor import RaggedTensor
  from tensorflow_text.python.ops.tokenization import TokenizerWithOffsets
  from tensorflow.python.framework import load_library
  from tensorflow.python.platform import resource_loader
label_desc: Use image node_id 4 for calling the UnicodeScriptTokenizer obj's underlying member method code with example usage: obj._tokenize_with_offsets_encode_decode_wrapper(input_tensor) and returns: ragged_string_ops,  array_ops,  array_ops