class_docstr: Tokenizes a tensor of UTF-8 strings on Unicode character boundaries.
  
  
  Resulting tokens are integers (unicode codepoints). Scalar input will
  produce a `Tensor` output containing the codepoints. Tensor inputs will
  produce `RaggedTensor` outputs.
  
  Example:
  
  >>> tokenizer = tf_text.UnicodeCharTokenizer()
  >>> tokens = tokenizer.tokenize("abc")
  >>> print(tokens)
  tf.Tensor([97 98 99], shape=(3,), dtype=int32)
  
  >>> tokens = tokenizer.tokenize(["abc", "de"])
  >>> print(tokens)
  <tf.RaggedTensor [[97, 98, 99], [100, 101]]>
  
  Note: any remaining illegal and special UTF-8 characters (like BOM
  characters) in the input string will not be treated specially by the tokenizer
  and show up in the output tokens. These should be normalized out before
  or after tokenization if they are unwanted in the application.
  
  >>> t = ["abc" + chr(0xfffe) + chr(0x1fffe) ]
  >>> tokens = tokenizer.tokenize(t)
  >>> print(tokens.to_list())
  [[97, 98, 99, 65534, 131070]]
  
  Passing malformed UTF-8 will result in unpredictable behavior. Make sure
  inputs conform to UTF-8.
class_docstr_tok: ["Tokenizes","a","tensor","of","UTF-8","strings","on","Unicode","character","boundaries",".","Resulting","tokens","are","integers","(","unicode","codepoints",")",".","Scalar","input","will","produce","a","`","Tensor","`","output","containing","the","codepoints",".","Tensor","inputs","will","produce","`","RaggedTensor","`","outputs",".","Example",":",">",">",">","tokenizer","=","tf_text.UnicodeCharTokenizer","(",")",">",">",">","tokens","=","tokenizer.tokenize","(","``","abc","''",")",">",">",">","print","(","tokens",")","tf.Tensor","(","[","97","98","99","]",",","shape=","(","3",",",")",",","dtype=int32",")",">",">",">","tokens","=","tokenizer.tokenize","(","[","``","abc","''",",","``","de","''","]",")",">",">",">","print","(","tokens",")","<","tf.RaggedTensor","[","[","97",",","98",",","99","]",",","[","100",",","101","]","]",">","Note",":","any","remaining","illegal","and","special","UTF-8","characters","(","like","BOM","characters",")","in","the","input","string","will","not","be","treated","specially","by","the","tokenizer","and","show","up","in","the","output","tokens",".","These","should","be","normalized","out","before","or","after","tokenization","if","they","are","unwanted","in","the","application",".",">",">",">","t","=","[","``","abc","''","+","chr","(","0xfffe",")","+","chr","(","0x1fffe",")","]",">",">",">","tokens","=","tokenizer.tokenize","(","t",")",">",">",">","print","(","tokens.to_list","(",")",")","[","[","97",",","98",",","99",",","65534",",","131070","]","]","Passing","malformed","UTF-8","will","result","in","unpredictable","behavior",".","Make","sure","inputs","conform","to","UTF-8","."]
docstr: Tokenizes a tensor of UTF-8 strings to Unicode characters.
  
  Example:
  
  >>> tokenizer = tf_text.UnicodeCharTokenizer()
  >>> tokens = tokenizer.tokenize_with_offsets("a"+chr(8364)+chr(10340))
  >>> print(tokens[0])
  tf.Tensor([   97  8364 10340], shape=(3,), dtype=int32)
  >>> print(tokens[1])
  tf.Tensor([0 1 4], shape=(3,), dtype=int64)
  >>> print(tokens[2])
  tf.Tensor([1 4 7], shape=(3,), dtype=int64)
  
  The `start_offsets` and `end_offsets` are in byte indices of the original
  string. When calling with multiple string inputs, the offset indices will
  be relative to the individual source strings:
  
  >>> toks = tokenizer.tokenize_with_offsets(["a"+chr(8364), "b"+chr(10300) ])
  >>> print(toks[0])
  <tf.RaggedTensor [[97, 8364], [98, 10300]]>
  >>> print(toks[1])
  <tf.RaggedTensor [[0, 1], [0, 1]]>
  >>> print(toks[2])
  <tf.RaggedTensor [[1, 4], [1, 4]]>
  
  Args:
    input: A `RaggedTensor`or `Tensor` of UTF-8 strings with any shape.
  
  Returns:
    A tuple `(tokens, start_offsets, end_offsets)` where:
  
      * `tokens`: A `RaggedTensor` of code points (integer type).
      * `start_offsets`: A `RaggedTensor` of the tokens' starting byte offset.
      * `end_offsets`: A `RaggedTensor` of the tokens' ending byte offset.
docstr_tok: ["Tokenizes","a","tensor","of","UTF-8","strings","to","Unicode","characters",".","Example",":",">",">",">","tokenizer","=","tf_text.UnicodeCharTokenizer","(",")",">",">",">","tokens","=","tokenizer.tokenize_with_offsets","(","``","a","''","+chr","(","8364",")","+chr","(","10340",")",")",">",">",">","print","(","tokens","[","0","]",")","tf.Tensor","(","[","97","8364","10340","]",",","shape=","(","3",",",")",",","dtype=int32",")",">",">",">","print","(","tokens","[","1","]",")","tf.Tensor","(","[","0","1","4","]",",","shape=","(","3",",",")",",","dtype=int64",")",">",">",">","print","(","tokens","[","2","]",")","tf.Tensor","(","[","1","4","7","]",",","shape=","(","3",",",")",",","dtype=int64",")","The","`","start_offsets","`","and","`","end_offsets","`","are","in","byte","indices","of","the","original","string",".","When","calling","with","multiple","string","inputs",",","the","offset","indices","will","be","relative","to","the","individual","source","strings",":",">",">",">","toks","=","tokenizer.tokenize_with_offsets","(","[","``","a","''","+chr","(","8364",")",",","``","b","''","+chr","(","10300",")","]",")",">",">",">","print","(","toks","[","0","]",")","<","tf.RaggedTensor","[","[","97",",","8364","]",",","[","98",",","10300","]","]",">",">",">",">","print","(","toks","[","1","]",")","<","tf.RaggedTensor","[","[","0",",","1","]",",","[","0",",","1","]","]",">",">",">",">","print","(","toks","[","2","]",")","<","tf.RaggedTensor","[","[","1",",","4","]",",","[","1",",","4","]","]",">","Args",":","input",":","A","`","RaggedTensor","`","or","`","Tensor","`","of","UTF-8","strings","with","any","shape",".","Returns",":","A","tuple","`","(","tokens",",","start_offsets",",","end_offsets",")","`","where",":","*","`","tokens","`",":","A","`","RaggedTensor","`","of","code","points","(","integer","type",")",".","*","`","start_offsets","`",":","A","`","RaggedTensor","`","of","the","tokens","'","starting","byte","offset",".","*","`","end_offsets","`",":","A","`","RaggedTensor","`","of","the","tokens","'","ending","byte","offset","."]
code: def tokenize_with_offsets(
      self, input
  ):  # pylint: disable=redefined-builtin
      """Tokenizes a tensor of UTF-8 strings to Unicode characters.
  
      Example:
  
      >>> tokenizer = tf_text.UnicodeCharTokenizer()
      >>> tokens = tokenizer.tokenize_with_offsets("a"+chr(8364)+chr(10340))
      >>> print(tokens[0])
      tf.Tensor([   97  8364 10340], shape=(3,), dtype=int32)
      >>> print(tokens[1])
      tf.Tensor([0 1 4], shape=(3,), dtype=int64)
      >>> print(tokens[2])
      tf.Tensor([1 4 7], shape=(3,), dtype=int64)
  
      The `start_offsets` and `end_offsets` are in byte indices of the original
      string. When calling with multiple string inputs, the offset indices will
      be relative to the individual source strings:
  
      >>> toks = tokenizer.tokenize_with_offsets(["a"+chr(8364), "b"+chr(10300) ])
      >>> print(toks[0])
      <tf.RaggedTensor [[97, 8364], [98, 10300]]>
      >>> print(toks[1])
      <tf.RaggedTensor [[0, 1], [0, 1]]>
      >>> print(toks[2])
      <tf.RaggedTensor [[1, 4], [1, 4]]>
  
      Args:
        input: A `RaggedTensor`or `Tensor` of UTF-8 strings with any shape.
  
      Returns:
        A tuple `(tokens, start_offsets, end_offsets)` where:
  
          * `tokens`: A `RaggedTensor` of code points (integer type).
          * `start_offsets`: A `RaggedTensor` of the tokens' starting byte offset.
          * `end_offsets`: A `RaggedTensor` of the tokens' ending byte offset.
      """
      name = None
      with ops.name_scope(name, "UnicodeCharTokenize", [input]):
          input_tensor = (
              ragged_tensor.convert_to_tensor_or_ragged_tensor(input)
          )
          (
              codepoints,
              byte_start_offsets,
          ) = ragged_string_ops.unicode_decode_with_offsets(
              input_tensor, "UTF-8"
          )
          strlens = math_ops.cast(
              array_ops.expand_dims(
                  string_ops.string_length(input_tensor), -1
              ),
              dtypes.int64,
          )
          # Adjust strlens to set 0-length strings to empty array (there will be no
          # tokens in this case).
          final_ends = ragged_array_ops.boolean_mask(
              strlens, strlens > 0
          )
          byte_end_offsets = array_ops.concat(
              [byte_start_offsets[..., 1:], final_ends], -1
          )
          return codepoints, byte_start_offsets, byte_end_offsets
  
imports: from __future__ import absolute_import
  from __future__ import division
  from __future__ import print_function
  from tensorflow.python.eager import monitoring
  from tensorflow.python.framework import dtypes
  from tensorflow.python.framework import ops
  from tensorflow.python.ops import array_ops
  from tensorflow.python.ops import math_ops
  from tensorflow.python.ops import string_ops
  from tensorflow.python.ops.ragged import ragged_array_ops
  from tensorflow.python.ops.ragged import ragged_string_ops
  from tensorflow.python.ops.ragged import ragged_tensor
  from tensorflow_text.python.ops.tokenization import Detokenizer
  from tensorflow_text.python.ops.tokenization import TokenizerWithOffsets
label_desc: Use image node_id 3 for calling the UnicodeCharTokenizer obj's underlying member method code with example usage: obj.tokenize_with_offsets(input) and returns: codepoints,  byte_start_offsets,  byte_end_offsets