class_docstr: Tokenizes a tensor of UTF-8 string tokens into subword pieces.
  
  Each UTF-8 string token in the input is split into its corresponding
  wordpieces, drawing from the list in the file `vocab_lookup_table`.
  
  Algorithm summary: For each token, the longest token prefix that is in the
  vocabulary is split off. Any part of the token that remains is prefixed using
  the `suffix_indicator`, and the process of removing the longest token prefix
  continues. The `unknown_token` (UNK) is used when what remains of the token is
  not in the vocabulary, or if the token is too long.
  
  When `token_out_type` is tf.string, the output tensor contains strings
  in the vocabulary (or UNK). When it is an integer type, the output tensor
  contains indices into the vocabulary list (with UNK being after the last
  entry).
  
  Example:
  >>> import pathlib
  >>> pathlib.Path('/tmp/tok_vocab.txt').write_text(
  ...   "they ##' ##re the great ##est".replace(' ', '\n'))
  >>> tokenizer = WordpieceTokenizer('/tmp/tok_vocab.txt',
  ...   token_out_type=tf.string)
  
  >>> tokenizer.tokenize(["they're", "the", "greatest"])
  <tf.RaggedTensor [[b'they', b"##'", b'##re'], [b'the'], [b'great', b'##est']]>
  
  >>> tokenizer.tokenize(["they", "are", "great"])
  <tf.RaggedTensor [[b'they'], [b'[UNK]'], [b'great']]>
  
  >>> int_tokenizer = WordpieceTokenizer('/tmp/tok_vocab.txt',
  ...   token_out_type=tf.int32)
  
  >>> int_tokenizer.tokenize(["the", "greatest"])
  <tf.RaggedTensor [[3], [4, 5]]>
  
  >>> int_tokenizer.tokenize(["really", "the", "greatest"])
  <tf.RaggedTensor [[6], [3], [4, 5]]>
  
  Tensor or ragged tensor inputs result in ragged tensor outputs. Scalar
  inputs (which are just a single token) result in tensor outputs.
  
  >>> tokenizer.tokenize("they're")
  <tf.Tensor: shape=(3,), dtype=string, numpy=array([b'they', b"##'", b'##re'],
  dtype=object)>
  >>> tokenizer.tokenize(["they're"])
  <tf.RaggedTensor [[b'they', b"##'", b'##re']]>
  >>> tokenizer.tokenize(tf.ragged.constant([["they're"]]))
  <tf.RaggedTensor [[[b'they', b"##'", b'##re']]]>
  
  Empty strings are tokenized into empty (ragged) tensors.
  
  >>> tokenizer.tokenize([""])
  <tf.RaggedTensor [[]]>
class_docstr_tok: ["Tokenizes","a","tensor","of","UTF-8","string","tokens","into","subword","pieces",".","Each","UTF-8","string","token","in","the","input","is","split","into","its","corresponding","wordpieces",",","drawing","from","the","list","in","the","file","`","vocab_lookup_table","`",".","Algorithm","summary",":","For","each","token",",","the","longest","token","prefix","that","is","in","the","vocabulary","is","split","off",".","Any","part","of","the","token","that","remains","is","prefixed","using","the","`","suffix_indicator","`",",","and","the","process","of","removing","the","longest","token","prefix","continues",".","The","`","unknown_token","`","(","UNK",")","is","used","when","what","remains","of","the","token","is","not","in","the","vocabulary",",","or","if","the","token","is","too","long",".","When","`","token_out_type","`","is","tf.string",",","the","output","tensor","contains","strings","in","the","vocabulary","(","or","UNK",")",".","When","it","is","an","integer","type",",","the","output","tensor","contains","indices","into","the","vocabulary","list","(","with","UNK","being","after","the","last","entry",")",".","Example",":",">",">",">","import","pathlib",">",">",">","pathlib.Path","(","'\/tmp\/tok_vocab.txt","'",")",".write_text","(","...","``","they","#","#","'","#","#","re","the","great","#","#","est","''",".replace","(","'","'",",","'\\n","'",")",")",">",">",">","tokenizer","=","WordpieceTokenizer","(","'\/tmp\/tok_vocab.txt","'",",","...","token_out_type=tf.string",")",">",">",">","tokenizer.tokenize","(","[","``","they","'re","''",",","``","the","''",",","``","greatest","''","]",")","<","tf.RaggedTensor","[","[","b'they","'",",","b","''","#","#","'","''",",","b","'","#","#","re","'","]",",","[","b'the","'","]",",","[","b'great","'",",","b","'","#","#","est","'","]","]",">",">",">",">","tokenizer.tokenize","(","[","``","they","''",",","``","are","''",",","``","great","''","]",")","<","tf.RaggedTensor","[","[","b'they","'","]",",","[","b","'","[","UNK","]","'","]",",","[","b'great","'","]","]",">",">",">",">","int_tokenizer","=","WordpieceTokenizer","(","'\/tmp\/tok_vocab.txt","'",",","...","token_out_type=tf.int32",")",">",">",">","int_tokenizer.tokenize","(","[","``","the","''",",","``","greatest","''","]",")","<","tf.RaggedTensor","[","[","3","]",",","[","4",",","5","]","]",">",">",">",">","int_tokenizer.tokenize","(","[","``","really","''",",","``","the","''",",","``","greatest","''","]",")","<","tf.RaggedTensor","[","[","6","]",",","[","3","]",",","[","4",",","5","]","]",">","Tensor","or","ragged","tensor","inputs","result","in","ragged","tensor","outputs",".","Scalar","inputs","(","which","are","just","a","single","token",")","result","in","tensor","outputs",".",">",">",">","tokenizer.tokenize","(","``","they","'re","''",")","<","tf.Tensor",":","shape=","(","3",",",")",",","dtype=string",",","numpy=array","(","[","b'they","'",",","b","''","#","#","'","''",",","b","'","#","#","re","'","]",",","dtype=object",")",">",">",">",">","tokenizer.tokenize","(","[","``","they","'re","''","]",")","<","tf.RaggedTensor","[","[","b'they","'",",","b","''","#","#","'","''",",","b","'","#","#","re","'","]","]",">",">",">",">","tokenizer.tokenize","(","tf.ragged.constant","(","[","[","``","they","'re","''","]","]",")",")","<","tf.RaggedTensor","[","[","[","b'they","'",",","b","''","#","#","'","''",",","b","'","#","#","re","'","]","]","]",">","Empty","strings","are","tokenized","into","empty","(","ragged",")","tensors",".",">",">",">","tokenizer.tokenize","(","[","``","''","]",")","<","tf.RaggedTensor","[","[","]","]",">"]
docstr: Tokenizes a tensor of UTF-8 string tokens further into subword tokens.
  
  ### Example:
  
  >>> import pathlib
  >>> pathlib.Path('/tmp/tok_vocab.txt').write_text(
  ...     "they ##' ##re the great ##est".replace(' ', '\n'))
  >>> tokens = [["they're", 'the', 'greatest']]
  >>> tokenizer = WordpieceTokenizer('/tmp/tok_vocab.txt',
  ...                                token_out_type=tf.string)
  >>> subtokens, starts, ends = tokenizer.tokenize_with_offsets(tokens)
  >>> subtokens
  <tf.RaggedTensor [[[b'they', b"##'", b'##re'], [b'the'],
                     [b'great', b'##est']]]>
  >>> starts
  <tf.RaggedTensor [[[0, 4, 5], [0], [0, 5]]]>
  >>> ends
  <tf.RaggedTensor [[[4, 5, 7], [3], [5, 8]]]>
  
  Args:
    input: An N-dimensional `Tensor` or `RaggedTensor` of UTF-8 strings.
  
  Returns:
    A tuple `(tokens, start_offsets, end_offsets)` where:
  
    tokens[i1...iN, j]: is a `RaggedTensor` of the string contents (or ID
      in the vocab_lookup_table representing that string) of the `jth` token
      in `input[i1...iN]`.
    start_offsets[i1...iN, j]: is a `RaggedTensor` of the byte offsets
      for the inclusive start of the `jth` token in `input[i1...iN]`.
    end_offsets[i1...iN, j]: is a `RaggedTensor` of the byte offsets for
      the exclusive end of the `jth` token in `input[i`...iN]` (exclusive,
      i.e., first byte after the end of the token).
docstr_tok: ["Tokenizes","a","tensor","of","UTF-8","string","tokens","further","into","subword","tokens",".","#","#","#","Example",":",">",">",">","import","pathlib",">",">",">","pathlib.Path","(","'\/tmp\/tok_vocab.txt","'",")",".write_text","(","...","``","they","#","#","'","#","#","re","the","great","#","#","est","''",".replace","(","'","'",",","'\\n","'",")",")",">",">",">","tokens","=","[","[","``","they","'re","''",",","'the","'",",","'greatest","'","]","]",">",">",">","tokenizer","=","WordpieceTokenizer","(","'\/tmp\/tok_vocab.txt","'",",","...","token_out_type=tf.string",")",">",">",">","subtokens",",","starts",",","ends","=","tokenizer.tokenize_with_offsets","(","tokens",")",">",">",">","subtokens","<","tf.RaggedTensor","[","[","[","b'they","'",",","b","''","#","#","'","''",",","b","'","#","#","re","'","]",",","[","b'the","'","]",",","[","b'great","'",",","b","'","#","#","est","'","]","]","]",">",">",">",">","starts","<","tf.RaggedTensor","[","[","[","0",",","4",",","5","]",",","[","0","]",",","[","0",",","5","]","]","]",">",">",">",">","ends","<","tf.RaggedTensor","[","[","[","4",",","5",",","7","]",",","[","3","]",",","[","5",",","8","]","]","]",">","Args",":","input",":","An","N-dimensional","`","Tensor","`","or","`","RaggedTensor","`","of","UTF-8","strings",".","Returns",":","A","tuple","`","(","tokens",",","start_offsets",",","end_offsets",")","`","where",":","tokens","[","i1","...","iN",",","j","]",":","is","a","`","RaggedTensor","`","of","the","string","contents","(","or","ID","in","the","vocab_lookup_table","representing","that","string",")","of","the","`","jth","`","token","in","`","input","[","i1","...","iN","]","`",".","start_offsets","[","i1","...","iN",",","j","]",":","is","a","`","RaggedTensor","`","of","the","byte","offsets","for","the","inclusive","start","of","the","`","jth","`","token","in","`","input","[","i1","...","iN","]","`",".","end_offsets","[","i1","...","iN",",","j","]",":","is","a","`","RaggedTensor","`","of","the","byte","offsets","for","the","exclusive","end","of","the","`","jth","`","token","in","`","input","[","i","`","...","iN","]","`","(","exclusive",",","i.e.",",","first","byte","after","the","end","of","the","token",")","."]
code: def tokenize_with_offsets(
      self, input
  ):  # pylint: disable=redefined-builtin
      r"""Tokenizes a tensor of UTF-8 string tokens further into subword tokens.
  
      ### Example:
  
      >>> import pathlib
      >>> pathlib.Path('/tmp/tok_vocab.txt').write_text(
      ...     "they ##' ##re the great ##est".replace(' ', '\n'))
      >>> tokens = [["they're", 'the', 'greatest']]
      >>> tokenizer = WordpieceTokenizer('/tmp/tok_vocab.txt',
      ...                                token_out_type=tf.string)
      >>> subtokens, starts, ends = tokenizer.tokenize_with_offsets(tokens)
      >>> subtokens
      <tf.RaggedTensor [[[b'they', b"##'", b'##re'], [b'the'],
                         [b'great', b'##est']]]>
      >>> starts
      <tf.RaggedTensor [[[0, 4, 5], [0], [0, 5]]]>
      >>> ends
      <tf.RaggedTensor [[[4, 5, 7], [3], [5, 8]]]>
  
      Args:
        input: An N-dimensional `Tensor` or `RaggedTensor` of UTF-8 strings.
  
      Returns:
        A tuple `(tokens, start_offsets, end_offsets)` where:
  
        tokens[i1...iN, j]: is a `RaggedTensor` of the string contents (or ID
          in the vocab_lookup_table representing that string) of the `jth` token
          in `input[i1...iN]`.
        start_offsets[i1...iN, j]: is a `RaggedTensor` of the byte offsets
          for the inclusive start of the `jth` token in `input[i1...iN]`.
        end_offsets[i1...iN, j]: is a `RaggedTensor` of the byte offsets for
          the exclusive end of the `jth` token in `input[i`...iN]` (exclusive,
          i.e., first byte after the end of the token).
      """
      name = None
      if not isinstance(
          self._vocab_lookup_table, lookup_ops.LookupInterface
      ):
          raise TypeError(
              "vocab_lookup_table must be a LookupInterface"
          )
      with ops.name_scope(
          name,
          "WordpieceTokenizeWithOffsets",
          [input, self._vocab_lookup_table, self._suffix_indicator],
      ):
          # Check that the types are expected and the ragged rank is appropriate.
          tokens = ragged_tensor.convert_to_tensor_or_ragged_tensor(
              input
          )
          rank = tokens.shape.ndims
          if rank is None:
              raise ValueError("input must have a known rank.")
  
          if rank == 0:
              wordpieces, starts, ends = self.tokenize_with_offsets(
                  array_ops_stack.stack([tokens])
              )
              return wordpieces.values, starts.values, ends.values
  
          elif rank > 1:
              if not ragged_tensor.is_ragged(tokens):
                  tokens = ragged_tensor.RaggedTensor.from_tensor(
                      tokens, ragged_rank=rank - 1
                  )
              wordpieces, starts, ends = self.tokenize_with_offsets(
                  tokens.flat_values
              )
              wordpieces = wordpieces.with_row_splits_dtype(
                  tokens.row_splits.dtype
              )
              starts = starts.with_row_splits_dtype(
                  tokens.row_splits.dtype
              )
              ends = ends.with_row_splits_dtype(tokens.row_splits.dtype)
              return (
                  tokens.with_flat_values(wordpieces),
                  tokens.with_flat_values(starts),
                  tokens.with_flat_values(ends),
              )
  
          if compat.forward_compatible(2019, 8, 25):
              kwargs = dict(output_row_partition_type="row_splits")
              from_row_partition = RaggedTensor.from_row_splits
          else:
              kwargs = {}
              from_row_partition = RaggedTensor.from_row_lengths
  
          # Tokenize the tokens into subwords
          (
              values,
              row_splits,
              starts,
              ends,
          ) = gen_wordpiece_tokenizer.wordpiece_tokenize_with_offsets(
              input_values=tokens,
              vocab_lookup_table=self._vocab_lookup_table.resource_handle,
              suffix_indicator=self._suffix_indicator,
              use_unknown_token=self._use_unknown_token,
              max_bytes_per_word=self._max_bytes_per_word,
              max_chars_per_token=self._max_chars_per_token,
              unknown_token=self._unknown_token,
              split_unknown_characters=self._split_unknown_characters,
              **kwargs,
          )
  
          # If ids are desired, look them up in the vocab table. Otherwise just
          # return the string values.
          if self._token_out_type == dtypes.int64:
              values = math_ops.cast(
                  self._vocab_lookup_table.lookup(values), dtypes.int64
              )
  
          if self._token_out_type == dtypes.int32:
              values = math_ops.cast(
                  self._vocab_lookup_table.lookup(values), dtypes.int32
              )
  
          wordpieces = from_row_partition(
              values, row_splits, validate=False
          )
          starts = from_row_partition(
              starts, row_splits, validate=False
          )
          ends = from_row_partition(ends, row_splits, validate=False)
  
          return wordpieces, starts, ends
  
imports: from __future__ import absolute_import
  from __future__ import division
  from __future__ import print_function
  import re
  from tensorflow.python.compat import compat
  from tensorflow.python.eager import monitoring
  from tensorflow.python.framework import dtypes
  from tensorflow.python.framework import ops
  from tensorflow.python.framework import tensor
  from tensorflow.python.ops import array_ops
  from tensorflow.python.ops import array_ops_stack
  from tensorflow.python.ops import check_ops
  from tensorflow.python.ops import control_flow_assert
  from tensorflow.python.ops import lookup_ops
  from tensorflow.python.ops import math_ops
  from tensorflow.python.ops import sort_ops
  from tensorflow.python.ops import string_ops
  from tensorflow.python.ops.ragged import ragged_string_ops
  from tensorflow.python.ops.ragged import ragged_tensor
  from tensorflow.python.ops.ragged.ragged_tensor import RaggedTensor
  from tensorflow_text.python.ops.tokenization import Detokenizer
  from tensorflow_text.python.ops.tokenization import TokenizerWithOffsets
  from tensorflow.python.framework import load_library
  from tensorflow.python.platform import resource_loader
label_desc: Use image node_id 5 for calling the WordpieceTokenizer obj's underlying member method code with example usage: obj.tokenize_with_offsets(input) and returns: wordpieces,  starts,  ends, wordpieces,  starts,  ends, tokens,  tokens,  tokens