class_docstr: Tokenizes a tensor of UTF-8 string tokens into subword pieces.
  
  Each UTF-8 string token in the input is split into its corresponding
  wordpieces, drawing from the list in the file `vocab_lookup_table`.
  
  Algorithm summary: For each token, the longest token prefix that is in the
  vocabulary is split off. Any part of the token that remains is prefixed using
  the `suffix_indicator`, and the process of removing the longest token prefix
  continues. The `unknown_token` (UNK) is used when what remains of the token is
  not in the vocabulary, or if the token is too long.
  
  When `token_out_type` is tf.string, the output tensor contains strings
  in the vocabulary (or UNK). When it is an integer type, the output tensor
  contains indices into the vocabulary list (with UNK being after the last
  entry).
  
  Example:
  >>> import pathlib
  >>> pathlib.Path('/tmp/tok_vocab.txt').write_text(
  ...   "they ##' ##re the great ##est".replace(' ', '\n'))
  >>> tokenizer = WordpieceTokenizer('/tmp/tok_vocab.txt',
  ...   token_out_type=tf.string)
  
  >>> tokenizer.tokenize(["they're", "the", "greatest"])
  <tf.RaggedTensor [[b'they', b"##'", b'##re'], [b'the'], [b'great', b'##est']]>
  
  >>> tokenizer.tokenize(["they", "are", "great"])
  <tf.RaggedTensor [[b'they'], [b'[UNK]'], [b'great']]>
  
  >>> int_tokenizer = WordpieceTokenizer('/tmp/tok_vocab.txt',
  ...   token_out_type=tf.int32)
  
  >>> int_tokenizer.tokenize(["the", "greatest"])
  <tf.RaggedTensor [[3], [4, 5]]>
  
  >>> int_tokenizer.tokenize(["really", "the", "greatest"])
  <tf.RaggedTensor [[6], [3], [4, 5]]>
  
  Tensor or ragged tensor inputs result in ragged tensor outputs. Scalar
  inputs (which are just a single token) result in tensor outputs.
  
  >>> tokenizer.tokenize("they're")
  <tf.Tensor: shape=(3,), dtype=string, numpy=array([b'they', b"##'", b'##re'],
  dtype=object)>
  >>> tokenizer.tokenize(["they're"])
  <tf.RaggedTensor [[b'they', b"##'", b'##re']]>
  >>> tokenizer.tokenize(tf.ragged.constant([["they're"]]))
  <tf.RaggedTensor [[[b'they', b"##'", b'##re']]]>
  
  Empty strings are tokenized into empty (ragged) tensors.
  
  >>> tokenizer.tokenize([""])
  <tf.RaggedTensor [[]]>
class_docstr_tok: ["Tokenizes","a","tensor","of","UTF-8","string","tokens","into","subword","pieces",".","Each","UTF-8","string","token","in","the","input","is","split","into","its","corresponding","wordpieces",",","drawing","from","the","list","in","the","file","`","vocab_lookup_table","`",".","Algorithm","summary",":","For","each","token",",","the","longest","token","prefix","that","is","in","the","vocabulary","is","split","off",".","Any","part","of","the","token","that","remains","is","prefixed","using","the","`","suffix_indicator","`",",","and","the","process","of","removing","the","longest","token","prefix","continues",".","The","`","unknown_token","`","(","UNK",")","is","used","when","what","remains","of","the","token","is","not","in","the","vocabulary",",","or","if","the","token","is","too","long",".","When","`","token_out_type","`","is","tf.string",",","the","output","tensor","contains","strings","in","the","vocabulary","(","or","UNK",")",".","When","it","is","an","integer","type",",","the","output","tensor","contains","indices","into","the","vocabulary","list","(","with","UNK","being","after","the","last","entry",")",".","Example",":",">",">",">","import","pathlib",">",">",">","pathlib.Path","(","'\/tmp\/tok_vocab.txt","'",")",".write_text","(","...","``","they","#","#","'","#","#","re","the","great","#","#","est","''",".replace","(","'","'",",","'\\n","'",")",")",">",">",">","tokenizer","=","WordpieceTokenizer","(","'\/tmp\/tok_vocab.txt","'",",","...","token_out_type=tf.string",")",">",">",">","tokenizer.tokenize","(","[","``","they","'re","''",",","``","the","''",",","``","greatest","''","]",")","<","tf.RaggedTensor","[","[","b'they","'",",","b","''","#","#","'","''",",","b","'","#","#","re","'","]",",","[","b'the","'","]",",","[","b'great","'",",","b","'","#","#","est","'","]","]",">",">",">",">","tokenizer.tokenize","(","[","``","they","''",",","``","are","''",",","``","great","''","]",")","<","tf.RaggedTensor","[","[","b'they","'","]",",","[","b","'","[","UNK","]","'","]",",","[","b'great","'","]","]",">",">",">",">","int_tokenizer","=","WordpieceTokenizer","(","'\/tmp\/tok_vocab.txt","'",",","...","token_out_type=tf.int32",")",">",">",">","int_tokenizer.tokenize","(","[","``","the","''",",","``","greatest","''","]",")","<","tf.RaggedTensor","[","[","3","]",",","[","4",",","5","]","]",">",">",">",">","int_tokenizer.tokenize","(","[","``","really","''",",","``","the","''",",","``","greatest","''","]",")","<","tf.RaggedTensor","[","[","6","]",",","[","3","]",",","[","4",",","5","]","]",">","Tensor","or","ragged","tensor","inputs","result","in","ragged","tensor","outputs",".","Scalar","inputs","(","which","are","just","a","single","token",")","result","in","tensor","outputs",".",">",">",">","tokenizer.tokenize","(","``","they","'re","''",")","<","tf.Tensor",":","shape=","(","3",",",")",",","dtype=string",",","numpy=array","(","[","b'they","'",",","b","''","#","#","'","''",",","b","'","#","#","re","'","]",",","dtype=object",")",">",">",">",">","tokenizer.tokenize","(","[","``","they","'re","''","]",")","<","tf.RaggedTensor","[","[","b'they","'",",","b","''","#","#","'","''",",","b","'","#","#","re","'","]","]",">",">",">",">","tokenizer.tokenize","(","tf.ragged.constant","(","[","[","``","they","'re","''","]","]",")",")","<","tf.RaggedTensor","[","[","[","b'they","'",",","b","''","#","#","'","''",",","b","'","#","#","re","'","]","]","]",">","Empty","strings","are","tokenized","into","empty","(","ragged",")","tensors",".",">",">",">","tokenizer.tokenize","(","[","``","''","]",")","<","tf.RaggedTensor","[","[","]","]",">"]
docstr: Convert a `Tensor` or `RaggedTensor` of wordpiece IDs to string-words.
  
  >>> import pathlib
  >>> pathlib.Path('/tmp/detok_vocab.txt').write_text(
  ...     'a b c ##a ##b ##c'.replace(' ', '\n'))
  >>> wordpiece = WordpieceTokenizer('/tmp/detok_vocab.txt')
  >>> token_ids = [[0, 4, 5, 2, 5, 5, 5]]
  >>> wordpiece.detokenize(token_ids)
  <tf.RaggedTensor [[b'abc', b'cccc']]>
  
  The word pieces are joined along the innermost axis to make words. So the
  result has the same rank as the input, but the innermost axis of the result
  indexes words instead of word pieces.
  
  The shape transformation is: `[..., wordpieces] => [..., words]`
  
  When the input shape is `[..., words, wordpieces]` (like the output of
  `WordpieceTokenizer.tokenize`) the result's shape is `[..., words, 1]`.
  The additional ragged axis can be removed using `words.merge_dims(-2, -1)`.
  
  Note: This method assumes wordpiece IDs are dense on the interval
  `[0, vocab_size)`.
  
  Args:
    token_ids: A `RaggedTensor` or `Tensor` with an int dtype. Must have
    `ndims >= 2`
  
  Returns:
    A `RaggedTensor` with dtype `string` and the rank as the input
    `token_ids`.
docstr_tok: ["Convert","a","`","Tensor","`","or","`","RaggedTensor","`","of","wordpiece","IDs","to","string-words",".",">",">",">","import","pathlib",">",">",">","pathlib.Path","(","'\/tmp\/detok_vocab.txt","'",")",".write_text","(","...","'","a","b","c","#","#","a","#","#","b","#","#","c'.replace","(","'","'",",","'\\n","'",")",")",">",">",">","wordpiece","=","WordpieceTokenizer","(","'\/tmp\/detok_vocab.txt","'",")",">",">",">","token_ids","=","[","[","0",",","4",",","5",",","2",",","5",",","5",",","5","]","]",">",">",">","wordpiece.detokenize","(","token_ids",")","<","tf.RaggedTensor","[","[","b'abc","'",",","b'cccc","'","]","]",">","The","word","pieces","are","joined","along","the","innermost","axis","to","make","words",".","So","the","result","has","the","same","rank","as","the","input",",","but","the","innermost","axis","of","the","result","indexes","words","instead","of","word","pieces",".","The","shape","transformation","is",":","`","[","...",",","wordpieces","]","=",">","[","...",",","words","]","`","When","the","input","shape","is","`","[","...",",","words",",","wordpieces","]","`","(","like","the","output","of","`","WordpieceTokenizer.tokenize","`",")","the","result","'s","shape","is","`","[","...",",","words",",","1","]","`",".","The","additional","ragged","axis","can","be","removed","using","`","words.merge_dims","(","-2",",","-1",")","`",".","Note",":","This","method","assumes","wordpiece","IDs","are","dense","on","the","interval","`","[","0",",","vocab_size",")","`",".","Args",":","token_ids",":","A","`","RaggedTensor","`","or","`","Tensor","`","with","an","int","dtype",".","Must","have","`","ndims",">","=","2","`","Returns",":","A","`","RaggedTensor","`","with","dtype","`","string","`","and","the","rank","as","the","input","`","token_ids","`","."]
code: def detokenize(self, token_ids):
      r"""Convert a `Tensor` or `RaggedTensor` of wordpiece IDs to string-words.
  
      >>> import pathlib
      >>> pathlib.Path('/tmp/detok_vocab.txt').write_text(
      ...     'a b c ##a ##b ##c'.replace(' ', '\n'))
      >>> wordpiece = WordpieceTokenizer('/tmp/detok_vocab.txt')
      >>> token_ids = [[0, 4, 5, 2, 5, 5, 5]]
      >>> wordpiece.detokenize(token_ids)
      <tf.RaggedTensor [[b'abc', b'cccc']]>
  
      The word pieces are joined along the innermost axis to make words. So the
      result has the same rank as the input, but the innermost axis of the result
      indexes words instead of word pieces.
  
      The shape transformation is: `[..., wordpieces] => [..., words]`
  
      When the input shape is `[..., words, wordpieces]` (like the output of
      `WordpieceTokenizer.tokenize`) the result's shape is `[..., words, 1]`.
      The additional ragged axis can be removed using `words.merge_dims(-2, -1)`.
  
      Note: This method assumes wordpiece IDs are dense on the interval
      `[0, vocab_size)`.
  
      Args:
        token_ids: A `RaggedTensor` or `Tensor` with an int dtype. Must have
        `ndims >= 2`
  
      Returns:
        A `RaggedTensor` with dtype `string` and the rank as the input
        `token_ids`.
      """
      # If there are performance issues with this method or problems with lookup
      # tables using sparse IDs see the notes in b/177610044.
      vocab, ids = self._get_vocab_and_ids()
      token_ids = ragged_tensor.convert_to_tensor_or_ragged_tensor(
          token_ids
      )
  
      first_is_zero = math_ops.equal(ids[0], 0)
      steps = ids[1:] - ids[:-1]
      all_one_step = math_ops.reduce_all(math_ops.equal(steps, 1))
  
      check = control_flow_assert.Assert(
          first_is_zero & all_one_step,
          data=[
              (
                  "`detokenize` only works with vocabulary tables where the "
                  "indices are dense on the interval `[0, vocab_size)`"
              )
          ],
      )
      with ops.control_dependencies([check]):
          token_ids = math_ops.minimum(
              token_ids,
              # Limit the OOV buckets to a single index.
              math_ops.cast(array_ops.size(vocab), token_ids.dtype),
          )
  
      # Add the unknown token at that index.
      vocab = array_ops.concat([vocab, [self._unknown_token]], axis=0)
  
      # Lookup the text tokens and join them along the innermost axis.
      txt_tokens = array_ops.gather(vocab, token_ids)
  
      # Ensure the input is Ragged.
      if not isinstance(txt_tokens, RaggedTensor):
          txt_tokens = RaggedTensor.from_tensor(txt_tokens)
  
      # Join the tokens along the last axis.
      words = string_ops.reduce_join_v2(
          txt_tokens, axis=-1, separator=" "
      )
  
      # Collapse " ##" in all strings to make words.
      words = string_ops.regex_replace(
          words, " " + re.escape(self._suffix_indicator), ""
      )
  
      # Strip leading and trailing spaces.
      words = string_ops.regex_replace(words, "^ +| +$", "")
  
      # Split on spaces so the last axis is "words".
      words = ragged_string_ops.string_split_v2(words, sep=" ")
      return words
  
imports: from __future__ import absolute_import
  from __future__ import division
  from __future__ import print_function
  import re
  from tensorflow.python.compat import compat
  from tensorflow.python.eager import monitoring
  from tensorflow.python.framework import dtypes
  from tensorflow.python.framework import ops
  from tensorflow.python.framework import tensor
  from tensorflow.python.ops import array_ops
  from tensorflow.python.ops import array_ops_stack
  from tensorflow.python.ops import check_ops
  from tensorflow.python.ops import control_flow_assert
  from tensorflow.python.ops import lookup_ops
  from tensorflow.python.ops import math_ops
  from tensorflow.python.ops import sort_ops
  from tensorflow.python.ops import string_ops
  from tensorflow.python.ops.ragged import ragged_string_ops
  from tensorflow.python.ops.ragged import ragged_tensor
  from tensorflow.python.ops.ragged.ragged_tensor import RaggedTensor
  from tensorflow_text.python.ops.tokenization import Detokenizer
  from tensorflow_text.python.ops.tokenization import TokenizerWithOffsets
  from tensorflow.python.framework import load_library
  from tensorflow.python.platform import resource_loader
label_desc: Use image node_id 6 for calling the WordpieceTokenizer obj's underlying member method code with example usage: obj.detokenize(token_ids) and returns: words