class_docstr: Tokenizes a tensor of UTF-8 string tokens into subword pieces.
  
  Each UTF-8 string token in the input is split into its corresponding
  wordpieces, drawing from the list in the file `vocab_lookup_table`.
  
  Algorithm summary: For each token, the longest token prefix that is in the
  vocabulary is split off. Any part of the token that remains is prefixed using
  the `suffix_indicator`, and the process of removing the longest token prefix
  continues. The `unknown_token` (UNK) is used when what remains of the token is
  not in the vocabulary, or if the token is too long.
  
  When `token_out_type` is tf.string, the output tensor contains strings
  in the vocabulary (or UNK). When it is an integer type, the output tensor
  contains indices into the vocabulary list (with UNK being after the last
  entry).
  
  Example:
  >>> import pathlib
  >>> pathlib.Path('/tmp/tok_vocab.txt').write_text(
  ...   "they ##' ##re the great ##est".replace(' ', '\n'))
  >>> tokenizer = WordpieceTokenizer('/tmp/tok_vocab.txt',
  ...   token_out_type=tf.string)
  
  >>> tokenizer.tokenize(["they're", "the", "greatest"])
  <tf.RaggedTensor [[b'they', b"##'", b'##re'], [b'the'], [b'great', b'##est']]>
  
  >>> tokenizer.tokenize(["they", "are", "great"])
  <tf.RaggedTensor [[b'they'], [b'[UNK]'], [b'great']]>
  
  >>> int_tokenizer = WordpieceTokenizer('/tmp/tok_vocab.txt',
  ...   token_out_type=tf.int32)
  
  >>> int_tokenizer.tokenize(["the", "greatest"])
  <tf.RaggedTensor [[3], [4, 5]]>
  
  >>> int_tokenizer.tokenize(["really", "the", "greatest"])
  <tf.RaggedTensor [[6], [3], [4, 5]]>
  
  Tensor or ragged tensor inputs result in ragged tensor outputs. Scalar
  inputs (which are just a single token) result in tensor outputs.
  
  >>> tokenizer.tokenize("they're")
  <tf.Tensor: shape=(3,), dtype=string, numpy=array([b'they', b"##'", b'##re'],
  dtype=object)>
  >>> tokenizer.tokenize(["they're"])
  <tf.RaggedTensor [[b'they', b"##'", b'##re']]>
  >>> tokenizer.tokenize(tf.ragged.constant([["they're"]]))
  <tf.RaggedTensor [[[b'they', b"##'", b'##re']]]>
  
  Empty strings are tokenized into empty (ragged) tensors.
  
  >>> tokenizer.tokenize([""])
  <tf.RaggedTensor [[]]>
class_docstr_tok: ["Tokenizes","a","tensor","of","UTF-8","string","tokens","into","subword","pieces",".","Each","UTF-8","string","token","in","the","input","is","split","into","its","corresponding","wordpieces",",","drawing","from","the","list","in","the","file","`","vocab_lookup_table","`",".","Algorithm","summary",":","For","each","token",",","the","longest","token","prefix","that","is","in","the","vocabulary","is","split","off",".","Any","part","of","the","token","that","remains","is","prefixed","using","the","`","suffix_indicator","`",",","and","the","process","of","removing","the","longest","token","prefix","continues",".","The","`","unknown_token","`","(","UNK",")","is","used","when","what","remains","of","the","token","is","not","in","the","vocabulary",",","or","if","the","token","is","too","long",".","When","`","token_out_type","`","is","tf.string",",","the","output","tensor","contains","strings","in","the","vocabulary","(","or","UNK",")",".","When","it","is","an","integer","type",",","the","output","tensor","contains","indices","into","the","vocabulary","list","(","with","UNK","being","after","the","last","entry",")",".","Example",":",">",">",">","import","pathlib",">",">",">","pathlib.Path","(","'\/tmp\/tok_vocab.txt","'",")",".write_text","(","...","``","they","#","#","'","#","#","re","the","great","#","#","est","''",".replace","(","'","'",",","'\\n","'",")",")",">",">",">","tokenizer","=","WordpieceTokenizer","(","'\/tmp\/tok_vocab.txt","'",",","...","token_out_type=tf.string",")",">",">",">","tokenizer.tokenize","(","[","``","they","'re","''",",","``","the","''",",","``","greatest","''","]",")","<","tf.RaggedTensor","[","[","b'they","'",",","b","''","#","#","'","''",",","b","'","#","#","re","'","]",",","[","b'the","'","]",",","[","b'great","'",",","b","'","#","#","est","'","]","]",">",">",">",">","tokenizer.tokenize","(","[","``","they","''",",","``","are","''",",","``","great","''","]",")","<","tf.RaggedTensor","[","[","b'they","'","]",",","[","b","'","[","UNK","]","'","]",",","[","b'great","'","]","]",">",">",">",">","int_tokenizer","=","WordpieceTokenizer","(","'\/tmp\/tok_vocab.txt","'",",","...","token_out_type=tf.int32",")",">",">",">","int_tokenizer.tokenize","(","[","``","the","''",",","``","greatest","''","]",")","<","tf.RaggedTensor","[","[","3","]",",","[","4",",","5","]","]",">",">",">",">","int_tokenizer.tokenize","(","[","``","really","''",",","``","the","''",",","``","greatest","''","]",")","<","tf.RaggedTensor","[","[","6","]",",","[","3","]",",","[","4",",","5","]","]",">","Tensor","or","ragged","tensor","inputs","result","in","ragged","tensor","outputs",".","Scalar","inputs","(","which","are","just","a","single","token",")","result","in","tensor","outputs",".",">",">",">","tokenizer.tokenize","(","``","they","'re","''",")","<","tf.Tensor",":","shape=","(","3",",",")",",","dtype=string",",","numpy=array","(","[","b'they","'",",","b","''","#","#","'","''",",","b","'","#","#","re","'","]",",","dtype=object",")",">",">",">",">","tokenizer.tokenize","(","[","``","they","'re","''","]",")","<","tf.RaggedTensor","[","[","b'they","'",",","b","''","#","#","'","''",",","b","'","#","#","re","'","]","]",">",">",">",">","tokenizer.tokenize","(","tf.ragged.constant","(","[","[","``","they","'re","''","]","]",")",")","<","tf.RaggedTensor","[","[","[","b'they","'",",","b","''","#","#","'","''",",","b","'","#","#","re","'","]","]","]",">","Empty","strings","are","tokenized","into","empty","(","ragged",")","tensors",".",">",">",">","tokenizer.tokenize","(","[","``","''","]",")","<","tf.RaggedTensor","[","[","]","]",">"]
docstr: Tokenizes a tensor of UTF-8 string tokens further into subword tokens.
  
  ### Example:
  
  >>> import pathlib
  >>> pathlib.Path('/tmp/tok_vocab.txt').write_text(
  ...     "they ##' ##re the great ##est".replace(' ', '\n'))
  >>> tokens = [["they're", 'the', 'greatest']]
  >>> tokenizer = WordpieceTokenizer('/tmp/tok_vocab.txt',
  ...                                token_out_type=tf.string)
  >>> tokenizer.tokenize(tokens)
  <tf.RaggedTensor [[[b'they', b"##'", b'##re'], [b'the'],
                     [b'great', b'##est']]]>
  
  Args:
    input: An N-dimensional `Tensor` or `RaggedTensor` of UTF-8 strings.
  
  Returns:
    A `RaggedTensor` of tokens where `tokens[i1...iN, j]` is the string
    contents (or ID in the vocab_lookup_table representing that string)
    of the `jth` token in `input[i1...iN]`
docstr_tok: ["Tokenizes","a","tensor","of","UTF-8","string","tokens","further","into","subword","tokens",".","#","#","#","Example",":",">",">",">","import","pathlib",">",">",">","pathlib.Path","(","'\/tmp\/tok_vocab.txt","'",")",".write_text","(","...","``","they","#","#","'","#","#","re","the","great","#","#","est","''",".replace","(","'","'",",","'\\n","'",")",")",">",">",">","tokens","=","[","[","``","they","'re","''",",","'the","'",",","'greatest","'","]","]",">",">",">","tokenizer","=","WordpieceTokenizer","(","'\/tmp\/tok_vocab.txt","'",",","...","token_out_type=tf.string",")",">",">",">","tokenizer.tokenize","(","tokens",")","<","tf.RaggedTensor","[","[","[","b'they","'",",","b","''","#","#","'","''",",","b","'","#","#","re","'","]",",","[","b'the","'","]",",","[","b'great","'",",","b","'","#","#","est","'","]","]","]",">","Args",":","input",":","An","N-dimensional","`","Tensor","`","or","`","RaggedTensor","`","of","UTF-8","strings",".","Returns",":","A","`","RaggedTensor","`","of","tokens","where","`","tokens","[","i1","...","iN",",","j","]","`","is","the","string","contents","(","or","ID","in","the","vocab_lookup_table","representing","that","string",")","of","the","`","jth","`","token","in","`","input","[","i1","...","iN","]","`"]
code: def tokenize(self, input):  # pylint: disable=redefined-builtin
      r"""Tokenizes a tensor of UTF-8 string tokens further into subword tokens.
  
      ### Example:
  
      >>> import pathlib
      >>> pathlib.Path('/tmp/tok_vocab.txt').write_text(
      ...     "they ##' ##re the great ##est".replace(' ', '\n'))
      >>> tokens = [["they're", 'the', 'greatest']]
      >>> tokenizer = WordpieceTokenizer('/tmp/tok_vocab.txt',
      ...                                token_out_type=tf.string)
      >>> tokenizer.tokenize(tokens)
      <tf.RaggedTensor [[[b'they', b"##'", b'##re'], [b'the'],
                         [b'great', b'##est']]]>
  
      Args:
        input: An N-dimensional `Tensor` or `RaggedTensor` of UTF-8 strings.
  
      Returns:
        A `RaggedTensor` of tokens where `tokens[i1...iN, j]` is the string
        contents (or ID in the vocab_lookup_table representing that string)
        of the `jth` token in `input[i1...iN]`
      """
      subword, _, _ = self.tokenize_with_offsets(input)
      return subword
  
imports: from __future__ import absolute_import
  from __future__ import division
  from __future__ import print_function
  import re
  from tensorflow.python.compat import compat
  from tensorflow.python.eager import monitoring
  from tensorflow.python.framework import dtypes
  from tensorflow.python.framework import ops
  from tensorflow.python.framework import tensor
  from tensorflow.python.ops import array_ops
  from tensorflow.python.ops import array_ops_stack
  from tensorflow.python.ops import check_ops
  from tensorflow.python.ops import control_flow_assert
  from tensorflow.python.ops import lookup_ops
  from tensorflow.python.ops import math_ops
  from tensorflow.python.ops import sort_ops
  from tensorflow.python.ops import string_ops
  from tensorflow.python.ops.ragged import ragged_string_ops
  from tensorflow.python.ops.ragged import ragged_tensor
  from tensorflow.python.ops.ragged.ragged_tensor import RaggedTensor
  from tensorflow_text.python.ops.tokenization import Detokenizer
  from tensorflow_text.python.ops.tokenization import TokenizerWithOffsets
  from tensorflow.python.framework import load_library
  from tensorflow.python.platform import resource_loader
label_desc: Use image node_id 4 for calling the WordpieceTokenizer obj's underlying member method code with example usage: obj.tokenize(input) and returns: subword