class_docstr: Tokenizes a tensor of UTF-8 string tokens into subword pieces.
  
  Each UTF-8 string token in the input is split into its corresponding
  wordpieces, drawing from the list in the file `vocab_lookup_table`.
  
  Algorithm summary: For each token, the longest token prefix that is in the
  vocabulary is split off. Any part of the token that remains is prefixed using
  the `suffix_indicator`, and the process of removing the longest token prefix
  continues. The `unknown_token` (UNK) is used when what remains of the token is
  not in the vocabulary, or if the token is too long.
  
  When `token_out_type` is tf.string, the output tensor contains strings
  in the vocabulary (or UNK). When it is an integer type, the output tensor
  contains indices into the vocabulary list (with UNK being after the last
  entry).
  
  Example:
  >>> import pathlib
  >>> pathlib.Path('/tmp/tok_vocab.txt').write_text(
  ...   "they ##' ##re the great ##est".replace(' ', '\n'))
  >>> tokenizer = WordpieceTokenizer('/tmp/tok_vocab.txt',
  ...   token_out_type=tf.string)
  
  >>> tokenizer.tokenize(["they're", "the", "greatest"])
  <tf.RaggedTensor [[b'they', b"##'", b'##re'], [b'the'], [b'great', b'##est']]>
  
  >>> tokenizer.tokenize(["they", "are", "great"])
  <tf.RaggedTensor [[b'they'], [b'[UNK]'], [b'great']]>
  
  >>> int_tokenizer = WordpieceTokenizer('/tmp/tok_vocab.txt',
  ...   token_out_type=tf.int32)
  
  >>> int_tokenizer.tokenize(["the", "greatest"])
  <tf.RaggedTensor [[3], [4, 5]]>
  
  >>> int_tokenizer.tokenize(["really", "the", "greatest"])
  <tf.RaggedTensor [[6], [3], [4, 5]]>
  
  Tensor or ragged tensor inputs result in ragged tensor outputs. Scalar
  inputs (which are just a single token) result in tensor outputs.
  
  >>> tokenizer.tokenize("they're")
  <tf.Tensor: shape=(3,), dtype=string, numpy=array([b'they', b"##'", b'##re'],
  dtype=object)>
  >>> tokenizer.tokenize(["they're"])
  <tf.RaggedTensor [[b'they', b"##'", b'##re']]>
  >>> tokenizer.tokenize(tf.ragged.constant([["they're"]]))
  <tf.RaggedTensor [[[b'they', b"##'", b'##re']]]>
  
  Empty strings are tokenized into empty (ragged) tensors.
  
  >>> tokenizer.tokenize([""])
  <tf.RaggedTensor [[]]>
class_docstr_tok: ["Tokenizes","a","tensor","of","UTF-8","string","tokens","into","subword","pieces",".","Each","UTF-8","string","token","in","the","input","is","split","into","its","corresponding","wordpieces",",","drawing","from","the","list","in","the","file","`","vocab_lookup_table","`",".","Algorithm","summary",":","For","each","token",",","the","longest","token","prefix","that","is","in","the","vocabulary","is","split","off",".","Any","part","of","the","token","that","remains","is","prefixed","using","the","`","suffix_indicator","`",",","and","the","process","of","removing","the","longest","token","prefix","continues",".","The","`","unknown_token","`","(","UNK",")","is","used","when","what","remains","of","the","token","is","not","in","the","vocabulary",",","or","if","the","token","is","too","long",".","When","`","token_out_type","`","is","tf.string",",","the","output","tensor","contains","strings","in","the","vocabulary","(","or","UNK",")",".","When","it","is","an","integer","type",",","the","output","tensor","contains","indices","into","the","vocabulary","list","(","with","UNK","being","after","the","last","entry",")",".","Example",":",">",">",">","import","pathlib",">",">",">","pathlib.Path","(","'\/tmp\/tok_vocab.txt","'",")",".write_text","(","...","``","they","#","#","'","#","#","re","the","great","#","#","est","''",".replace","(","'","'",",","'\\n","'",")",")",">",">",">","tokenizer","=","WordpieceTokenizer","(","'\/tmp\/tok_vocab.txt","'",",","...","token_out_type=tf.string",")",">",">",">","tokenizer.tokenize","(","[","``","they","'re","''",",","``","the","''",",","``","greatest","''","]",")","<","tf.RaggedTensor","[","[","b'they","'",",","b","''","#","#","'","''",",","b","'","#","#","re","'","]",",","[","b'the","'","]",",","[","b'great","'",",","b","'","#","#","est","'","]","]",">",">",">",">","tokenizer.tokenize","(","[","``","they","''",",","``","are","''",",","``","great","''","]",")","<","tf.RaggedTensor","[","[","b'they","'","]",",","[","b","'","[","UNK","]","'","]",",","[","b'great","'","]","]",">",">",">",">","int_tokenizer","=","WordpieceTokenizer","(","'\/tmp\/tok_vocab.txt","'",",","...","token_out_type=tf.int32",")",">",">",">","int_tokenizer.tokenize","(","[","``","the","''",",","``","greatest","''","]",")","<","tf.RaggedTensor","[","[","3","]",",","[","4",",","5","]","]",">",">",">",">","int_tokenizer.tokenize","(","[","``","really","''",",","``","the","''",",","``","greatest","''","]",")","<","tf.RaggedTensor","[","[","6","]",",","[","3","]",",","[","4",",","5","]","]",">","Tensor","or","ragged","tensor","inputs","result","in","ragged","tensor","outputs",".","Scalar","inputs","(","which","are","just","a","single","token",")","result","in","tensor","outputs",".",">",">",">","tokenizer.tokenize","(","``","they","'re","''",")","<","tf.Tensor",":","shape=","(","3",",",")",",","dtype=string",",","numpy=array","(","[","b'they","'",",","b","''","#","#","'","''",",","b","'","#","#","re","'","]",",","dtype=object",")",">",">",">",">","tokenizer.tokenize","(","[","``","they","'re","''","]",")","<","tf.RaggedTensor","[","[","b'they","'",",","b","''","#","#","'","''",",","b","'","#","#","re","'","]","]",">",">",">",">","tokenizer.tokenize","(","tf.ragged.constant","(","[","[","``","they","'re","''","]","]",")",")","<","tf.RaggedTensor","[","[","[","b'they","'",",","b","''","#","#","'","''",",","b","'","#","#","re","'","]","]","]",">","Empty","strings","are","tokenized","into","empty","(","ragged",")","tensors",".",">",">",">","tokenizer.tokenize","(","[","``","''","]",")","<","tf.RaggedTensor","[","[","]","]",">"]
docstr: Initializes the WordpieceTokenizer.
  
  Args:
    vocab_lookup_table: A lookup table implementing the LookupInterface
      containing the vocabulary of subwords or a string which is the file path
      to the vocab.txt file.
    suffix_indicator: (optional) The characters prepended to a wordpiece to
      indicate that it is a suffix to another subword. Default is '##'.
    max_bytes_per_word: (optional) Max size of input token. Default is 100.
    max_chars_per_token: (optional) Max size of subwords, excluding suffix
      indicator. If known, providing this improves the efficiency of decoding
      long words.
    token_out_type: (optional) The type of the token to return. This can be
      `tf.int64` or `tf.int32` IDs, or `tf.string` subwords. The default is
      `tf.int64`.
    unknown_token: (optional) The string value to substitute for an unknown
      token. Default is "[UNK]". If set to `None`, no substitution occurs.
      If `token_out_type` is `tf.int32`/`tf.int64`, the `vocab_lookup_table`
      is used (after substitution) to convert the unknown token to an integer.
    split_unknown_characters: (optional) Whether to split out single unknown
      characters as subtokens. If False (default), words containing unknown
      characters will be treated as single unknown tokens.
docstr_tok: ["Initializes","the","WordpieceTokenizer",".","Args",":","vocab_lookup_table",":","A","lookup","table","implementing","the","LookupInterface","containing","the","vocabulary","of","subwords","or","a","string","which","is","the","file","path","to","the","vocab.txt","file",".","suffix_indicator",":","(","optional",")","The","characters","prepended","to","a","wordpiece","to","indicate","that","it","is","a","suffix","to","another","subword",".","Default","is","'","#","#","'",".","max_bytes_per_word",":","(","optional",")","Max","size","of","input","token",".","Default","is","100.","max_chars_per_token",":","(","optional",")","Max","size","of","subwords",",","excluding","suffix","indicator",".","If","known",",","providing","this","improves","the","efficiency","of","decoding","long","words",".","token_out_type",":","(","optional",")","The","type","of","the","token","to","return",".","This","can","be","`","tf.int64","`","or","`","tf.int32","`","IDs",",","or","`","tf.string","`","subwords",".","The","default","is","`","tf.int64","`",".","unknown_token",":","(","optional",")","The","string","value","to","substitute","for","an","unknown","token",".","Default","is","``","[","UNK","]","''",".","If","set","to","`","None","`",",","no","substitution","occurs",".","If","`","token_out_type","`","is","`","tf.int32","`","\/","`","tf.int64","`",",","the","`","vocab_lookup_table","`","is","used","(","after","substitution",")","to","convert","the","unknown","token","to","an","integer",".","split_unknown_characters",":","(","optional",")","Whether","to","split","out","single","unknown","characters","as","subtokens",".","If","False","(","default",")",",","words","containing","unknown","characters","will","be","treated","as","single","unknown","tokens","."]
code: def __init__(
      self,
      vocab_lookup_table,
      suffix_indicator="##",
      max_bytes_per_word=100,
      max_chars_per_token=None,
      token_out_type=dtypes.int64,
      unknown_token="[UNK]",
      split_unknown_characters=False,
  ):
      """Initializes the WordpieceTokenizer.
  
      Args:
        vocab_lookup_table: A lookup table implementing the LookupInterface
          containing the vocabulary of subwords or a string which is the file path
          to the vocab.txt file.
        suffix_indicator: (optional) The characters prepended to a wordpiece to
          indicate that it is a suffix to another subword. Default is '##'.
        max_bytes_per_word: (optional) Max size of input token. Default is 100.
        max_chars_per_token: (optional) Max size of subwords, excluding suffix
          indicator. If known, providing this improves the efficiency of decoding
          long words.
        token_out_type: (optional) The type of the token to return. This can be
          `tf.int64` or `tf.int32` IDs, or `tf.string` subwords. The default is
          `tf.int64`.
        unknown_token: (optional) The string value to substitute for an unknown
          token. Default is "[UNK]". If set to `None`, no substitution occurs.
          If `token_out_type` is `tf.int32`/`tf.int64`, the `vocab_lookup_table`
          is used (after substitution) to convert the unknown token to an integer.
        split_unknown_characters: (optional) Whether to split out single unknown
          characters as subtokens. If False (default), words containing unknown
          characters will be treated as single unknown tokens.
      """
      super(WordpieceTokenizer, self).__init__()
      _tf_text_wordpiece_tokenizer_op_create_counter.get_cell().increase_by(
          1
      )
  
      if isinstance(vocab_lookup_table, str) or (
          isinstance(vocab_lookup_table, tensor.Tensor)
          and vocab_lookup_table.dtype == dtypes.string
      ):
          init = lookup_ops.TextFileIdTableInitializer(
              vocab_lookup_table
          )
          vocab_lookup_table = lookup_ops.StaticVocabularyTableV1(
              init, num_oov_buckets=1, lookup_key_dtype=dtypes.string
          )
  
      if not isinstance(vocab_lookup_table, lookup_ops.LookupInterface):
          raise TypeError(
              "Unable to build a lookup table from {}".format(
                  vocab_lookup_table
              )
          )
  
      self._vocab_lookup_table = vocab_lookup_table
      self._suffix_indicator = suffix_indicator
      self._max_bytes_per_word = max_bytes_per_word
      self._max_chars_per_token = (
          0 if max_chars_per_token is None else max_chars_per_token
      )
      self._token_out_type = token_out_type
      self._unknown_token = unknown_token if unknown_token else "[UNK]"
      self._use_unknown_token = True if unknown_token else False
      self._split_unknown_characters = split_unknown_characters
  
imports: from __future__ import absolute_import
  from __future__ import division
  from __future__ import print_function
  import re
  from tensorflow.python.compat import compat
  from tensorflow.python.eager import monitoring
  from tensorflow.python.framework import dtypes
  from tensorflow.python.framework import ops
  from tensorflow.python.framework import tensor
  from tensorflow.python.ops import array_ops
  from tensorflow.python.ops import array_ops_stack
  from tensorflow.python.ops import check_ops
  from tensorflow.python.ops import control_flow_assert
  from tensorflow.python.ops import lookup_ops
  from tensorflow.python.ops import math_ops
  from tensorflow.python.ops import sort_ops
  from tensorflow.python.ops import string_ops
  from tensorflow.python.ops.ragged import ragged_string_ops
  from tensorflow.python.ops.ragged import ragged_tensor
  from tensorflow.python.ops.ragged.ragged_tensor import RaggedTensor
  from tensorflow_text.python.ops.tokenization import Detokenizer
  from tensorflow_text.python.ops.tokenization import TokenizerWithOffsets
  from tensorflow.python.framework import load_library
  from tensorflow.python.platform import resource_loader
label_desc: Use image node_id 1 to create a new WordpieceTokenizer object from inherited base classes: TokenizerWithOffsets, Detokenizer with example: obj = WordpieceTokenizer(vocab_lookup_table, suffix_indicator, max_bytes_per_word, max_chars_per_token, token_out_type, unknown_token, split_unknown_characters)