class_docstr: Runs WordPiece tokenziation.
docstr: Tokenizes a piece of text into its word pieces.
  
  This uses a greedy longest-match-first algorithm to perform tokenization
  using the given vocabulary.
  
  For example:
    input = "unaffable"
    output = ["un", "##aff", "##able"]
  
  Args:
    text: A single token or whitespace separated tokens. This should have
      already been passed through `BasicTokenizer.
  
  Returns:
    A list of wordpiece tokens.
docstr_tok: ["Tokenizes","a","piece","of","text","into","its","word","pieces",".","This","uses","a","greedy","longest-match-first","algorithm","to","perform","tokenization","using","the","given","vocabulary",".","For","example",":","input","=","``","unaffable","''","output","=","[","``","un","''",",","``","#","#","aff","''",",","``","#","#","able","''","]","Args",":","text",":","A","single","token","or","whitespace","separated","tokens",".","This","should","have","already","been","passed","through","`","BasicTokenizer",".","Returns",":","A","list","of","wordpiece","tokens","."]
code: def tokenize(self, text):
      """Tokenizes a piece of text into its word pieces.
  
      This uses a greedy longest-match-first algorithm to perform tokenization
      using the given vocabulary.
  
      For example:
        input = "unaffable"
        output = ["un", "##aff", "##able"]
  
      Args:
        text: A single token or whitespace separated tokens. This should have
          already been passed through `BasicTokenizer.
  
      Returns:
        A list of wordpiece tokens.
      """
  
      text = convert_to_unicode(text)
  
      output_tokens = []
      for token in whitespace_tokenize(text):
          chars = list(token)
          if len(chars) > self.max_input_chars_per_word:
              output_tokens.append(self.unk_token)
              continue
  
          is_bad = False
          start = 0
          sub_tokens = []
          while start < len(chars):
              end = len(chars)
              cur_substr = None
              while start < end:
                  substr = "".join(chars[start:end])
                  if start > 0:
                      substr = "##" + substr
                  if substr in self.vocab:
                      cur_substr = substr
                      break
                  end -= 1
              if cur_substr is None:
                  is_bad = True
                  break
              sub_tokens.append(cur_substr)
              start = end
  
          if is_bad:
              output_tokens.append(self.unk_token)
          else:
              output_tokens.extend(sub_tokens)
      return output_tokens
  
imports: from __future__ import absolute_import
  from __future__ import division
  from __future__ import print_function
  import collections
  import re
  import unicodedata
  import six
label_desc: Use image node_id 2 for calling the WordpieceTokenizer obj's underlying member method code with example usage: obj.tokenize(text) and returns: output_tokens