class_docstr: Process unit for text tokenization.
docstr: Tokenizes a piece of text into its word pieces.
  
  This uses a greedy longest-match-first algorithm to perform
   tokenization using the given vocabulary.
  
  For example:
  >>> input_list = ["unaffable"]
  >>> vocab = {"un": 0, "##aff": 1, "##able":2}
  >>> wordpiece_unit = WordPieceTokenize(vocab)
  >>> output = wordpiece_unit.transform(input_list)
  >>> golden_output = ["un", "##aff", "##able"]
  >>> assert output == golden_output
  
  :param input_: token list.
  
  :return tokens: A list of wordpiece tokens.
docstr_tok: ["Tokenizes","a","piece","of","text","into","its","word","pieces",".","This","uses","a","greedy","longest-match-first","algorithm","to","perform","tokenization","using","the","given","vocabulary",".","For","example",":",">",">",">","input_list","=","[","``","unaffable","''","]",">",">",">","vocab","=","{","``","un","''",":","0",",","``","#","#","aff","''",":","1",",","``","#","#","able","''",":2","}",">",">",">","wordpiece_unit","=","WordPieceTokenize","(","vocab",")",">",">",">","output","=","wordpiece_unit.transform","(","input_list",")",">",">",">","golden_output","=","[","``","un","''",",","``","#","#","aff","''",",","``","#","#","able","''","]",">",">",">","assert","output","==","golden_output",":","param","input_",":","token","list",".",":","return","tokens",":","A","list","of","wordpiece","tokens","."]
code: def transform(self, input_: list) -> list:
      """
      Tokenizes a piece of text into its word pieces.
  
      This uses a greedy longest-match-first algorithm to perform
       tokenization using the given vocabulary.
  
      For example:
      >>> input_list = ["unaffable"]
      >>> vocab = {"un": 0, "##aff": 1, "##able":2}
      >>> wordpiece_unit = WordPieceTokenize(vocab)
      >>> output = wordpiece_unit.transform(input_list)
      >>> golden_output = ["un", "##aff", "##able"]
      >>> assert output == golden_output
  
      :param input_: token list.
  
      :return tokens: A list of wordpiece tokens.
      """
      output_tokens = []
      for token in input_:
          chars = list(token)
          token_length = len(chars)
          if token_length > self.max_input_chars_per_word:
              output_tokens.append(self.unk_token)
              continue
  
          unknown_suffix = False
          start = 0
          sub_tokens = []
          while start < token_length:
              end = token_length
              cur_substr = None
              while start < end:
                  substr = "".join(chars[start:end])
                  if start > 0:
                      substr = "##" + substr
                  if substr in self.vocab:
                      cur_substr = substr
                      break
                  end -= 1
              if cur_substr is None:
                  unknown_suffix = True
                  break
              sub_tokens.append(cur_substr)
              start = end
  
          if unknown_suffix:
              output_tokens.append(self.unk_token)
          else:
              output_tokens.extend(sub_tokens)
      return output_tokens
  
file_path: MatchZoo/matchzoo/preprocessors/units/tokenize.py
imports: import nltk
  from matchzoo.utils.bert_utils import is_chinese_char, whitespace_tokenize, run_split_on_punc
  from .unit import Unit
label_desc: Use image node_id 2 for calling the WordPieceTokenize obj's underlying member method code with example usage: obj.transform(input_) and returns: output_tokens