class_docstr: Runs WordPiece tokenziation.
class_docstr_tok: ["Runs","WordPiece","tokenziation","."]
docstr: Tokenizes a piece of text into its word pieces. This uses a greedy longest-match-first algorithm to perform tokenization
  using the given vocabulary.
  
  For example:
      input = "unaffable"
      output = ["un", "##aff", "##able"]
  
  Args:
      text: A single token or whitespace separated tokens. This should have already been passed through `BasicTokenizer`.
  Returns:
      output_tokens: A list of wordpiece tokens.
      current_positions: A list of the current positions for the original words in text .
docstr_tok: ["Tokenizes","a","piece","of","text","into","its","word","pieces",".","This","uses","a","greedy","longest-match-first","algorithm","to","perform","tokenization","using","the","given","vocabulary",".","For","example",":","input","=","``","unaffable","''","output","=","[","``","un","''",",","``","#","#","aff","''",",","``","#","#","able","''","]","Args",":","text",":","A","single","token","or","whitespace","separated","tokens",".","This","should","have","already","been","passed","through","`","BasicTokenizer","`",".","Returns",":","output_tokens",":","A","list","of","wordpiece","tokens",".","current_positions",":","A","list","of","the","current","positions","for","the","original","words","in","text","."]
code: def tokenize(self, text):
      """Tokenizes a piece of text into its word pieces. This uses a greedy longest-match-first algorithm to perform tokenization
      using the given vocabulary.
  
      For example:
          input = "unaffable"
          output = ["un", "##aff", "##able"]
  
      Args:
          text: A single token or whitespace separated tokens. This should have already been passed through `BasicTokenizer`.
      Returns:
          output_tokens: A list of wordpiece tokens.
          current_positions: A list of the current positions for the original words in text .
      """
      text = convert_to_unicode(text)
      text = clean_text(text)
      text = tokenize_chinese_chars(text)
      output_tokens = []
      current_positions = []
      token_list = split_on_whitespace(text)
      for chars in token_list:
          if len(chars) > self.max_input_chars_per_word:
              output_tokens.append(self.unk_token)
              continue
          is_bad = False
          start = 0
          sub_tokens = []
          while start < len(chars):
              end = len(chars)
              if start > 0:
                  substr = "##" + chars[start:end]
              else:
                  substr = chars[start:end]
              cur_substr = None
              while start < end:
                  if substr in self.vocab:
                      cur_substr = substr
                      break
                  end -= 1
                  substr = substr[:-1]
              if cur_substr is None:
                  is_bad = True
                  break
              else:
                  sub_tokens.append(cur_substr)
                  start = end
          current_positions.append([])
          if is_bad:
              current_positions[-1].append(len(output_tokens))
              output_tokens.append(self.unk_token)
              current_positions[-1].append(len(output_tokens))
          else:
              current_positions[-1].append(len(output_tokens))
              output_tokens.extend(sub_tokens)
              current_positions[-1].append(len(output_tokens))
  
      return output_tokens, current_positions
  
file_path: OpenNRE/opennre/tokenization/word_piece_tokenizer.py
imports: from __future__ import absolute_import
  from __future__ import division
  from __future__ import print_function
  import unicodedata
  from .utils import load_vocab, convert_to_unicode, clean_text, split_on_whitespace, convert_by_vocab, tokenize_chinese_chars
label_desc: Use image node_id 2 for calling the WordpieceTokenizer obj's underlying member method code with example usage: obj.tokenize(text) and returns: output_tokens,  current_positions