class_docstr: Runs end-to-end tokenziation.
class_docstr_tok: ["Runs","end-to-end","tokenziation","."]
code: def __init__(
      self,
      vocab_file: str,
      do_lower_case: bool = True,
      use_sentence_piece_vocab: bool = False,
  ):
      self.vocab = load_vocab(vocab_file)
      self.inv_vocab = {v: k for k, v in self.vocab.items()}
      self.basic_tokenizer = BasicTokenizer(do_lower_case=do_lower_case)
      self.use_sentence_piece_vocab = use_sentence_piece_vocab
      self.wordpiece_tokenizer = WordpieceTokenizer(
          vocab=self.vocab,
          use_sentence_piece_vocab=self.use_sentence_piece_vocab,
      )
  
imports: import collections
  import io
  import pickle
  import unicodedata
  from typing import List, Union
label_desc: Use image node_id 1 to create a new FullTokenizer object from inherited base classes: object with example: obj = FullTokenizer(vocab_file, do_lower_case, use_sentence_piece_vocab)