class_docstr: Constructs a BasicTokenizer that will run basic tokenization (punctuation splitting, lower casing, etc.).
  
  Args:
      do_lower_case (`bool`, *optional*, defaults to `True`):
          Whether or not to lowercase the input when tokenizing.
      never_split (`Iterable`, *optional*):
          Collection of tokens which will never be split during tokenization. Only has an effect when
          `do_basic_tokenize=True`
      tokenize_chinese_chars (`bool`, *optional*, defaults to `True`):
          Whether or not to tokenize Chinese characters.
  
          This should likely be deactivated for Japanese (see this
          [issue](https://github.com/huggingface/transformers/issues/328)).
      strip_accents (`bool`, *optional*):
          Whether or not to strip all accents. If this option is not specified, then it will be determined by the
          value for `lowercase` (as in the original BERT).
      do_split_on_punc (`bool`, *optional*, defaults to `True`):
          In some instances we want to skip the basic punctuation splitting so that later tokenization can capture
          the full context of the words, such as contractions.
class_docstr_tok: ["Constructs","a","BasicTokenizer","that","will","run","basic","tokenization","(","punctuation","splitting",",","lower","casing",",","etc.",")",".","Args",":","do_lower_case","(","`","bool","`",",","*","optional","*",",","defaults","to","`","True","`",")",":","Whether","or","not","to","lowercase","the","input","when","tokenizing",".","never_split","(","`","Iterable","`",",","*","optional","*",")",":","Collection","of","tokens","which","will","never","be","split","during","tokenization",".","Only","has","an","effect","when","`","do_basic_tokenize=True","`","tokenize_chinese_chars","(","`","bool","`",",","*","optional","*",",","defaults","to","`","True","`",")",":","Whether","or","not","to","tokenize","Chinese","characters",".","This","should","likely","be","deactivated","for","Japanese","(","see","this","[","issue","]","(","https",":","\/\/github.com\/huggingface\/transformers\/issues\/328",")",")",".","strip_accents","(","`","bool","`",",","*","optional","*",")",":","Whether","or","not","to","strip","all","accents",".","If","this","option","is","not","specified",",","then","it","will","be","determined","by","the","value","for","`","lowercase","`","(","as","in","the","original","BERT",")",".","do_split_on_punc","(","`","bool","`",",","*","optional","*",",","defaults","to","`","True","`",")",":","In","some","instances","we","want","to","skip","the","basic","punctuation","splitting","so","that","later","tokenization","can","capture","the","full","context","of","the","words",",","such","as","contractions","."]
code: def __init__(
      self,
      do_lower_case=True,
      never_split=None,
      tokenize_chinese_chars=True,
      strip_accents=None,
      do_split_on_punc=True,
  ):
      if never_split is None:
          never_split = []
      self.do_lower_case = do_lower_case
      self.never_split = set(never_split)
      self.tokenize_chinese_chars = tokenize_chinese_chars
      self.strip_accents = strip_accents
      self.do_split_on_punc = do_split_on_punc
  
imports: import collections
  import copy
  import os
  import unicodedata
  from typing import Any, Dict, List, Optional, Tuple
  from ...tokenization_utils import PreTrainedTokenizer, _is_control, _is_punctuation, _is_whitespace
  from ...utils import is_sentencepiece_available, logging
label_desc: Use image node_id 1 to create a new BasicTokenizer object from inherited base classes: object with example: obj = BasicTokenizer(do_lower_case, never_split, tokenize_chinese_chars, strip_accents, do_split_on_punc)