class_docstr: Runs basic tokenization (punctuation splitting, lower casing, etc.).
class_docstr_tok: ["Runs","basic","tokenization","(","punctuation","splitting",",","lower","casing",",","etc",".",")","."]
docstr: Constructs a BasicTokenizer.
  Args:
      do_lower_case: Whether to lower case the input.
      never_split: (`optional`) list of str
          List of token not to split.
      tokenize_chinese_chars: (`optional`) boolean (default True)
          Whether to tokenize Chinese characters.
          This should likely be deactivated for Japanese:
          see: https://github.com/huggingface/pytorch-pretrained-BERT/issues/328
docstr_tok: ["Constructs","a","BasicTokenizer",".","Args",":","do_lower_case",":","Whether","to","lower","case","the","input",".","never_split",":","(","`","optional","`",")","list","of","str","List","of","token","not","to","split",".","tokenize_chinese_chars",":","(","`","optional","`",")","boolean","(","default","True",")","Whether","to","tokenize","Chinese","characters",".","This","should","likely","be","deactivated","for","Japanese",":","see",":","https",":","\/\/github.com\/huggingface\/pytorch-pretrained-BERT\/issues\/328"]
code: def __init__(
      self,
      do_lower_case: bool = True,
      never_split: List[str] = None,
      tokenize_chinese_chars: bool = True,
  ):
      """Constructs a BasicTokenizer.
      Args:
          do_lower_case: Whether to lower case the input.
          never_split: (`optional`) list of str
              List of token not to split.
          tokenize_chinese_chars: (`optional`) boolean (default True)
              Whether to tokenize Chinese characters.
              This should likely be deactivated for Japanese:
              see: https://github.com/huggingface/pytorch-pretrained-BERT/issues/328
      """
      if never_split is None:
          never_split = []
      self.do_lower_case = do_lower_case
      self.never_split = never_split
      self.tokenize_chinese_chars = tokenize_chinese_chars
  
imports: import collections
  import os
  import pickle
  import unicodedata
  from typing import Dict, List, Optional, Union, Tuple
  from paddle.utils import try_import
  from paddlehub.text.utils import load_vocab, is_whitespace, is_control, is_punctuation, whitespace_tokenize, is_chinese_char
label_desc: Use image node_id 1 to create a new BasicTokenizer object from inherited base classes: object with example: obj = BasicTokenizer(do_lower_case, never_split, tokenize_chinese_chars)