class_docstr: Constructs a BasicTokenizer that will run basic tokenization (punctuation splitting, lower casing, etc.).
  
  Args:
      do_lower_case (`bool`, *optional*, defaults to `True`):
          Whether or not to lowercase the input when tokenizing.
      never_split (`Iterable`, *optional*):
          Collection of tokens which will never be split during tokenization. Only has an effect when
          `do_basic_tokenize=True`
      tokenize_chinese_chars (`bool`, *optional*, defaults to `True`):
          Whether or not to tokenize Chinese characters.
  
          This should likely be deactivated for Japanese (see this
          [issue](https://github.com/huggingface/transformers/issues/328)).
      strip_accents (`bool`, *optional*):
          Whether or not to strip all accents. If this option is not specified, then it will be determined by the
          value for `lowercase` (as in the original BERT).
      do_split_on_punc (`bool`, *optional*, defaults to `True`):
          In some instances we want to skip the basic punctuation splitting so that later tokenization can capture
          the full context of the words, such as contractions.
class_docstr_tok: ["Constructs","a","BasicTokenizer","that","will","run","basic","tokenization","(","punctuation","splitting",",","lower","casing",",","etc.",")",".","Args",":","do_lower_case","(","`","bool","`",",","*","optional","*",",","defaults","to","`","True","`",")",":","Whether","or","not","to","lowercase","the","input","when","tokenizing",".","never_split","(","`","Iterable","`",",","*","optional","*",")",":","Collection","of","tokens","which","will","never","be","split","during","tokenization",".","Only","has","an","effect","when","`","do_basic_tokenize=True","`","tokenize_chinese_chars","(","`","bool","`",",","*","optional","*",",","defaults","to","`","True","`",")",":","Whether","or","not","to","tokenize","Chinese","characters",".","This","should","likely","be","deactivated","for","Japanese","(","see","this","[","issue","]","(","https",":","\/\/github.com\/huggingface\/transformers\/issues\/328",")",")",".","strip_accents","(","`","bool","`",",","*","optional","*",")",":","Whether","or","not","to","strip","all","accents",".","If","this","option","is","not","specified",",","then","it","will","be","determined","by","the","value","for","`","lowercase","`","(","as","in","the","original","BERT",")",".","do_split_on_punc","(","`","bool","`",",","*","optional","*",",","defaults","to","`","True","`",")",":","In","some","instances","we","want","to","skip","the","basic","punctuation","splitting","so","that","later","tokenization","can","capture","the","full","context","of","the","words",",","such","as","contractions","."]
docstr: Checks whether CP is the codepoint of a CJK character.
docstr_tok: ["Checks","whether","CP","is","the","codepoint","of","a","CJK","character","."]
code: def _is_chinese_char(self, cp):
      """Checks whether CP is the codepoint of a CJK character."""
      # This defines a "chinese character" as anything in the CJK Unicode block:
      #   https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_(Unicode_block)
      #
      # Note that the CJK Unicode block is NOT all Japanese and Korean characters,
      # despite its name. The modern Korean Hangul alphabet is a different block,
      # as is Japanese Hiragana and Katakana. Those alphabets are used to write
      # space-separated words, so they are not treated specially and handled
      # like the all of the other languages.
      if (
          (cp >= 0x4E00 and cp <= 0x9FFF)
          or (cp >= 0x3400 and cp <= 0x4DBF)  #
          or (cp >= 0x20000 and cp <= 0x2A6DF)  #
          or (cp >= 0x2A700 and cp <= 0x2B73F)  #
          or (cp >= 0x2B740 and cp <= 0x2B81F)  #
          or (cp >= 0x2B820 and cp <= 0x2CEAF)  #
          or (cp >= 0xF900 and cp <= 0xFAFF)
          or (cp >= 0x2F800 and cp <= 0x2FA1F)  #
      ):  #
          return True
  
      return False
  
imports: import collections
  import os
  import sys
  import unicodedata
  from typing import Dict, List, Optional, Tuple, Union
  from ...tokenization_utils import AddedToken, PreTrainedTokenizer, _is_control, _is_punctuation, _is_whitespace
  from ...tokenization_utils_base import BatchEncoding, EncodedInput, PreTokenizedInput, TextInput, TextInputPair, TruncationStrategy
  from ...utils import PaddingStrategy, TensorType, add_end_docstrings, logging
label_desc: Use image node_id 6 for calling the BasicTokenizer obj's underlying member method code with example usage: obj._is_chinese_char(cp) and returns: False, True