class_docstr: Runs basic tokenization (punctuation splitting, lower casing, etc.).
  
  Args:
      do_lower_case: Whether to lower case the input.
      never_split: A list of tokens not to split.
      tokenize_chinese_chars: Whether to tokenize Chinese characters.
          This should likely be deactivated for Japanese:
          see:
          `https://github.com/huggingface/pytorch-pretrained-BERT/issues/328`
class_docstr_tok: ["Runs","basic","tokenization","(","punctuation","splitting",",","lower","casing",",","etc.",")",".","Args",":","do_lower_case",":","Whether","to","lower","case","the","input",".","never_split",":","A","list","of","tokens","not","to","split",".","tokenize_chinese_chars",":","Whether","to","tokenize","Chinese","characters",".","This","should","likely","be","deactivated","for","Japanese",":","see",":","`","https",":","\/\/github.com\/huggingface\/pytorch-pretrained-BERT\/issues\/328","`"]
docstr: Checks whether cp is the codepoint of a CJK character.
docstr_tok: ["Checks","whether","cp","is","the","codepoint","of","a","CJK","character","."]
code: def _is_chinese_char(cls, cp: int) -> bool:
      r"""Checks whether cp is the codepoint of a CJK character."""
      # This defines a "chinese character" as anything in the CJK Unicode
      # block:
      #   https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_(Unicode_block)
      #
      # Note that the CJK Unicode block is NOT all Japanese and Korean
      # characters, despite its name. The modern Korean Hangul alphabet is a
      # different block, as is Japanese Hiragana and Katakana. Those
      # alphabets are used to write space-separated words, so they are not
      # treated specially and handled like the all of the other languages.
      if (
          (0x4E00 <= cp <= 0x9FFF)
          or (0x3400 <= cp <= 0x4DBF)
          or (0x20000 <= cp <= 0x2A6DF)
          or (0x2A700 <= cp <= 0x2B73F)
          or (0x2B740 <= cp <= 0x2B81F)
          or (0x2B820 <= cp <= 0x2CEAF)
          or (0xF900 <= cp <= 0xFAFF)
          or (0x2F800 <= cp <= 0x2FA1F)
      ):
          return True
  
      return False
  
file_path: texar/texar/tf/data/tokenizers/bert_tokenizer_utils.py
imports: from typing import Dict, List, Optional
  import collections
  import unicodedata
label_desc: Use image node_id 6 for calling the BasicTokenizer obj's underlying member method code with example usage: obj._is_chinese_char(cls, cp) and returns: False, True