class_docstr: Runs basic tokenization (punctuation splitting, lower casing, etc.).
  
  Args:
      do_lower_case: Whether to lower case the input.
      never_split: A list of tokens not to split.
      tokenize_chinese_chars: Whether to tokenize Chinese characters.
          This should likely be deactivated for Japanese:
          see:
          `https://github.com/huggingface/pytorch-pretrained-BERT/issues/328`
class_docstr_tok: ["Runs","basic","tokenization","(","punctuation","splitting",",","lower","casing",",","etc.",")",".","Args",":","do_lower_case",":","Whether","to","lower","case","the","input",".","never_split",":","A","list","of","tokens","not","to","split",".","tokenize_chinese_chars",":","Whether","to","tokenize","Chinese","characters",".","This","should","likely","be","deactivated","for","Japanese",":","see",":","`","https",":","\/\/github.com\/huggingface\/pytorch-pretrained-BERT\/issues\/328","`"]
docstr: Performs invalid character removal and whitespace cleanup on text.
  
  Example:
      text = 'Texar-PyTorch\tis an open-source\ntoolkit based on PyTorch.'
      _clean_text(text)
      # 'Texar-PyTorch is an open-source toolkit based on PyTorch.'
docstr_tok: ["Performs","invalid","character","removal","and","whitespace","cleanup","on","text",".","Example",":","text","=","'Texar-PyTorch\\tis","an","open-source\\ntoolkit","based","on","PyTorch",".","'","_clean_text","(","text",")","#","'Texar-PyTorch","is","an","open-source","toolkit","based","on","PyTorch",".","'"]
code: def _clean_text(cls, text: str) -> str:
      r"""Performs invalid character removal and whitespace cleanup on text.
  
      Example:
          text = 'Texar-PyTorch\tis an open-source\ntoolkit based on PyTorch.'
          _clean_text(text)
          # 'Texar-PyTorch is an open-source toolkit based on PyTorch.'
      """
      output = []
      for char in text:
          cp = ord(char)
          if cp == 0 or cp == 0xFFFD or _is_control(char):
              continue
          if _is_whitespace(char):
              output.append(" ")
          else:
              output.append(char)
      return "".join(output)
  
file_path: texar/texar/tf/data/tokenizers/bert_tokenizer_utils.py
imports: from typing import Dict, List, Optional
  import collections
  import unicodedata
label_desc: Use image node_id 7 for calling the BasicTokenizer obj's underlying member method code with example usage: obj._clean_text(cls, text) and returns: str