class_docstr: Runs basic tokenization (punctuation splitting, lower casing, etc.).
class_docstr_tok: ["Runs","basic","tokenization","(","punctuation","splitting",",","lower","casing",",","etc",".",")","."]
docstr: Performs invalid character removal and whitespace cleanup on text.
docstr_tok: ["Performs","invalid","character","removal","and","whitespace","cleanup","on","text","."]
code: def _clean_text(self, text: str):
      """Performs invalid character removal and whitespace cleanup on text."""
      output = []
      for char in text:
          cp = ord(char)
          if cp == 0 or cp == 0xFFFD or is_control(char):
              continue
          if is_whitespace(char):
              output.append(" ")
          else:
              output.append(char)
      return "".join(output)
  
imports: import collections
  import os
  import pickle
  import unicodedata
  from typing import Dict, List, Optional, Union, Tuple
  from paddle.utils import try_import
  from paddlehub.text.utils import load_vocab, is_whitespace, is_control, is_punctuation, whitespace_tokenize, is_chinese_char
label_desc: Use image node_id 6 for calling the BasicTokenizer obj's underlying member method code with example usage: obj._clean_text(text) and returns: str