class_docstr: Runs basic tokenization (punctuation splitting, lower casing, etc.).
  
  Args:
      do_lower_case: Whether to lower case the input.
      never_split: A list of tokens not to split.
      tokenize_chinese_chars: Whether to tokenize Chinese characters.
          This should likely be deactivated for Japanese:
          see:
          `https://github.com/huggingface/pytorch-pretrained-BERT/issues/328`
class_docstr_tok: ["Runs","basic","tokenization","(","punctuation","splitting",",","lower","casing",",","etc.",")",".","Args",":","do_lower_case",":","Whether","to","lower","case","the","input",".","never_split",":","A","list","of","tokens","not","to","split",".","tokenize_chinese_chars",":","Whether","to","tokenize","Chinese","characters",".","This","should","likely","be","deactivated","for","Japanese",":","see",":","`","https",":","\/\/github.com\/huggingface\/pytorch-pretrained-BERT\/issues\/328","`"]
docstr: Basic tokenization of a piece of text.
  
  Split on white spaces only, for sub-word tokenization, see
  WordPieceTokenizer.
  
  Args:
      text: An input string.
      never_split: A list of tokens not to split.
docstr_tok: ["Basic","tokenization","of","a","piece","of","text",".","Split","on","white","spaces","only",",","for","sub-word","tokenization",",","see","WordPieceTokenizer",".","Args",":","text",":","An","input","string",".","never_split",":","A","list","of","tokens","not","to","split","."]
code: def tokenize(
      self, text: str, never_split: Optional[List[str]] = None
  ) -> List[str]:
      r"""Basic tokenization of a piece of text.
  
      Split on white spaces only, for sub-word tokenization, see
      WordPieceTokenizer.
  
      Args:
          text: An input string.
          never_split: A list of tokens not to split.
      """
      never_split = self.never_split + (
          never_split if never_split is not None else []
      )
      text = self._clean_text(text)
  
      # This was added on November 1st, 2018 for the multilingual and Chinese
      # models. This is also applied to the English models now, but it doesn't
      # matter since the English models were not trained on any Chinese data
      # and generally don't have any Chinese data in them (there are Chinese
      # characters in the vocabulary because Wikipedia does have some Chinese
      # words in the English Wikipedia.).
      if self.tokenize_chinese_chars:
          text = self._tokenize_chinese_chars(text)
      # see: https://github.com/google-research/bert/blob/master/
      # tokenization.py#L201
  
      orig_tokens = whitespace_tokenize(text)
      split_tokens = []
      for token in orig_tokens:
          if self.do_lower_case and token not in never_split:
              token = token.lower()
              token = self._run_strip_accents(token)
          split_tokens.extend(self._run_split_on_punc(token))
  
      output_tokens = whitespace_tokenize(" ".join(split_tokens))
      return output_tokens
  
file_path: texar/texar/tf/data/tokenizers/bert_tokenizer_utils.py
imports: from typing import Dict, List, Optional
  import collections
  import unicodedata
label_desc: Use image node_id 2 for calling the BasicTokenizer obj's underlying member method code with example usage: obj.tokenize(text, never_split) and returns: output_tokens