class_docstr: Runs basic tokenization (punctuation splitting, lower casing, etc.).
  
  Args:
      do_lower_case: Whether to lower case the input.
      never_split: A list of tokens not to split.
      tokenize_chinese_chars: Whether to tokenize Chinese characters.
          This should likely be deactivated for Japanese:
          see:
          `https://github.com/huggingface/pytorch-pretrained-BERT/issues/328`
class_docstr_tok: ["Runs","basic","tokenization","(","punctuation","splitting",",","lower","casing",",","etc.",")",".","Args",":","do_lower_case",":","Whether","to","lower","case","the","input",".","never_split",":","A","list","of","tokens","not","to","split",".","tokenize_chinese_chars",":","Whether","to","tokenize","Chinese","characters",".","This","should","likely","be","deactivated","for","Japanese",":","see",":","`","https",":","\/\/github.com\/huggingface\/pytorch-pretrained-BERT\/issues\/328","`"]
docstr: Splits punctuation on a piece of text.
  
  Example:
      text = 'Texar-PyTorch is an open-source toolkit based on PyTorch.'
      _run_split_on_punc(text)
      # ['Texar', '-', 'PyTorch is an open', '-',
      # 'source toolkit based on PyTorch', '.']
docstr_tok: ["Splits","punctuation","on","a","piece","of","text",".","Example",":","text","=","'Texar-PyTorch","is","an","open-source","toolkit","based","on","PyTorch",".","'","_run_split_on_punc","(","text",")","#","[","'Texar","'",",","'-","'",",","'PyTorch","is","an","open","'",",","'-","'",",","#","'source","toolkit","based","on","PyTorch","'",",","'",".","'","]"]
code: def _run_split_on_punc(
      cls, text: str, never_split: Optional[List[str]] = None
  ) -> List[str]:
      r"""Splits punctuation on a piece of text.
  
      Example:
          text = 'Texar-PyTorch is an open-source toolkit based on PyTorch.'
          _run_split_on_punc(text)
          # ['Texar', '-', 'PyTorch is an open', '-',
          # 'source toolkit based on PyTorch', '.']
      """
      if never_split is not None and text in never_split:
          return [text]
      chars = list(text)
      i = 0
      start_new_word = True
      output = []
      while i < len(chars):
          char = chars[i]
          if _is_punctuation(char):
              output.append([char])
              start_new_word = True
          else:
              if start_new_word:
                  output.append([])
              start_new_word = False
              output[-1].append(char)
          i += 1
  
      return ["".join(x) for x in output]
  
file_path: texar/texar/tf/data/tokenizers/bert_tokenizer_utils.py
imports: from typing import Dict, List, Optional
  import collections
  import unicodedata
label_desc: Use image node_id 4 for calling the BasicTokenizer obj's underlying member method code with example usage: obj._run_split_on_punc(cls, text, never_split) and returns: unknown, list