code: def split_sentence(s, tok=False):
      """split sentence with some segmentation rules."""
      if tok:
          s = s.lower()
          s = s.replace("\u2019", "'")
          s = digit_pattern.sub("0", s)
      words = []
      for word in s.split():
          if tok:
              words.extend(split_pattern.split(word))
          else:
              words.append(word)
      words = [w for w in words if w]
      return words
  
file_path: texar/examples/transformer/utils/preprocess.py
imports: import collections
  import re
  import json
  import os
  import numpy
  import pickle
  import argparse
  from io import open
label_desc: Use image node_id 1 for calling a global function with example usage: split_sentence(s, tok) and returns: words