class_docstr: Chinese Text Processor class
class_docstr_tok: ["Chinese","Text","Processor","class"]
docstr: tokenize fiction
  
  :param str sent: the sentence need to be tokenized
  :param bool cws: If cws is True, tokenize sentence at word level. If cws is False, tokenize sentence at char level.
  :return: list.the tokens in it
docstr_tok: ["tokenize","fiction",":","param","str","sent",":","the","sentence","need","to","be","tokenized",":","param","bool","cws",":","If","cws","is","True",",","tokenize","sentence","at","word","level",".","If","cws","is","False",",","tokenize","sentence","at","char","level",".",":","return",":","list.the","tokens","in","it"]
code: def tokenize(sent, cws=True):
      r"""
      tokenize fiction
  
      :param str sent: the sentence need to be tokenized
      :param bool cws: If cws is True, tokenize sentence at word level. If cws is False, tokenize sentence at char level.
      :return: list.the tokens in it
      """
      assert isinstance(sent, str)
      if sent == "":
          return []
      if cws:
          from ltp import LTP
  
          ltp = LTP()
          segment, _ = ltp.seg([sent])
          return segment[0]
      else:
          return [word for word in sent]
  
file_path: textflint/textflint/common/preprocess/cn_processor.py
imports: import threading
  from ..settings import CN_SYNONYM_PATH, CN_ANTONYM_PATH
  from ..utils.install import download_if_needed
label_desc: Use image node_id 3 for calling the CnProcessor obj's underlying member method code with example usage: obj.tokenize(sent, cws) and returns: list, segment, unknown