code: def test_tokenize(self):
      self.assertRaises(
          AssertionError, self.test_processor.tokenize, []
      )
      self.assertEqual([], self.test_processor.tokenize(""))
      self.assertEqual(
          [
              "我",
              "们",
              "正",
              "在",
              "玩",
              "乒",
              "乓",
              "球",
              "，",
              "你",
              "真",
              "的",
              "太",
              "懒",
              "了",
              "。",
              "她",
              "打",
              "的",
              "很",
              "漂",
              "亮",
              "。",
          ],
          self.test_processor.tokenize(sents[0], cws=False),
      )
      self.assertEqual(
          [
              "我们",
              "正在",
              "玩",
              "乒乓球",
              "，",
              "你",
              "真的",
              "太",
              "懒",
              "了",
              "。",
              "她",
              "打",
              "的",
              "很",
              "漂亮",
              "。",
          ],
          self.test_processor.tokenize(sents[0], cws=True),
      )
  
file_path: textflint/test/common/preprocess/test_cn_processor.py
imports: import unittest
  from textflint.common.preprocess.cn_processor import CnProcessor
label_desc: Use image node_id 1 for calling the TestCnProcessor obj's underlying member method code with example usage: obj.test_tokenize() without return types