code: def test_pre_tokenization(self):
      tokenizer = CpmAntTokenizer.from_pretrained("openbmb/cpm-ant-10b")
      texts = "今天天气真好！"
      jieba_tokens = ["今天", "天气", "真", "好", "！"]
      tokens = tokenizer.tokenize(texts)
      self.assertListEqual(tokens, jieba_tokens)
      normalized_text = "今天天气真好！"
      input_tokens = [tokenizer.bos_token] + tokens
  
      input_jieba_tokens = [6, 9802, 14962, 2082, 831, 244]
      self.assertListEqual(
          tokenizer.convert_tokens_to_ids(input_tokens),
          input_jieba_tokens,
      )
  
      reconstructed_text = tokenizer.decode(input_jieba_tokens)
      self.assertEqual(reconstructed_text, normalized_text)
  
file_path: transformers/tests/models/cpmant/test_tokenization_cpmant.py
imports: import os
  import unittest
  from transformers.models.cpmant.tokenization_cpmant import VOCAB_FILES_NAMES, CpmAntTokenizer
  from transformers.testing_utils import require_jieba, tooslow
  from ...test_tokenization_common import TokenizerTesterMixin
label_desc: Use image node_id 2 for calling the CPMAntTokenizationTest obj's underlying member method code with example usage: obj.test_pre_tokenization() without return types