class_docstr: Runs pre-tokenization with Jieba segmentation tool. It is used in CPM models.
class_docstr_tok: ["Runs","pre-tokenization","with","Jieba","segmentation","tool",".","It","is","used","in","CPM","models","."]
code: def _batch_encode_plus(
      self, batch_text_or_text_pairs, *args, **kwargs
  ):
      batch_text_or_text_pairs = [
          " ".join(
              [
                  x.translate(self.translator)
                  for x in self.jieba.cut(text, cut_all=False)
              ]
          )
          for text in batch_text_or_text_pairs
      ]
      return super()._batch_encode_plus(
          batch_text_or_text_pairs, *args, **kwargs
      )
  
file_path: H2O/h2o_flexgen/benchmark/third_party/transformers/src/transformers/models/cpm/tokenization_cpm_fast.py
imports: import os
  from shutil import copyfile
  from typing import List, Optional, Tuple
  from ...tokenization_utils_fast import AddedToken, PreTrainedTokenizerFast
  from ...utils import logging
label_desc: Use image node_id 5 for calling the CpmTokenizerFast obj's underlying member method code with example usage: obj._batch_encode_plus(batch_text_or_text_pairs) and returns: super