class_docstr: Runs pre-tokenization with Jieba segmentation tool. It is used in CPM models.
class_docstr_tok: ["Runs","pre-tokenization","with","Jieba","segmentation","tool",".","It","is","used","in","CPM","models","."]
docstr: Converts a sequence of tokens (strings for sub-words) in a single string.
docstr_tok: ["Converts","a","sequence","of","tokens","(","strings","for","sub-words",")","in","a","single","string","."]
code: def convert_tokens_to_string(self, tokens):
      """Converts a sequence of tokens (strings for sub-words) in a single string."""
      out_string = (
          "".join(tokens).replace(SPIECE_UNDERLINE, " ").strip()
      )
      return out_string
  
imports: import os
  import unicodedata
  from shutil import copyfile
  from typing import Any, Dict, List, Optional, Tuple
  import sentencepiece
  from ...tokenization_utils import AddedToken, PreTrainedTokenizer
  from ...utils import SPIECE_UNDERLINE, logging
label_desc: Use image node_id 10 for calling the CpmTokenizer obj's underlying member method code with example usage: obj.convert_tokens_to_string(tokens) and returns: out_string