class_docstr: Construct a BaiChuan tokenizer. Based on byte-level Byte-Pair-Encoding.
  
  Args:
      vocab_file (`str`):
          Path to the vocabulary file.
class_docstr_tok: ["Construct","a","BaiChuan","tokenizer",".","Based","on","byte-level","Byte-Pair-Encoding",".","Args",":","vocab_file","(","`","str","`",")",":","Path","to","the","vocabulary","file","."]
docstr: Converts a sequence of tokens (string) in a single string.
docstr_tok: ["Converts","a","sequence","of","tokens","(","string",")","in","a","single","string","."]
code: def convert_tokens_to_string(self, tokens):
      """Converts a sequence of tokens (string) in a single string."""
      current_sub_tokens = []
      out_string = ""
      prev_is_special = False
      for i, token in enumerate(tokens):
          # make sure that special tokens are not decoded using sentencepiece model
          if token in self.all_special_tokens:
              if not prev_is_special and i != 0:
                  out_string += " "
              out_string += (
                  self.sp_model.decode(current_sub_tokens) + token
              )
              prev_is_special = True
              current_sub_tokens = []
          else:
              current_sub_tokens.append(token)
              prev_is_special = False
      out_string += self.sp_model.decode(current_sub_tokens)
      return out_string
  
imports: import os
  from shutil import copyfile
  from typing import Any, Dict, List, Optional, Tuple
  import sentencepiece
  from transformers.tokenization_utils import AddedToken, PreTrainedTokenizer
  from transformers.utils import logging
label_desc: Use image node_id 9 for calling the BaiChuanTokenizer obj's underlying member method code with example usage: obj.convert_tokens_to_string(tokens) and returns: out_string