class_docstr: Construct a BaiChuan tokenizer. Based on byte-level Byte-Pair-Encoding.
  
  Args:
      vocab_file (`str`):
          Path to the vocabulary file.
class_docstr_tok: ["Construct","a","BaiChuan","tokenizer",".","Based","on","byte-level","Byte-Pair-Encoding",".","Args",":","vocab_file","(","`","str","`",")",":","Path","to","the","vocabulary","file","."]
code: def __init__(
      self,
      vocab_file,
      unk_token="<unk>",
      bos_token="<s>",
      eos_token="</s>",
      pad_token=None,
      sp_model_kwargs: Optional[Dict[str, Any]] = None,
      add_bos_token=True,
      add_eos_token=False,
      clean_up_tokenization_spaces=False,
      **kwargs,
  ):
      self.sp_model_kwargs = (
          {} if sp_model_kwargs is None else sp_model_kwargs
      )
      bos_token = (
          AddedToken(bos_token, lstrip=False, rstrip=False)
          if isinstance(bos_token, str)
          else bos_token
      )
      eos_token = (
          AddedToken(eos_token, lstrip=False, rstrip=False)
          if isinstance(eos_token, str)
          else eos_token
      )
      unk_token = (
          AddedToken(unk_token, lstrip=False, rstrip=False)
          if isinstance(unk_token, str)
          else unk_token
      )
      pad_token = (
          AddedToken(pad_token, lstrip=False, rstrip=False)
          if isinstance(pad_token, str)
          else pad_token
      )
      self.vocab_file = vocab_file
      self.add_bos_token = add_bos_token
      self.add_eos_token = add_eos_token
      self.sp_model = spm.SentencePieceProcessor(**self.sp_model_kwargs)
      self.sp_model.Load(vocab_file)
      super().__init__(
          bos_token=bos_token,
          eos_token=eos_token,
          unk_token=unk_token,
          pad_token=pad_token,
          add_bos_token=add_bos_token,
          add_eos_token=add_eos_token,
          sp_model_kwargs=self.sp_model_kwargs,
          clean_up_tokenization_spaces=clean_up_tokenization_spaces,
          **kwargs,
      )
  
imports: import os
  from shutil import copyfile
  from typing import Any, Dict, List, Optional, Tuple
  import sentencepiece
  from transformers.tokenization_utils import AddedToken, PreTrainedTokenizer
  from transformers.utils import logging
label_desc: Use image node_id 1 to create a new BaiChuanTokenizer object from inherited base classes: PreTrainedTokenizer with example: obj = BaiChuanTokenizer(vocab_file, unk_token, bos_token, eos_token, pad_token, sp_model_kwargs, add_bos_token, add_eos_token, clean_up_tokenization_spaces)