class_docstr: Construct a Llama tokenizer. Based on byte-level Byte-Pair-Encoding.
  
  This uses notably ByteFallback and no normalization.
  
  ```python
  >>> from transformers import LlamaTokenizerFast
  
  >>> tokenizer = LlamaTokenizerFast.from_pretrained("hf-internal-testing/llama-tokenizer")
  >>> tokenizer.encode("Hello this is a test")
  [1, 15043, 445, 338, 263, 1243]
  ```
  
  If you want to change the `bos_token` or the `eos_token`, make sure to specify them when initializing the model, or
  call `tokenizer.update_post_processor()` to make sure that the post-processing is correctly done (otherwise the
  values of the first token and final token of an encoded sequence will not be correct). For more details, checkout
  [post-processors] (https://huggingface.co/docs/tokenizers/api/post-processors) documentation.
  
  
  This tokenizer inherits from [`PreTrainedTokenizerFast`] which contains most of the main methods. Users should
  refer to this superclass for more information regarding those methods.
  
  Args:
      vocab_file (`str`, *optional*):
          [SentencePiece](https://github.com/google/sentencepiece) file (generally has a .model extension) that
          contains the vocabulary necessary to instantiate a tokenizer.
      tokenizer_file (`str`, *optional*):
          [tokenizers](https://github.com/huggingface/tokenizers) file (generally has a .json extension) that
          contains everything needed to load the tokenizer.
      clean_up_tokenization_spaces (`bool`, *optional*, defaults to `False`):
          Whether or not to cleanup spaces after decoding, cleanup consists in removing potential artifacts like
          extra spaces.
      unk_token (`str` or `tokenizers.AddedToken`, *optional*, defaults to `"<unk>"`):
          The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
          token instead.
      bos_token (`str` or `tokenizers.AddedToken`, *optional*, defaults to `"<s>"`):
          The beginning of sequence token that was used during pretraining. Can be used a sequence classifier token.
      eos_token (`str` or `tokenizers.AddedToken`, *optional*, defaults to `"</s>"`):
          The end of sequence token.
      add_bos_token (`bool`, *optional*, defaults to `True`):
          Whether or not to add an `bos_token` at the start of sequences.
      add_eos_token (`bool`, *optional*, defaults to `False`):
          Whether or not to add an `eos_token` at the end of sequences.
      use_default_system_prompt (`bool`, *optional*, defaults to `False`):
          Whether or not the default system prompt for Llama should be used.
class_docstr_tok: ["Construct","a","Llama","tokenizer",".","Based","on","byte-level","Byte-Pair-Encoding",".","This","uses","notably","ByteFallback","and","no","normalization",".","``","`","python",">",">",">","from","transformers","import","LlamaTokenizerFast",">",">",">","tokenizer","=","LlamaTokenizerFast.from_pretrained","(","``","hf-internal-testing\/llama-tokenizer","''",")",">",">",">","tokenizer.encode","(","``","Hello","this","is","a","test","''",")","[","1",",","15043",",","445",",","338",",","263",",","1243","]","``","`","If","you","want","to","change","the","`","bos_token","`","or","the","`","eos_token","`",",","make","sure","to","specify","them","when","initializing","the","model",",","or","call","`","tokenizer.update_post_processor","(",")","`","to","make","sure","that","the","post-processing","is","correctly","done","(","otherwise","the","values","of","the","first","token","and","final","token","of","an","encoded","sequence","will","not","be","correct",")",".","For","more","details",",","checkout","[","post-processors","]","(","https",":","\/\/huggingface.co\/docs\/tokenizers\/api\/post-processors",")","documentation",".","This","tokenizer","inherits","from","[","`","PreTrainedTokenizerFast","`","]","which","contains","most","of","the","main","methods",".","Users","should","refer","to","this","superclass","for","more","information","regarding","those","methods",".","Args",":","vocab_file","(","`","str","`",",","*","optional","*",")",":","[","SentencePiece","]","(","https",":","\/\/github.com\/google\/sentencepiece",")","file","(","generally","has","a",".model","extension",")","that","contains","the","vocabulary","necessary","to","instantiate","a","tokenizer",".","tokenizer_file","(","`","str","`",",","*","optional","*",")",":","[","tokenizers","]","(","https",":","\/\/github.com\/huggingface\/tokenizers",")","file","(","generally","has","a",".json","extension",")","that","contains","everything","needed","to","load","the","tokenizer",".","clean_up_tokenization_spaces","(","`","bool","`",",","*","optional","*",",","defaults","to","`","False","`",")",":","Whether","or","not","to","cleanup","spaces","after","decoding",",","cleanup","consists","in","removing","potential","artifacts","like","extra","spaces",".","unk_token","(","`","str","`","or","`","tokenizers.AddedToken","`",",","*","optional","*",",","defaults","to","`","``","<","unk",">","''","`",")",":","The","unknown","token",".","A","token","that","is","not","in","the","vocabulary","can","not","be","converted","to","an","ID","and","is","set","to","be","this","token","instead",".","bos_token","(","`","str","`","or","`","tokenizers.AddedToken","`",",","*","optional","*",",","defaults","to","`","``","<","s",">","''","`",")",":","The","beginning","of","sequence","token","that","was","used","during","pretraining",".","Can","be","used","a","sequence","classifier","token",".","eos_token","(","`","str","`","or","`","tokenizers.AddedToken","`",",","*","optional","*",",","defaults","to","`","``","<","\/s",">","''","`",")",":","The","end","of","sequence","token",".","add_bos_token","(","`","bool","`",",","*","optional","*",",","defaults","to","`","True","`",")",":","Whether","or","not","to","add","an","`","bos_token","`","at","the","start","of","sequences",".","add_eos_token","(","`","bool","`",",","*","optional","*",",","defaults","to","`","False","`",")",":","Whether","or","not","to","add","an","`","eos_token","`","at","the","end","of","sequences",".","use_default_system_prompt","(","`","bool","`",",","*","optional","*",",","defaults","to","`","False","`",")",":","Whether","or","not","the","default","system","prompt","for","Llama","should","be","used","."]
code: def save_vocabulary(
      self, save_directory: str, filename_prefix: Optional[str] = None
  ) -> Tuple[str]:
      if not self.can_save_slow_tokenizer:
          raise ValueError(
              "Your fast tokenizer does not have the necessary information to save the vocabulary for a slow "
              "tokenizer."
          )
  
      if not os.path.isdir(save_directory):
          logger.error(
              f"Vocabulary path ({save_directory}) should be a directory"
          )
          return
      out_vocab_file = os.path.join(
          save_directory,
          (filename_prefix + "-" if filename_prefix else "")
          + VOCAB_FILES_NAMES["vocab_file"],
      )
  
      if os.path.abspath(self.vocab_file) != os.path.abspath(
          out_vocab_file
      ):
          copyfile(self.vocab_file, out_vocab_file)
  
      return (out_vocab_file,)
  
file_path: transformers/src/transformers/models/llama/tokenization_llama_fast.py
imports: import os
  from shutil import copyfile
  from typing import Optional, Tuple
  from tokenizers import processors
  from ...tokenization_utils_fast import PreTrainedTokenizerFast
  from ...utils import is_sentencepiece_available, logging
  from ...utils.versions import require_version
label_desc: Use image node_id 8 for calling the LlamaTokenizerFast obj's underlying member method code with example usage: obj.save_vocabulary(save_directory, filename_prefix) and returns: out_vocab_file