class_docstr: Construct a CodeLlama tokenizer. Based on byte-level Byte-Pair-Encoding. The default padding token is unset as
  there is no padding token in the original model.
  
  The default configuration match that of
  [codellama/CodeLlama-7b-Instruct-hf](https://huggingface.co/codellama/CodeLlama-7b-Instruct-hf/blob/main/tokenizer_config.json)
  which supports prompt infilling.
  
  Args:
      vocab_file (`str`):
          Path to the vocabulary file.
      unk_token (`str`, *optional*, defaults to `"<unk>"`):
          The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
          token instead.
      bos_token (`str`, *optional*, defaults to `"<s>"`):
          The beginning of sequence token that was used during pretraining. Can be used a sequence classifier token.
      eos_token (`str`, *optional*, defaults to `"</s>"`):
          The end of sequence token.
  
          <Tip>
  
          When building a sequence using special tokens, this is not the token that is used for the end of sequence.
          The token used is the `sep_token`.
  
          </Tip>
  
      prefix_token (`str`, *optional*, defaults to `"▁<PRE>"`):
          Prefix token used for infilling.
      middle_token (`str`, *optional*, defaults to `"▁<MID>"`):
          Middle token used for infilling.
      suffix_token (`str`, *optional*, defaults to `"▁<SUF>"`):
          Suffix token used for infilling.
      eot_token (`str`, *optional*, defaults to `"▁<EOT>"`):
          End of text token used for infilling.
      fill_token (`str`, *optional*, defaults to `"<FILL_ME>"`):
          The token used to split the input between the prefix and suffix.
      suffix_first (`bool`, *optional*, defaults to `False`):
          Whether the input prompt and suffix should be formatted with the suffix first.
      sp_model_kwargs (`dict`, *optional*):
          Will be passed to the `SentencePieceProcessor.__init__()` method. The [Python wrapper for
          SentencePiece](https://github.com/google/sentencepiece/tree/master/python) can be used, among other things,
          to set:
  
          - `enable_sampling`: Enable subword regularization.
          - `nbest_size`: Sampling parameters for unigram. Invalid for BPE-Dropout.
  
            - `nbest_size = {0,1}`: No sampling is performed.
            - `nbest_size > 1`: samples from the nbest_size results.
            - `nbest_size < 0`: assuming that nbest_size is infinite and samples from the all hypothesis (lattice)
              using forward-filtering-and-backward-sampling algorithm.
  
          - `alpha`: Smoothing parameter for unigram sampling, and dropout probability of merge operations for
            BPE-dropout.
      add_bos_token (`bool`, *optional*, defaults to `True`):
          Whether to add a beginning of sequence token at the start of sequences.
      add_eos_token (`bool`, *optional*, defaults to `False`):
          Whether to add an end of sequence token at the end of sequences.
      clean_up_tokenization_spaces (`bool`, *optional*, defaults to `False`):
          Whether or not to clean up the tokenization spaces.
      additional_special_tokens (`List[str]`, *optional*):
          Additional special tokens used by the tokenizer.
      use_default_system_prompt (`bool`, *optional*, defaults to `False`):
          Whether or not the default system prompt for Llama should be used.
class_docstr_tok: ["Construct","a","CodeLlama","tokenizer",".","Based","on","byte-level","Byte-Pair-Encoding",".","The","default","padding","token","is","unset","as","there","is","no","padding","token","in","the","original","model",".","The","default","configuration","match","that","of","[","codellama\/CodeLlama-7b-Instruct-hf","]","(","https",":","\/\/huggingface.co\/codellama\/CodeLlama-7b-Instruct-hf\/blob\/main\/tokenizer_config.json",")","which","supports","prompt","infilling",".","Args",":","vocab_file","(","`","str","`",")",":","Path","to","the","vocabulary","file",".","unk_token","(","`","str","`",",","*","optional","*",",","defaults","to","`","``","<","unk",">","''","`",")",":","The","unknown","token",".","A","token","that","is","not","in","the","vocabulary","can","not","be","converted","to","an","ID","and","is","set","to","be","this","token","instead",".","bos_token","(","`","str","`",",","*","optional","*",",","defaults","to","`","``","<","s",">","''","`",")",":","The","beginning","of","sequence","token","that","was","used","during","pretraining",".","Can","be","used","a","sequence","classifier","token",".","eos_token","(","`","str","`",",","*","optional","*",",","defaults","to","`","``","<","\/s",">","''","`",")",":","The","end","of","sequence","token",".","<","Tip",">","When","building","a","sequence","using","special","tokens",",","this","is","not","the","token","that","is","used","for","the","end","of","sequence",".","The","token","used","is","the","`","sep_token","`",".","<","\/Tip",">","prefix_token","(","`","str","`",",","*","optional","*",",","defaults","to","`","``","\u2581","<","PRE",">","''","`",")",":","Prefix","token","used","for","infilling",".","middle_token","(","`","str","`",",","*","optional","*",",","defaults","to","`","``","\u2581","<","MID",">","''","`",")",":","Middle","token","used","for","infilling",".","suffix_token","(","`","str","`",",","*","optional","*",",","defaults","to","`","``","\u2581","<","SUF",">","''","`",")",":","Suffix","token","used","for","infilling",".","eot_token","(","`","str","`",",","*","optional","*",",","defaults","to","`","``","\u2581","<","EOT",">","''","`",")",":","End","of","text","token","used","for","infilling",".","fill_token","(","`","str","`",",","*","optional","*",",","defaults","to","`","``","<","FILL_ME",">","''","`",")",":","The","token","used","to","split","the","input","between","the","prefix","and","suffix",".","suffix_first","(","`","bool","`",",","*","optional","*",",","defaults","to","`","False","`",")",":","Whether","the","input","prompt","and","suffix","should","be","formatted","with","the","suffix","first",".","sp_model_kwargs","(","`","dict","`",",","*","optional","*",")",":","Will","be","passed","to","the","`","SentencePieceProcessor.__init__","(",")","`","method",".","The","[","Python","wrapper","for","SentencePiece","]","(","https",":","\/\/github.com\/google\/sentencepiece\/tree\/master\/python",")","can","be","used",",","among","other","things",",","to","set",":","-","`","enable_sampling","`",":","Enable","subword","regularization",".","-","`","nbest_size","`",":","Sampling","parameters","for","unigram",".","Invalid","for","BPE-Dropout",".","-","`","nbest_size","=","{","0,1","}","`",":","No","sampling","is","performed",".","-","`","nbest_size",">","1","`",":","samples","from","the","nbest_size","results",".","-","`","nbest_size","<","0","`",":","assuming","that","nbest_size","is","infinite","and","samples","from","the","all","hypothesis","(","lattice",")","using","forward-filtering-and-backward-sampling","algorithm",".","-","`","alpha","`",":","Smoothing","parameter","for","unigram","sampling",",","and","dropout","probability","of","merge","operations","for","BPE-dropout",".","add_bos_token","(","`","bool","`",",","*","optional","*",",","defaults","to","`","True","`",")",":","Whether","to","add","a","beginning","of","sequence","token","at","the","start","of","sequences",".","add_eos_token","(","`","bool","`",",","*","optional","*",",","defaults","to","`","False","`",")",":","Whether","to","add","an","end","of","sequence","token","at","the","end","of","sequences",".","clean_up_tokenization_spaces","(","`","bool","`",",","*","optional","*",",","defaults","to","`","False","`",")",":","Whether","or","not","to","clean","up","the","tokenization","spaces",".","additional_special_tokens","(","`","List","[","str","]","`",",","*","optional","*",")",":","Additional","special","tokens","used","by","the","tokenizer",".","use_default_system_prompt","(","`","bool","`",",","*","optional","*",",","defaults","to","`","False","`",")",":","Whether","or","not","the","default","system","prompt","for","Llama","should","be","used","."]
docstr: LLaMA uses [INST] and [/INST] to indicate user messages, and <<SYS>> and <</SYS>> to indicate system messages.
  Assistant messages do not have special tokens, because LLaMA chat models are generally trained with strict
  user/assistant/user/assistant message ordering, and so assistant messages can be identified from the ordering
  rather than needing special tokens. The system message is partly 'embedded' in the first user message, which
  results in an unusual token ordering when it is present. This template should definitely be changed if you wish
  to fine-tune a model with more flexible role ordering!
  
  The output should look something like:
  
  <bos>[INST] B_SYS SystemPrompt E_SYS Prompt [/INST] Answer <eos><bos>[INST] Prompt [/INST] Answer <eos>
  <bos>[INST] Prompt [/INST]
  
  The reference for this chat template is [this code
  snippet](https://github.com/facebookresearch/llama/blob/556949fdfb72da27c2f4a40b7f0e4cf0b8153a28/llama/generation.py#L320-L362)
  in the original repository.
docstr_tok: ["LLaMA","uses","[","INST","]","and","[","\/INST","]","to","indicate","user","messages",",","and","<","<","SYS",">",">","and","<","<","\/SYS",">",">","to","indicate","system","messages",".","Assistant","messages","do","not","have","special","tokens",",","because","LLaMA","chat","models","are","generally","trained","with","strict","user\/assistant\/user\/assistant","message","ordering",",","and","so","assistant","messages","can","be","identified","from","the","ordering","rather","than","needing","special","tokens",".","The","system","message","is","partly","'embedded","'","in","the","first","user","message",",","which","results","in","an","unusual","token","ordering","when","it","is","present",".","This","template","should","definitely","be","changed","if","you","wish","to","fine-tune","a","model","with","more","flexible","role","ordering","!","The","output","should","look","something","like",":","<","bos",">","[","INST","]","B_SYS","SystemPrompt","E_SYS","Prompt","[","\/INST","]","Answer","<","eos",">","<","bos",">","[","INST","]","Prompt","[","\/INST","]","Answer","<","eos",">","<","bos",">","[","INST","]","Prompt","[","\/INST","]","The","reference","for","this","chat","template","is","[","this","code","snippet","]","(","https",":","\/\/github.com\/facebookresearch\/llama\/blob\/556949fdfb72da27c2f4a40b7f0e4cf0b8153a28\/llama\/generation.py","#","L320-L362",")","in","the","original","repository","."]
code: def default_chat_template(self):
      """
      LLaMA uses [INST] and [/INST] to indicate user messages, and <<SYS>> and <</SYS>> to indicate system messages.
      Assistant messages do not have special tokens, because LLaMA chat models are generally trained with strict
      user/assistant/user/assistant message ordering, and so assistant messages can be identified from the ordering
      rather than needing special tokens. The system message is partly 'embedded' in the first user message, which
      results in an unusual token ordering when it is present. This template should definitely be changed if you wish
      to fine-tune a model with more flexible role ordering!
  
      The output should look something like:
  
      <bos>[INST] B_SYS SystemPrompt E_SYS Prompt [/INST] Answer <eos><bos>[INST] Prompt [/INST] Answer <eos>
      <bos>[INST] Prompt [/INST]
  
      The reference for this chat template is [this code
      snippet](https://github.com/facebookresearch/llama/blob/556949fdfb72da27c2f4a40b7f0e4cf0b8153a28/llama/generation.py#L320-L362)
      in the original repository.
      """
      logger.warning_once(
          "\nNo chat template is defined for this tokenizer - using the default template "
          f"for the {self.__class__.__name__} class. If the default is not appropriate for "
          "your model, please set `tokenizer.chat_template` to an appropriate template. "
          "See https://huggingface.co/docs/transformers/main/chat_templating for more information.\n"
      )
      template = (
          "{% if messages[0]['role'] == 'system' %}"
          "{% set loop_messages = messages[1:] %}"  # Extract system message if it's present
          "{% set system_message = messages[0]['content'] %}"
          "{% elif USE_DEFAULT_PROMPT == true and not '<<SYS>>' in messages[0]['content'] %}"
          "{% set loop_messages = messages %}"  # Or use the default system message if the flag is set
          "{% set system_message = 'DEFAULT_SYSTEM_MESSAGE' %}"
          "{% else %}"
          "{% set loop_messages = messages %}"
          "{% set system_message = false %}"
          "{% endif %}"
          "{% for message in loop_messages %}"  # Loop over all non-system messages
          "{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}"
          "{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}"
          "{% endif %}"
          "{% if loop.index0 == 0 and system_message != false %}"  # Embed system message in first message
          "{% set content = '<<SYS>>\\n' + system_message + '\\n<</SYS>>\\n\\n' + message['content'] %}"
          "{% else %}"
          "{% set content = message['content'] %}"
          "{% endif %}"
          "{% if message['role'] == 'user' %}"  # After all of that, handle messages/roles in a fairly normal way
          "{{ bos_token + '[INST] ' + content.strip() + ' [/INST]' }}"
          "{% elif message['role'] == 'system' %}"
          "{{ '<<SYS>>\\n' + content.strip() + '\\n<</SYS>>\\n\\n' }}"
          "{% elif message['role'] == 'assistant' %}"
          "{{ ' '  + content.strip() + ' ' + eos_token }}"
          "{% endif %}"
          "{% endfor %}"
      )
      template = template.replace(
          "USE_DEFAULT_PROMPT",
          "true" if self.use_default_system_prompt else "false",
      )
      default_message = DEFAULT_SYSTEM_PROMPT.replace(
          "\n", "\\n"
      ).replace("'", "\\'")
      template = template.replace(
          "DEFAULT_SYSTEM_MESSAGE", default_message
      )
  
      return template
  
imports: import os
  from shutil import copyfile
  from typing import Any, Dict, List, Optional, Tuple
  import sentencepiece
  from ...convert_slow_tokenizer import import_protobuf
  from ...tokenization_utils import AddedToken, PreTrainedTokenizer
  from ...utils import logging, requires_backends
label_desc: Use image node_id 23 for calling the CodeLlamaTokenizer obj's underlying member method code with example usage: obj.default_chat_template() and returns: template