class_docstr: Construct a CodeLlama tokenizer. Based on byte-level Byte-Pair-Encoding. The default padding token is unset as
  there is no padding token in the original model.
  
  The default configuration match that of
  [codellama/CodeLlama-7b-Instruct-hf](https://huggingface.co/codellama/CodeLlama-7b-Instruct-hf/blob/main/tokenizer_config.json)
  which supports prompt infilling.
  
  Args:
      vocab_file (`str`):
          Path to the vocabulary file.
      unk_token (`str`, *optional*, defaults to `"<unk>"`):
          The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
          token instead.
      bos_token (`str`, *optional*, defaults to `"<s>"`):
          The beginning of sequence token that was used during pretraining. Can be used a sequence classifier token.
      eos_token (`str`, *optional*, defaults to `"</s>"`):
          The end of sequence token.
  
          <Tip>
  
          When building a sequence using special tokens, this is not the token that is used for the end of sequence.
          The token used is the `sep_token`.
  
          </Tip>
  
      prefix_token (`str`, *optional*, defaults to `"▁<PRE>"`):
          Prefix token used for infilling.
      middle_token (`str`, *optional*, defaults to `"▁<MID>"`):
          Middle token used for infilling.
      suffix_token (`str`, *optional*, defaults to `"▁<SUF>"`):
          Suffix token used for infilling.
      eot_token (`str`, *optional*, defaults to `"▁<EOT>"`):
          End of text token used for infilling.
      fill_token (`str`, *optional*, defaults to `"<FILL_ME>"`):
          The token used to split the input between the prefix and suffix.
      suffix_first (`bool`, *optional*, defaults to `False`):
          Whether the input prompt and suffix should be formatted with the suffix first.
      sp_model_kwargs (`dict`, *optional*):
          Will be passed to the `SentencePieceProcessor.__init__()` method. The [Python wrapper for
          SentencePiece](https://github.com/google/sentencepiece/tree/master/python) can be used, among other things,
          to set:
  
          - `enable_sampling`: Enable subword regularization.
          - `nbest_size`: Sampling parameters for unigram. Invalid for BPE-Dropout.
  
            - `nbest_size = {0,1}`: No sampling is performed.
            - `nbest_size > 1`: samples from the nbest_size results.
            - `nbest_size < 0`: assuming that nbest_size is infinite and samples from the all hypothesis (lattice)
              using forward-filtering-and-backward-sampling algorithm.
  
          - `alpha`: Smoothing parameter for unigram sampling, and dropout probability of merge operations for
            BPE-dropout.
      add_bos_token (`bool`, *optional*, defaults to `True`):
          Whether to add a beginning of sequence token at the start of sequences.
      add_eos_token (`bool`, *optional*, defaults to `False`):
          Whether to add an end of sequence token at the end of sequences.
      clean_up_tokenization_spaces (`bool`, *optional*, defaults to `False`):
          Whether or not to clean up the tokenization spaces.
      additional_special_tokens (`List[str]`, *optional*):
          Additional special tokens used by the tokenizer.
      use_default_system_prompt (`bool`, *optional*, defaults to `False`):
          Whether or not the default system prompt for Llama should be used.
class_docstr_tok: ["Construct","a","CodeLlama","tokenizer",".","Based","on","byte-level","Byte-Pair-Encoding",".","The","default","padding","token","is","unset","as","there","is","no","padding","token","in","the","original","model",".","The","default","configuration","match","that","of","[","codellama\/CodeLlama-7b-Instruct-hf","]","(","https",":","\/\/huggingface.co\/codellama\/CodeLlama-7b-Instruct-hf\/blob\/main\/tokenizer_config.json",")","which","supports","prompt","infilling",".","Args",":","vocab_file","(","`","str","`",")",":","Path","to","the","vocabulary","file",".","unk_token","(","`","str","`",",","*","optional","*",",","defaults","to","`","``","<","unk",">","''","`",")",":","The","unknown","token",".","A","token","that","is","not","in","the","vocabulary","can","not","be","converted","to","an","ID","and","is","set","to","be","this","token","instead",".","bos_token","(","`","str","`",",","*","optional","*",",","defaults","to","`","``","<","s",">","''","`",")",":","The","beginning","of","sequence","token","that","was","used","during","pretraining",".","Can","be","used","a","sequence","classifier","token",".","eos_token","(","`","str","`",",","*","optional","*",",","defaults","to","`","``","<","\/s",">","''","`",")",":","The","end","of","sequence","token",".","<","Tip",">","When","building","a","sequence","using","special","tokens",",","this","is","not","the","token","that","is","used","for","the","end","of","sequence",".","The","token","used","is","the","`","sep_token","`",".","<","\/Tip",">","prefix_token","(","`","str","`",",","*","optional","*",",","defaults","to","`","``","\u2581","<","PRE",">","''","`",")",":","Prefix","token","used","for","infilling",".","middle_token","(","`","str","`",",","*","optional","*",",","defaults","to","`","``","\u2581","<","MID",">","''","`",")",":","Middle","token","used","for","infilling",".","suffix_token","(","`","str","`",",","*","optional","*",",","defaults","to","`","``","\u2581","<","SUF",">","''","`",")",":","Suffix","token","used","for","infilling",".","eot_token","(","`","str","`",",","*","optional","*",",","defaults","to","`","``","\u2581","<","EOT",">","''","`",")",":","End","of","text","token","used","for","infilling",".","fill_token","(","`","str","`",",","*","optional","*",",","defaults","to","`","``","<","FILL_ME",">","''","`",")",":","The","token","used","to","split","the","input","between","the","prefix","and","suffix",".","suffix_first","(","`","bool","`",",","*","optional","*",",","defaults","to","`","False","`",")",":","Whether","the","input","prompt","and","suffix","should","be","formatted","with","the","suffix","first",".","sp_model_kwargs","(","`","dict","`",",","*","optional","*",")",":","Will","be","passed","to","the","`","SentencePieceProcessor.__init__","(",")","`","method",".","The","[","Python","wrapper","for","SentencePiece","]","(","https",":","\/\/github.com\/google\/sentencepiece\/tree\/master\/python",")","can","be","used",",","among","other","things",",","to","set",":","-","`","enable_sampling","`",":","Enable","subword","regularization",".","-","`","nbest_size","`",":","Sampling","parameters","for","unigram",".","Invalid","for","BPE-Dropout",".","-","`","nbest_size","=","{","0,1","}","`",":","No","sampling","is","performed",".","-","`","nbest_size",">","1","`",":","samples","from","the","nbest_size","results",".","-","`","nbest_size","<","0","`",":","assuming","that","nbest_size","is","infinite","and","samples","from","the","all","hypothesis","(","lattice",")","using","forward-filtering-and-backward-sampling","algorithm",".","-","`","alpha","`",":","Smoothing","parameter","for","unigram","sampling",",","and","dropout","probability","of","merge","operations","for","BPE-dropout",".","add_bos_token","(","`","bool","`",",","*","optional","*",",","defaults","to","`","True","`",")",":","Whether","to","add","a","beginning","of","sequence","token","at","the","start","of","sequences",".","add_eos_token","(","`","bool","`",",","*","optional","*",",","defaults","to","`","False","`",")",":","Whether","to","add","an","end","of","sequence","token","at","the","end","of","sequences",".","clean_up_tokenization_spaces","(","`","bool","`",",","*","optional","*",",","defaults","to","`","False","`",")",":","Whether","or","not","to","clean","up","the","tokenization","spaces",".","additional_special_tokens","(","`","List","[","str","]","`",",","*","optional","*",")",":","Additional","special","tokens","used","by","the","tokenizer",".","use_default_system_prompt","(","`","bool","`",",","*","optional","*",",","defaults","to","`","False","`",")",":","Whether","or","not","the","default","system","prompt","for","Llama","should","be","used","."]
docstr: Creates a mask from the two sequences passed to be used in a sequence-pair classification task. An ALBERT
  sequence pair mask has the following format:
  
  ```
  0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1
  | first sequence    | second sequence |
  ```
  
  if token_ids_1 is None, only returns the first portion of the mask (0s).
  
  Args:
      token_ids_0 (`List[int]`):
          List of ids.
      token_ids_1 (`List[int]`, *optional*):
          Optional second list of IDs for sequence pairs.
  
  Returns:
      `List[int]`: List of [token type IDs](../glossary#token-type-ids) according to the given sequence(s).
docstr_tok: ["Creates","a","mask","from","the","two","sequences","passed","to","be","used","in","a","sequence-pair","classification","task",".","An","ALBERT","sequence","pair","mask","has","the","following","format",":","``","`","0","0","0","0","0","0","0","0","0","0","0","1","1","1","1","1","1","1","1","1","|","first","sequence","|","second","sequence","|","``","`","if","token_ids_1","is","None",",","only","returns","the","first","portion","of","the","mask","(","0s",")",".","Args",":","token_ids_0","(","`","List","[","int","]","`",")",":","List","of","ids",".","token_ids_1","(","`","List","[","int","]","`",",","*","optional","*",")",":","Optional","second","list","of","IDs","for","sequence","pairs",".","Returns",":","`","List","[","int","]","`",":","List","of","[","token","type","IDs","]","(","..","\/glossary","#","token-type-ids",")","according","to","the","given","sequence","(","s",")","."]
code: def create_token_type_ids_from_sequences(
      self,
      token_ids_0: List[int],
      token_ids_1: Optional[List[int]] = None,
  ) -> List[int]:
      """
      Creates a mask from the two sequences passed to be used in a sequence-pair classification task. An ALBERT
      sequence pair mask has the following format:
  
      ```
      0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1
      | first sequence    | second sequence |
      ```
  
      if token_ids_1 is None, only returns the first portion of the mask (0s).
  
      Args:
          token_ids_0 (`List[int]`):
              List of ids.
          token_ids_1 (`List[int]`, *optional*):
              Optional second list of IDs for sequence pairs.
  
      Returns:
          `List[int]`: List of [token type IDs](../glossary#token-type-ids) according to the given sequence(s).
      """
      bos_token_id = [self.bos_token_id] if self.add_bos_token else []
      eos_token_id = [self.eos_token_id] if self.add_eos_token else []
  
      output = [0] * len(bos_token_id + token_ids_0 + eos_token_id)
  
      if token_ids_1 is not None:
          output += [1] * len(bos_token_id + token_ids_1 + eos_token_id)
  
      return output
  
imports: import os
  from shutil import copyfile
  from typing import Any, Dict, List, Optional, Tuple
  import sentencepiece
  from ...convert_slow_tokenizer import import_protobuf
  from ...tokenization_utils import AddedToken, PreTrainedTokenizer
  from ...utils import logging, requires_backends
label_desc: Use image node_id 22 for calling the CodeLlamaTokenizer obj's underlying member method code with example usage: obj.create_token_type_ids_from_sequences(token_ids_0, token_ids_1) and returns: output