





"""simple docstring"""
import unittest

from transformers import AutoTokenizer, FalconConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
   import torch

   from transformers import (
       FalconForCausalLM,
       FalconForQuestionAnswering,
       FalconForSequenceClassification,
       FalconForTokenClassification,
       FalconModel,
   )
class lowerCamelCase__	:



   def __init__(							self   , snake_case   , snake_case=3   , snake_case=7   , snake_case=True   , snake_case=True   , snake_case=False   , snake_case=True   , snake_case=9_9   , snake_case=3_2   , snake_case=5   , snake_case=4   , snake_case=3_7   , snake_case="gelu"   , snake_case=0.1   , snake_case=0.1   , snake_case=5_1_2   , snake_case=1_6   , snake_case=2   , snake_case=0.02   , snake_case=3   , snake_case=4   , snake_case=None   , )			->     Optional[Any]:







      """simple docstring"""
      lowercase     :      Any	=							parent
      lowercase     :      str	=							batch_size
      lowercase     :      int	=							seq_length
      lowercase     :      List[str]	=							is_training
      lowercase     :      Tuple	=							use_input_mask
      lowercase     :      str	=							use_token_type_ids
      lowercase     :      List[Any]	=							use_labels
      lowercase     :      Optional[int]	=							vocab_size
      lowercase     :      Optional[Any]	=							hidden_size
      lowercase     :      Union[str, Any]	=							num_hidden_layers
      lowercase     :      str	=							num_attention_heads
      lowercase     :      Any	=							intermediate_size
      lowercase     :      List[str]	=							hidden_act
      lowercase     :      Optional[Any]	=							hidden_dropout_prob
      lowercase     :      Optional[Any]	=							attention_probs_dropout_prob
      lowercase     :      str	=							max_position_embeddings
      lowercase     :      List[Any]	=							type_vocab_size
      lowercase     :      Tuple	=							type_sequence_label_size
      lowercase     :      Union[str, Any]	=							initializer_range
      lowercase     :      Tuple	=							num_labels
      lowercase     :      Union[str, Any]	=							num_choices
      lowercase     :      Optional[Any]	=							scope



   def  _UpperCAmelCase						(							self						)			->     Optional[Any]:







      """simple docstring"""
      lowercase     :      Any	=							ids_tensor([self.batch_size, self.seq_length]   , self.vocab_size						)

      lowercase     :      Tuple	=							None
      if self.use_input_mask:
         lowercase     :      Dict	=							random_attention_mask([self.batch_size, self.seq_length]						)

      lowercase     :      str	=							None

      lowercase     :      str	=							None
      lowercase     :      List[str]	=							None
      lowercase     :      str	=							None
      if self.use_labels:
         lowercase     :      Optional[Any]	=							ids_tensor([self.batch_size]   , self.type_sequence_label_size						)
         lowercase     :      Tuple	=							ids_tensor([self.batch_size, self.seq_length]   , self.num_labels						)
         lowercase     :      str	=							ids_tensor([self.batch_size]   , self.num_choices						)

      lowercase     :      str	=							self.get_config()

      return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels



   def  _UpperCAmelCase						(							self						)			->     Optional[int]:







      """simple docstring"""
      return FalconConfig(
          vocab_size=self.vocab_size   , hidden_size=self.hidden_size   , num_hidden_layers=self.num_hidden_layers   , num_attention_heads=self.num_attention_heads   , intermediate_size=self.intermediate_size   , hidden_act=self.hidden_act   , hidden_dropout_prob=self.hidden_dropout_prob   , attention_probs_dropout_prob=self.attention_probs_dropout_prob   , max_position_embeddings=self.max_position_embeddings   , type_vocab_size=self.type_vocab_size   , is_decoder=__lowerCamelCase   , initializer_range=self.initializer_range   , pad_token_id=1   , new_decoder_architecture=__lowerCamelCase   , )



   def  _UpperCAmelCase						(							self   , snake_case   , snake_case   , snake_case   , snake_case   , snake_case   , snake_case   , snake_case						)			->     Union[str, Any]:







      """simple docstring"""
      lowercase     :      Union[str, Any]	=							FalconModel(config=__lowerCamelCase						)
      model.to(__lowerCamelCase						)
      model.eval()
      lowercase     :      Any	=							model(__lowerCamelCase   , attention_mask=__lowerCamelCase						)
      lowercase     :      Optional[int]	=							model(__lowerCamelCase						)
      self.parent.assertEqual(result.last_hidden_state.shape   , (self.batch_size, self.seq_length, self.hidden_size)						)



   def  _UpperCAmelCase						(							self   , snake_case   , snake_case   , snake_case   , snake_case   , snake_case   , snake_case   , snake_case   , snake_case   , snake_case   , )			->     Optional[int]:







      """simple docstring"""
      lowercase     :      Optional[int]	=							True
      lowercase     :      Any	=							FalconModel(__lowerCamelCase						)
      model.to(__lowerCamelCase						)
      model.eval()
      lowercase     :      Optional[int]	=							model(
          __lowerCamelCase   , attention_mask=__lowerCamelCase   , encoder_hidden_states=__lowerCamelCase   , encoder_attention_mask=__lowerCamelCase   , )
      lowercase     :      Optional[Any]	=							model(
          __lowerCamelCase   , attention_mask=__lowerCamelCase   , encoder_hidden_states=__lowerCamelCase   , )
      lowercase     :      Optional[Any]	=							model(__lowerCamelCase   , attention_mask=__lowerCamelCase						)
      self.parent.assertEqual(result.last_hidden_state.shape   , (self.batch_size, self.seq_length, self.hidden_size)						)



   def  _UpperCAmelCase						(							self   , snake_case   , snake_case   , snake_case   , snake_case   , snake_case   , snake_case   , snake_case   , snake_case   , snake_case   , )			->     List[Any]:







      """simple docstring"""
      lowercase     :      Dict	=							FalconForCausalLM(config=__lowerCamelCase						)
      model.to(__lowerCamelCase						)
      model.eval()
      lowercase     :      List[str]	=							model(__lowerCamelCase   , attention_mask=__lowerCamelCase   , labels=__lowerCamelCase						)
      self.parent.assertEqual(result.logits.shape   , (self.batch_size, self.seq_length, self.vocab_size)						)



   def  _UpperCAmelCase						(							self   , snake_case   , snake_case   , snake_case   , snake_case   , snake_case   , snake_case   , snake_case   , snake_case   , snake_case   , )			->     Optional[Any]:







      """simple docstring"""
      lowercase     :      int	=							True
      lowercase     :      List[Any]	=							True
      lowercase     :      Any	=							FalconForCausalLM(config=__lowerCamelCase						)
      model.to(__lowerCamelCase						)
      model.eval()

      # first forward pass
      lowercase     :      Any	=							model(
          __lowerCamelCase   , attention_mask=__lowerCamelCase   , encoder_hidden_states=__lowerCamelCase   , encoder_attention_mask=__lowerCamelCase   , use_cache=__lowerCamelCase   , )
      lowercase     :      str	=							outputs.past_key_values

      # create hypothetical multiple next token and extent to next_input_ids
      lowercase     :      Tuple	=							ids_tensor((self.batch_size, 3)   , config.vocab_size						)
      lowercase     :      Tuple	=							ids_tensor((self.batch_size, 3)   , vocab_size=2						)

      # append to next input_ids and
      lowercase     :      Any	=							torch.cat([input_ids, next_tokens]   , dim=-1						)
      lowercase     :      List[str]	=							torch.cat([input_mask, next_mask]   , dim=-1						)

      lowercase     :      int	=							model(
          __lowerCamelCase   , attention_mask=__lowerCamelCase   , encoder_hidden_states=__lowerCamelCase   , encoder_attention_mask=__lowerCamelCase   , output_hidden_states=__lowerCamelCase   , )["""hidden_states"""][0]
      lowercase     :      Optional[Any]	=							model(
          __lowerCamelCase   , attention_mask=__lowerCamelCase   , encoder_hidden_states=__lowerCamelCase   , encoder_attention_mask=__lowerCamelCase   , past_key_values=__lowerCamelCase   , output_hidden_states=__lowerCamelCase   , )["""hidden_states"""][0]

      # select random slice
      lowercase     :      Optional[Any]	=							ids_tensor((1,)   , output_from_past.shape[-1]						).item()
      lowercase     :      int	=							output_from_no_past[:, -3:, random_slice_idx].detach()
      lowercase     :      Dict	=							output_from_past[:, :, random_slice_idx].detach()

      self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1]						)

      # test that outputs are equal for slice
      self.parent.assertTrue(torch.allclose(__lowerCamelCase   , __lowerCamelCase   , atol=1E-3						)						)





   def  _UpperCAmelCase						(							self						)			->     List[Any]:







      """simple docstring"""
      lowercase     :      Optional[int]	=							self.prepare_config_and_inputs()
      (
          (
          lowercase	
      )	,       (
          lowercase	
      )	,       (
          lowercase	
      )	,       (
          lowercase	
      )	,       (
          lowercase	
      )	,       (
          lowercase	
      )	,       (
          lowercase	
      )	,       	
      )     :      Union[str, Any]	=							config_and_inputs
      lowercase     :      str	=							{"""input_ids""": input_ids, """attention_mask""": input_mask}
      return config, inputs_dict




@require_torch
class lowerCamelCase__	(     __snake_case	,					__snake_case	,					__snake_case	,					unittest.TestCase					):
   __UpperCAmelCase										=       (
       (
           FalconModel,
           FalconForCausalLM,
           FalconForSequenceClassification,
           FalconForTokenClassification,
           FalconForQuestionAnswering,
       )
       if is_torch_available()
       else ()
   )
   __UpperCAmelCase										=       (FalconForCausalLM,) if is_torch_available() else ()
   __UpperCAmelCase										=       (
       {
           """feature-extraction""": FalconModel,
           """text-classification""": FalconForSequenceClassification,
           """text-generation""": FalconForCausalLM,
           """question-answering""": FalconForQuestionAnswering,
           """token-classification""": FalconForTokenClassification,
           """zero-shot""": FalconForSequenceClassification,
       }
       if is_torch_available()
       else {}
   )
   __UpperCAmelCase										=       False
   __UpperCAmelCase										=       False



   def  _UpperCAmelCase						(							self						)			->     Optional[Any]:







      """simple docstring"""
      lowercase     :      Dict	=							FalconModelTester(self						)
      lowercase     :      Any	=							ConfigTester(self   , config_class=__lowerCamelCase   , hidden_size=3_7						)



   def  _UpperCAmelCase						(							self						)			->     List[str]:







      """simple docstring"""
      self.config_tester.run_common_tests()



   def  _UpperCAmelCase						(							self						)			->     int:







      """simple docstring"""
      lowercase     :      Any	=							self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_model(*__lowerCamelCase						)



   def  _UpperCAmelCase						(							self						)			->     Dict:







      """simple docstring"""
      lowercase	,       *lowercase     :      List[str]	=							self.model_tester.prepare_config_and_inputs()
      for alibi in [True, False]:
         lowercase     :      Union[str, Any]	=							alibi
         self.model_tester.create_and_check_model(__lowerCamelCase   , *__lowerCamelCase						)



   def  _UpperCAmelCase						(							self						)			->     int:







      """simple docstring"""
      lowercase	,       lowercase     :      Union[str, Any]	=							self.model_tester.prepare_config_and_inputs_for_common()
      lowercase     :      Any	=							3
      lowercase     :      List[str]	=							input_dict["""input_ids"""]
      lowercase     :      Tuple	=							input_ids.ne(1						).to(__lowerCamelCase						)
      lowercase     :      Optional[int]	=							ids_tensor([self.model_tester.batch_size]   , self.model_tester.type_sequence_label_size						)
      lowercase     :      str	=							FalconForSequenceClassification(__lowerCamelCase						)
      model.to(__lowerCamelCase						)
      model.eval()
      lowercase     :      Optional[int]	=							model(__lowerCamelCase   , attention_mask=__lowerCamelCase   , labels=__lowerCamelCase						)
      self.assertEqual(result.logits.shape   , (self.model_tester.batch_size, self.model_tester.num_labels)						)



   def  _UpperCAmelCase						(							self						)			->     Union[str, Any]:







      """simple docstring"""
      lowercase	,       lowercase     :      Tuple	=							self.model_tester.prepare_config_and_inputs_for_common()
      lowercase     :      Optional[Any]	=							3
      lowercase     :      List[str]	=							"""single_label_classification"""
      lowercase     :      Tuple	=							input_dict["""input_ids"""]
      lowercase     :      Optional[int]	=							input_ids.ne(1						).to(__lowerCamelCase						)
      lowercase     :      int	=							ids_tensor([self.model_tester.batch_size]   , self.model_tester.type_sequence_label_size						)
      lowercase     :      List[Any]	=							FalconForSequenceClassification(__lowerCamelCase						)
      model.to(__lowerCamelCase						)
      model.eval()
      lowercase     :      str	=							model(__lowerCamelCase   , attention_mask=__lowerCamelCase   , labels=__lowerCamelCase						)
      self.assertEqual(result.logits.shape   , (self.model_tester.batch_size, self.model_tester.num_labels)						)



   def  _UpperCAmelCase						(							self						)			->     int:







      """simple docstring"""
      lowercase	,       lowercase     :      Any	=							self.model_tester.prepare_config_and_inputs_for_common()
      lowercase     :      int	=							input_dict["""input_ids"""]
      lowercase     :      List[Any]	=							FalconForCausalLM(__lowerCamelCase						)
      model.to(__lowerCamelCase						)
      model.eval()
      lowercase     :      Optional[Any]	=							model(__lowerCamelCase   , use_cache=__lowerCamelCase						)
      lowercase     :      int	=							input_ids.shape[0]
      lowercase     :      Tuple	=							model._convert_to_rw_cache(result.past_key_values						)
      lowercase     :      List[Any]	=							model._convert_cache_to_standard_format(__lowerCamelCase   , __lowerCamelCase						)
      for layer in range(len(__lowerCamelCase						)						):
         for tensor_idx in range(2						):
            self.assertTrue(rw_cache[layer][tensor_idx].ndim == 3						)
            self.assertTrue(result.past_key_values[layer][tensor_idx].ndim == 4						)
            self.assertTrue(
                torch.all(result.past_key_values[layer][tensor_idx] == standard_cache[layer][tensor_idx]						)						)



   def  _UpperCAmelCase						(							self						)			->     List[Any]:







      """simple docstring"""
      lowercase	,       lowercase     :      int	=							self.model_tester.prepare_config_and_inputs_for_common()
      lowercase     :      Any	=							3
      lowercase     :      List[str]	=							"""multi_label_classification"""
      lowercase     :      Optional[int]	=							input_dict["""input_ids"""]
      lowercase     :      Union[str, Any]	=							input_ids.ne(1						).to(__lowerCamelCase						)
      lowercase     :      Optional[Any]	=							ids_tensor(
          [self.model_tester.batch_size, config.num_labels]   , self.model_tester.type_sequence_label_size						).to(torch.float						)
      lowercase     :      Optional[int]	=							FalconForSequenceClassification(__lowerCamelCase						)
      model.to(__lowerCamelCase						)
      model.eval()
      lowercase     :      Dict	=							model(__lowerCamelCase   , attention_mask=__lowerCamelCase   , labels=__lowerCamelCase						)
      self.assertEqual(result.logits.shape   , (self.model_tester.batch_size, self.model_tester.num_labels)						)





   def  _UpperCAmelCase						(							self						)			->     Optional[Any]:







      """simple docstring"""
      # Falcon can have different numbers of KV-heads than the number of query heads, so we need
      # to override this test to use the right head counts.
      for model_class in self.all_generative_model_classes:
         lowercase	,       lowercase     :      List[Any]	=							self.model_tester.prepare_config_and_inputs_for_common()

         # If it doesn't support cache, pass the test
         if not hasattr(__lowerCamelCase   , """use_cache"""						):
            return

         lowercase     :      List[Any]	=							model_class(__lowerCamelCase						).to(__lowerCamelCase						)
         if "use_cache" not in inputs:
            lowercase     :      Any	=							True
         lowercase     :      str	=							model(**__lowerCamelCase						)

         # If "past_key_values" is not returned, pass the test (e.g. RWKV uses a different cache name and format)
         if "past_key_values" not in outputs:
            return

         lowercase     :      Any	=							(
             getattr(__lowerCamelCase   , """decoder_layers"""   , __lowerCamelCase						)
             or getattr(__lowerCamelCase   , """num_decoder_layers"""   , __lowerCamelCase						)
             or config.num_hidden_layers
         )
         lowercase     :      str	=							getattr(__lowerCamelCase   , """num_kv_heads"""   , config.num_attention_heads						)
         lowercase     :      List[str]	=							getattr(__lowerCamelCase   , """d_model"""   , config.hidden_size						)
         lowercase     :      Optional[Any]	=							embed_dim // num_attention_heads

         lowercase     :      str	=							outputs["""past_key_values"""]
         self.assertEqual(len(__lowerCamelCase						)   , __lowerCamelCase						)

         lowercase	,       lowercase     :      Optional[Any]	=							inputs["""input_ids"""].shape
         for i in range(__lowerCamelCase						):
            if config.new_decoder_architecture:
               lowercase     :      str	=							config.num_attention_heads
            elif config.multi_query:
               lowercase     :      List[str]	=							1
            self.assertEqual(len(past_kv[0]						)   , 2						)  # K V for the decoder = 2
            self.assertEqual(
                past_kv[i][0].shape   , (batch_size, num_attention_heads, seq_length, per_head_embed_dim)						)
            self.assertEqual(
                past_kv[i][1].shape   , (batch_size, num_attention_heads, seq_length, per_head_embed_dim)						)




@require_torch
class lowerCamelCase__	(     unittest.TestCase					):



   @slow
   def  _UpperCAmelCase						(							self						)			->     Tuple:







      """simple docstring"""
      lowercase     :      Dict	=							AutoTokenizer.from_pretrained("""Rocketknight1/falcon-rw-1b"""						)
      lowercase     :      str	=							FalconForCausalLM.from_pretrained("""Rocketknight1/falcon-rw-1b"""						)
      model.eval()
      model.to(__lowerCamelCase						)
      lowercase     :      Optional[int]	=							tokenizer("""My favorite food is"""   , return_tensors="""pt"""						).to(__lowerCamelCase						)

      lowercase     :      Union[str, Any]	=							(
          """My favorite food is pizza. I love it so much that I have a pizza party every year for my birthday."""
      )

      lowercase     :      Optional[int]	=							model.generate(**__lowerCamelCase   , do_sample=__lowerCamelCase   , max_new_tokens=1_9						)
      lowercase     :      Tuple	=							tokenizer.batch_decode(__lowerCamelCase						)[0]

      self.assertEqual(__lowerCamelCase   , __lowerCamelCase						)



   @slow
   def  _UpperCAmelCase						(							self						)			->     Optional[int]:







      """simple docstring"""
      # The big models are way too big for the CI, so we use tiny random models that resemble their
      # architectures but with much smaller and fewer layers
      for repo in ["Rocketknight1/tiny-random-falcon-7b", "Rocketknight1/tiny-random-falcon-40b"]:
         lowercase     :      List[Any]	=							AutoTokenizer.from_pretrained(__lowerCamelCase						)
         lowercase     :      Tuple	=							FalconForCausalLM.from_pretrained(__lowerCamelCase						)
         model.eval()
         model.to(__lowerCamelCase						)
         lowercase     :      List[Any]	=							tokenizer("""My favorite food is"""   , return_tensors="""pt"""						).to(__lowerCamelCase						)

         # We just test that these run without errors - the models are randomly initialized
         # and so the actual text outputs will be garbage
         model.generate(**__lowerCamelCase   , do_sample=__lowerCamelCase   , max_new_tokens=4						)
         model.generate(**__lowerCamelCase   , do_sample=__lowerCamelCase   , max_new_tokens=4						)
         model.generate(**__lowerCamelCase   , num_beams=2   , max_new_tokens=4						)





   @slow
   def  _UpperCAmelCase						(							self						)			->     Optional[int]:







      """simple docstring"""
      # The big models are way too big for the CI, so we use tiny random models that resemble their
      # architectures but with much smaller and fewer layers
      with torch.no_grad():
         for repo in [
             "Rocketknight1/falcon-rw-1b",
             "Rocketknight1/tiny-random-falcon-7b",
             "Rocketknight1/tiny-random-falcon-40b",
         ]:
            lowercase     :      Tuple	=							AutoTokenizer.from_pretrained(__lowerCamelCase						)
            lowercase     :      Optional[Any]	=							FalconForCausalLM.from_pretrained(__lowerCamelCase						)
            model.eval()
            model.to(device=__lowerCamelCase						)
            lowercase     :      List[str]	=							tokenizer("""My favorite food is"""   , return_tensors="""pt"""						).to(__lowerCamelCase						)

            # Test results are the same with and without cache
            lowercase     :      int	=							model.generate(**__lowerCamelCase   , do_sample=__lowerCamelCase   , max_new_tokens=2_0   , use_cache=__lowerCamelCase						)
            lowercase     :      Optional[int]	=							model.generate(**__lowerCamelCase   , do_sample=__lowerCamelCase   , max_new_tokens=2_0   , use_cache=__lowerCamelCase						)
            self.assertTrue((outputs_cache - outputs_no_cache).sum().item() == 0						)





import json
import os
from functools import lru_cache
from typing import TYPE_CHECKING, List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


if TYPE_CHECKING:
    from transformers.pipelines.conversational import Conversation

__A			:					Any						=			logging.get_logger(__name__)


__A			:					Any						=			{
    'vocab_file': 'vocab.json',
    'merges_file': 'merges.txt',
    'tokenizer_config_file': 'tokenizer_config.json',
}

__A			:					Optional[Any]						=			{
    'vocab_file': {'facebook/blenderbot-3B': 'https://huggingface.co/facebook/blenderbot-3B/resolve/main/vocab.json'},
    'merges_file': {'facebook/blenderbot-3B': 'https://huggingface.co/facebook/blenderbot-3B/resolve/main/merges.txt'},
    'tokenizer_config_file': {
        'facebook/blenderbot-3B': 'https://huggingface.co/facebook/blenderbot-3B/resolve/main/tokenizer_config.json'
    },
}

__A			:					Union[str, Any]						=			{'facebook/blenderbot-3B': 1_2_8}





@lru_cache()
# Copied from transformers.models.roberta.tokenization_roberta.bytes_to_unicode
def 	__a     (      ):
    SCREAMING_SNAKE_CASE										=   (
        list(range(ord("!"       )			,  ord("~"       ) + 1       )       ) + list(range(ord("¡"       )			,  ord("¬"       ) + 1       )       ) + list(range(ord("®"       )			,  ord("ÿ"       ) + 1       )       )
    )
    SCREAMING_SNAKE_CASE										=   bs[:]
    SCREAMING_SNAKE_CASE										=   0
    for b in range(2**8       ):
        if b not in bs:
            bs.append(A__       )
            cs.append(2**8 + n       )
            n += 1
    SCREAMING_SNAKE_CASE										=   [chr(A__       ) for n in cs]
    return dict(zip(A__			,  A__       )       )





def 	__a     (      A__			:  List[Any]       ):
    SCREAMING_SNAKE_CASE										=   set()
    SCREAMING_SNAKE_CASE										=   word[0]
    for char in word[1:]:
        pairs.add((prev_char, char)       )
        SCREAMING_SNAKE_CASE										=   char
    return pairs





class 			_SCREAMING_SNAKE_CASE (   __snake_case		):







   '''simple docstring'''




   lowerCamelCase__				=      VOCAB_FILES_NAMES
   lowerCamelCase__				=      PRETRAINED_VOCAB_FILES_MAP
   lowerCamelCase__				=      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   lowerCamelCase__				=      ["input_ids", "attention_mask"]





   def __init__(       self					:			Union[str, Any]     , __lowerCamelCase					:			Dict     , __lowerCamelCase					:			Optional[Any]     , __lowerCamelCase					:			Any="replace"     , __lowerCamelCase					:			Optional[Any]="<s>"     , __lowerCamelCase					:			Optional[Any]="</s>"     , __lowerCamelCase					:			Any="</s>"     , __lowerCamelCase					:			Union[str, Any]="<s>"     , __lowerCamelCase					:			List[str]="<unk>"     , __lowerCamelCase					:			Optional[Any]="<pad>"     , __lowerCamelCase					:			Dict="<mask>"     , __lowerCamelCase					:			Any=False     , **__lowerCamelCase					:			Optional[Any]     , ):
       SCREAMING_SNAKE_CASE										=   AddedToken(__lowerCamelCase     , lstrip=__lowerCamelCase     , rstrip=__lowerCamelCase			) if isinstance(__lowerCamelCase     , __lowerCamelCase			) else bos_token
       SCREAMING_SNAKE_CASE										=   AddedToken(__lowerCamelCase     , lstrip=__lowerCamelCase     , rstrip=__lowerCamelCase			) if isinstance(__lowerCamelCase     , __lowerCamelCase			) else eos_token
       SCREAMING_SNAKE_CASE										=   AddedToken(__lowerCamelCase     , lstrip=__lowerCamelCase     , rstrip=__lowerCamelCase			) if isinstance(__lowerCamelCase     , __lowerCamelCase			) else sep_token
       SCREAMING_SNAKE_CASE										=   AddedToken(__lowerCamelCase     , lstrip=__lowerCamelCase     , rstrip=__lowerCamelCase			) if isinstance(__lowerCamelCase     , __lowerCamelCase			) else cls_token
       SCREAMING_SNAKE_CASE										=   AddedToken(__lowerCamelCase     , lstrip=__lowerCamelCase     , rstrip=__lowerCamelCase			) if isinstance(__lowerCamelCase     , __lowerCamelCase			) else unk_token
       SCREAMING_SNAKE_CASE										=   AddedToken(__lowerCamelCase     , lstrip=__lowerCamelCase     , rstrip=__lowerCamelCase			) if isinstance(__lowerCamelCase     , __lowerCamelCase			) else pad_token

       # Mask token behave like a normal word, i.e. include the space before it
       SCREAMING_SNAKE_CASE										=   AddedToken(__lowerCamelCase     , lstrip=__lowerCamelCase     , rstrip=__lowerCamelCase			) if isinstance(__lowerCamelCase     , __lowerCamelCase			) else mask_token

       super().__init__(
           errors=__lowerCamelCase     , bos_token=__lowerCamelCase     , eos_token=__lowerCamelCase     , unk_token=__lowerCamelCase     , sep_token=__lowerCamelCase     , cls_token=__lowerCamelCase     , pad_token=__lowerCamelCase     , mask_token=__lowerCamelCase     , add_prefix_space=__lowerCamelCase     , **__lowerCamelCase     , )

       with open(__lowerCamelCase     , encoding="utf-8"			) as vocab_handle:
           SCREAMING_SNAKE_CASE										=   json.load(__lowerCamelCase			)
       SCREAMING_SNAKE_CASE										=   {v: k for k, v in self.encoder.items()}
       SCREAMING_SNAKE_CASE										=   errors  # how to handle errors in decoding
       SCREAMING_SNAKE_CASE										=   bytes_to_unicode()
       SCREAMING_SNAKE_CASE										=   {v: k for k, v in self.byte_encoder.items()}
       with open(__lowerCamelCase     , encoding="utf-8"			) as merges_handle:
           SCREAMING_SNAKE_CASE										=   merges_handle.read().split("\n"			)[1:-1]
       SCREAMING_SNAKE_CASE										=   [tuple(merge.split()			) for merge in bpe_merges]
       SCREAMING_SNAKE_CASE										=   dict(zip(__lowerCamelCase     , range(len(__lowerCamelCase			)			)			)			)
       SCREAMING_SNAKE_CASE										=   {}
       SCREAMING_SNAKE_CASE										=   add_prefix_space

       # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
       SCREAMING_SNAKE_CASE										=   re.compile(r"'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"			)





   @property
   # Copied from transformers.models.roberta.tokenization_roberta.RobertaTokenizer.vocab_size with Roberta->Blenderbot, RoBERTa->Blenderbot
   def      _snake_case		(       self					:			str			):
       return len(self.encoder			)





   def      _snake_case		(       self					:			Union[str, Any]			):
       return dict(self.encoder     , **self.added_tokens_encoder			)





   def      _snake_case		(       self					:			Dict     , __lowerCamelCase					:			List[Any]			):
       if token in self.cache:
           return self.cache[token]
       SCREAMING_SNAKE_CASE										=   tuple(__lowerCamelCase			)
       SCREAMING_SNAKE_CASE										=   get_pairs(__lowerCamelCase			)

       if not pairs:
           return token

       while True:
           SCREAMING_SNAKE_CASE										=   min(__lowerCamelCase     , key=lambda __lowerCamelCase			: self.bpe_ranks.get(__lowerCamelCase     , float("inf"			)			)			)
           if bigram not in self.bpe_ranks:
               break
           SCREAMING_SNAKE_CASE							,	SCREAMING_SNAKE_CASE										=   bigram
           SCREAMING_SNAKE_CASE										=   []
           SCREAMING_SNAKE_CASE										=   0
           while i < len(__lowerCamelCase			):
               try:
                   SCREAMING_SNAKE_CASE										=   word.index(__lowerCamelCase     , __lowerCamelCase			)
               except ValueError:
                   new_word.extend(word[i:]			)
                   break
               else:
                   new_word.extend(word[i:j]			)
                   SCREAMING_SNAKE_CASE										=   j

               if word[i] == first and i < len(__lowerCamelCase			) - 1 and word[i + 1] == second:
                   new_word.append(first + second			)
                   i += 2
               else:
                   new_word.append(word[i]			)
                   i += 1
           SCREAMING_SNAKE_CASE										=   tuple(__lowerCamelCase			)
           SCREAMING_SNAKE_CASE										=   new_word
           if len(__lowerCamelCase			) == 1:
               break
           else:
               SCREAMING_SNAKE_CASE										=   get_pairs(__lowerCamelCase			)
       SCREAMING_SNAKE_CASE										=   " ".join(__lowerCamelCase			)
       SCREAMING_SNAKE_CASE										=   word
       return word





   def      _snake_case		(       self					:			Optional[Any]     , __lowerCamelCase					:			List[Any]			):
       SCREAMING_SNAKE_CASE										=   []
       for token in re.findall(self.pat     , __lowerCamelCase			):
           SCREAMING_SNAKE_CASE										=   "".join(
               self.byte_encoder[b] for b in token.encode("utf-8"			)			)  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
           bpe_tokens.extend(bpe_token for bpe_token in self.bpe(__lowerCamelCase			).split(" "			)			)
       return bpe_tokens





   def      _snake_case		(       self					:			Tuple     , __lowerCamelCase					:			Dict			):
       return self.encoder.get(__lowerCamelCase     , self.encoder.get(self.unk_token			)			)





   def      _snake_case		(       self					:			Any     , __lowerCamelCase					:			Optional[int]			):
       return self.decoder.get(__lowerCamelCase			)





   def      _snake_case		(       self					:			Optional[int]     , __lowerCamelCase					:			List[Any]			):
       SCREAMING_SNAKE_CASE										=   "".join(__lowerCamelCase			)
       SCREAMING_SNAKE_CASE										=   bytearray([self.byte_decoder[c] for c in text]			).decode("utf-8"     , errors=self.errors			)
       return text





   def      _snake_case		(       self					:			Any     , __lowerCamelCase					:			str     , __lowerCamelCase					:			Optional[str] = None			):
       if not os.path.isdir(__lowerCamelCase			):
           logger.error(f"Vocabulary path ({save_directory}) should be a directory"			)
           return
       SCREAMING_SNAKE_CASE										=   os.path.join(
           __lowerCamelCase     , (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]			)
       SCREAMING_SNAKE_CASE										=   os.path.join(
           __lowerCamelCase     , (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["merges_file"]			)

       with open(__lowerCamelCase     , "w"     , encoding="utf-8"			) as f:
           f.write(json.dumps(self.encoder     , indent=2     , sort_keys=__lowerCamelCase     , ensure_ascii=__lowerCamelCase			) + "\n"			)

       SCREAMING_SNAKE_CASE										=   0
       with open(__lowerCamelCase     , "w"     , encoding="utf-8"			) as writer:
           writer.write("#version: 0.2\n"			)
           for bpe_tokens, token_index in sorted(self.bpe_ranks.items()     , key=lambda __lowerCamelCase			: kv[1]			):
               if index != token_index:
                   logger.warning(
                       f"Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."
                       " Please check that the tokenizer is not corrupted!"			)
                   SCREAMING_SNAKE_CASE										=   token_index
               writer.write(" ".join(__lowerCamelCase			) + "\n"			)
               index += 1

       return vocab_file, merge_file





   def      _snake_case		(       self					:			Dict     , __lowerCamelCase					:			List[int]     , __lowerCamelCase					:			Optional[List[int]] = None     , __lowerCamelCase					:			bool = False			):
       if already_has_special_tokens:
           return super().get_special_tokens_mask(
               token_ids_a=__lowerCamelCase     , token_ids_a=__lowerCamelCase     , already_has_special_tokens=__lowerCamelCase			)

       if token_ids_a is None:
           return [1] + ([0] * len(__lowerCamelCase			)) + [1]
       return [1] + ([0] * len(__lowerCamelCase			)) + [1, 1] + ([0] * len(__lowerCamelCase			)) + [1]





   def      _snake_case		(       self					:			Optional[Any]     , __lowerCamelCase					:			List[int]     , __lowerCamelCase					:			Optional[List[int]] = None			):
       SCREAMING_SNAKE_CASE										=   [self.sep_token_id]
       SCREAMING_SNAKE_CASE										=   [self.cls_token_id]

       if token_ids_a is None:
           return len(cls + token_ids_a + sep			) * [0]
       return len(cls + token_ids_a + sep + sep + token_ids_a + sep			) * [0]





   def      _snake_case		(       self					:			Optional[int]     , __lowerCamelCase					:			Dict     , __lowerCamelCase					:			Any=False     , **__lowerCamelCase					:			Union[str, Any]			):
       SCREAMING_SNAKE_CASE										=   kwargs.pop("add_prefix_space"     , self.add_prefix_space			)
       if (is_split_into_words or add_prefix_space) and (len(__lowerCamelCase			) > 0 and not text[0].isspace()):
           SCREAMING_SNAKE_CASE										=   " " + text
       return (text, kwargs)





   def      _snake_case		(       self					:			Any     , __lowerCamelCase					:			List[int]     , __lowerCamelCase					:			Optional[List[int]] = None			):
       return token_ids_a + [self.eos_token_id]





   def      _snake_case		(       self					:			int     , __lowerCamelCase					:			"Conversation"			):
       SCREAMING_SNAKE_CASE										=   []
       for is_user, text in conversation.iter_texts():
           if is_user:
               # We need to space prefix as it's being done within blenderbot
               inputs.append(" " + text			)
           else:
               # Generated responses should contain them already.
               inputs.append(__lowerCamelCase			)

       SCREAMING_SNAKE_CASE										=   "  ".join(__lowerCamelCase			)
       SCREAMING_SNAKE_CASE										=   self.encode(__lowerCamelCase			)
       if len(__lowerCamelCase			) > self.model_max_length:
           SCREAMING_SNAKE_CASE										=   input_ids[-self.model_max_length :]
           logger.warning(f"Trimmed input from conversation as it was longer than {self.model_max_length} tokens."			)
       return input_ids