
"""simple docstring"""

import json
from typing import TYPE_CHECKING, List, Optional, Tuple

from tokenizers import pre_tokenizers, processors

from ...tokenization_utils_base import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_blenderbot import BlenderbotTokenizer


if TYPE_CHECKING:
   from transformers.pipelines.conversational import Conversation

SCREAMING_SNAKE_CASE_            =    logging.get_logger(__name__)


SCREAMING_SNAKE_CASE_            =    {
    '''vocab_file''': '''vocab.json''',
    '''merges_file''': '''merges.txt''',
    '''tokenizer_config_file''': '''tokenizer_config.json''',
}

SCREAMING_SNAKE_CASE_            =    {
    '''vocab_file''': {'''facebook/blenderbot-3B''': '''https://huggingface.co/facebook/blenderbot-3B/resolve/main/vocab.json'''},
    '''merges_file''': {'''facebook/blenderbot-3B''': '''https://huggingface.co/facebook/blenderbot-3B/resolve/main/merges.txt'''},
    '''tokenizer_config_file''': {
        '''facebook/blenderbot-3B''': '''https://huggingface.co/facebook/blenderbot-3B/resolve/main/tokenizer_config.json'''
    },
}

SCREAMING_SNAKE_CASE_            =    {'''facebook/blenderbot-3B''': 128}
class   _UpperCAmelCase				(		SCREAMING_SNAKE_CASE_				):
      __SCREAMING_SNAKE_CASE :				int				   =						VOCAB_FILES_NAMES
      __SCREAMING_SNAKE_CASE :				Dict				   =						PRETRAINED_VOCAB_FILES_MAP
      __SCREAMING_SNAKE_CASE :				int				   =						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      __SCREAMING_SNAKE_CASE :				Optional[Any]				   =						["input_ids", "attention_mask"]
      __SCREAMING_SNAKE_CASE :				Union[str, Any]				   =						BlenderbotTokenizer




      def __init__(				self ,							lowercase_=None ,							lowercase_=None ,							lowercase_=None ,							lowercase_="replace" ,							lowercase_="<s>" ,							lowercase_="</s>" ,							lowercase_="</s>" ,							lowercase_="<s>" ,							lowercase_="<unk>" ,							lowercase_="<pad>" ,							lowercase_="<mask>" ,							lowercase_=False ,							lowercase_=True ,							**lowercase_ ,							)    ->       Optional[Any]:
          super().__init__(
              lowercase_ ,							lowercase_ ,							tokenizer_file=lowercase_ ,							errors=lowercase_ ,							bos_token=lowercase_ ,							eos_token=lowercase_ ,							sep_token=lowercase_ ,							cls_token=lowercase_ ,							unk_token=lowercase_ ,							pad_token=lowercase_ ,							mask_token=lowercase_ ,							add_prefix_space=lowercase_ ,							trim_offsets=lowercase_ ,							**lowercase_ ,							)

          UpperCAmelCase          =       json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()				)
          if pre_tok_state.get('add_prefix_space' ,							lowercase_				) != add_prefix_space:
              UpperCAmelCase          =       getattr(lowercase_ ,							pre_tok_state.pop('type'				)				)
              UpperCAmelCase          =       add_prefix_space
              UpperCAmelCase          =       pre_tok_class(**lowercase_				)

          UpperCAmelCase          =       add_prefix_space

          UpperCAmelCase          =       'post_processor'
          UpperCAmelCase          =       getattr(self.backend_tokenizer ,							lowercase_ ,							lowercase_				)
          if tokenizer_component_instance:
              UpperCAmelCase          =       json.loads(tokenizer_component_instance.__getstate__()				)

              # The lists 'sep' and 'cls' must be cased in tuples for the object `post_processor_class`
              if "sep" in state:
                  UpperCAmelCase          =       tuple(state['sep']				)
              if "cls" in state:
                  UpperCAmelCase          =       tuple(state['cls']				)

              UpperCAmelCase          =       False

              if state.get('add_prefix_space' ,							lowercase_				) != add_prefix_space:
                  UpperCAmelCase          =       add_prefix_space
                  UpperCAmelCase          =       True

              if state.get('trim_offsets' ,							lowercase_				) != trim_offsets:
                  UpperCAmelCase          =       trim_offsets
                  UpperCAmelCase          =       True

              if changes_to_apply:
                  UpperCAmelCase          =       getattr(lowercase_ ,							state.pop('type'				)				)
                  UpperCAmelCase          =       component_class(**lowercase_				)
                  setattr(self.backend_tokenizer ,							lowercase_ ,							lowercase_				)




      @property
      # Copied from transformers.models.roberta.tokenization_roberta_fast.RobertaTokenizerFast.mask_token with Roberta->Blenderbot, RoBERTa->Blenderbot
      def      a_   (				self				)    ->       str:
          if self._mask_token is None:
              if self.verbose:
                  logger.error('Using mask_token, but it is not set yet.'				)
              return None
          return str(self._mask_token				)




      @mask_token.setter
      def      a_   (				self ,							lowercase_				)    ->       Tuple:
          UpperCAmelCase          =       AddedToken(lowercase_ ,							lstrip=lowercase_ ,							rstrip=lowercase_				) if isinstance(lowercase_ ,							lowercase_				) else value
          UpperCAmelCase          =       value




      def      a_   (				self ,							*lowercase_ ,							**lowercase_				)    ->       BatchEncoding:
          UpperCAmelCase          =       kwargs.get('is_split_into_words' ,							lowercase_				)
          assert self.add_prefix_space or not is_split_into_words, (
              F"You need to instantiate {self.__class__.__name__} with add_prefix_space=True "
              "to use it with pretokenized inputs."
          )

          return super()._batch_encode_plus(*lowercase_ ,							**lowercase_				)




      def      a_   (				self ,							*lowercase_ ,							**lowercase_				)    ->       BatchEncoding:
          UpperCAmelCase          =       kwargs.get('is_split_into_words' ,							lowercase_				)

          assert self.add_prefix_space or not is_split_into_words, (
              F"You need to instantiate {self.__class__.__name__} with add_prefix_space=True "
              "to use it with pretokenized inputs."
          )

          return super()._encode_plus(*lowercase_ ,							**lowercase_				)




      def      a_   (				self ,							lowercase_ ,							lowercase_ = None				)    ->       Tuple[str]:
          UpperCAmelCase          =       self._tokenizer.model.save(lowercase_ ,							name=lowercase_				)
          return tuple(lowercase_				)




      def      a_   (				self ,							lowercase_ ,							lowercase_ = None				)    ->       List[int]:
          UpperCAmelCase          =       [self.sep_token_id]
          UpperCAmelCase          =       [self.cls_token_id]

          if token_ids_a is None:
              return len(cls + token_ids_a + sep				) * [0]
          return len(cls + token_ids_a + sep + sep + token_ids_a + sep				) * [0]




      def      a_   (				self ,							lowercase_ ,							lowercase_ = None				)    ->       int:
          return token_ids_a + [self.eos_token_id]




      def      a_   (				self ,							lowercase_				)    ->       List[int]:
          UpperCAmelCase          =       []
          for is_user, text in conversation.iter_texts():
              if is_user:
                  # We need to space prefix as it's being done within blenderbot
                  inputs.append(' ' + text				)
              else:
                  # Generated responses should contain them already.
                  inputs.append(lowercase_				)

          UpperCAmelCase          =       '  '.join(lowercase_				)
          UpperCAmelCase          =       self.encode(lowercase_				)
          if len(lowercase_				) > self.model_max_length:
              UpperCAmelCase          =       input_ids[-self.model_max_length :]
              logger.warning(F"Trimmed input from conversation as it was longer than {self.model_max_length} tokens."				)
          return input_ids





"""simple docstring"""

from ...configuration_utils import PretrainedConfig
from ...utils import logging


SCREAMING_SNAKE_CASE_            =    logging.get_logger(__name__)

SCREAMING_SNAKE_CASE_            =    {
    '''tiiuae/falcon-40b''': '''https://huggingface.co/tiiuae/falcon-40b/resolve/main/config.json''',
    '''tiiuae/falcon-7b''': '''https://huggingface.co/tiiuae/falcon-7b/resolve/main/config.json''',
}
class   _UpperCAmelCase				(		SCREAMING_SNAKE_CASE_				):
      __SCREAMING_SNAKE_CASE :				Optional[Any]				   =						"falcon"
      __SCREAMING_SNAKE_CASE :				List[Any]				   =						["past_key_values"]




      def __init__(				self ,							lowercase_=6_5_0_2_4 ,							lowercase_=4_5_4_4 ,							lowercase_=3_2 ,							lowercase_=7_1 ,							lowercase_=1E-5 ,							lowercase_=0.0_2 ,							lowercase_=True ,							lowercase_=0.0 ,							lowercase_=0.0 ,							lowercase_=None ,							lowercase_=False ,							lowercase_=False ,							lowercase_=True ,							lowercase_=True ,							lowercase_=False ,							lowercase_=1_1 ,							lowercase_=1_1 ,							**lowercase_ ,							)    ->       Any:
          UpperCAmelCase          =       vocab_size
          # Backward compatibility with n_embed kwarg
          UpperCAmelCase          =       kwargs.pop('n_embed' ,							lowercase_				)
          UpperCAmelCase          =       hidden_size if n_embed is None else n_embed
          UpperCAmelCase          =       num_hidden_layers
          UpperCAmelCase          =       num_attention_heads
          UpperCAmelCase          =       layer_norm_epsilon
          UpperCAmelCase          =       initializer_range
          UpperCAmelCase          =       use_cache
          UpperCAmelCase          =       hidden_dropout
          UpperCAmelCase          =       attention_dropout

          UpperCAmelCase          =       bos_token_id
          UpperCAmelCase          =       eos_token_id
          UpperCAmelCase          =       num_attention_heads if num_kv_heads is None else num_kv_heads
          UpperCAmelCase          =       alibi
          UpperCAmelCase          =       new_decoder_architecture
          UpperCAmelCase          =       multi_query  # Ignored when new_decoder_architecture is True
          UpperCAmelCase          =       parallel_attn
          UpperCAmelCase          =       bias

          super().__init__(bos_token_id=lowercase_ ,							eos_token_id=lowercase_ ,							**lowercase_				)




      @property
      def      a_   (				self				)    ->       List[Any]:
          return self.hidden_size // self.num_attention_heads




      @property
      def      a_   (				self				)    ->       List[str]:
          return not self.alibi



