






from collections import OrderedDict
from typing import Any, Mapping, Optional

from ... import PreTrainedTokenizer
from ...configuration_utils import PretrainedConfig
from ...file_utils import TensorType, is_torch_available
from ...onnx import OnnxConfig, OnnxConfigWithPast, OnnxSeqaSeqConfigWithPast
from ...onnx.utils import compute_effective_axis_dimension
from ...utils import logging


UpperCAmelCase__							  =						logging.get_logger(__name__)

UpperCAmelCase__							  =						{
    "facebook/blenderbot_small-90M": "https://huggingface.co/facebook/blenderbot_small-90M/resolve/main/config.json",
    # See all BlenderbotSmall models at https://huggingface.co/models?filter=blenderbot_small
}






class 						lowercase_   (      lowercase  ):






      '''simple docstring'''




      __snake_case	    =    '''blenderbot-small'''
      __snake_case	    =    ['''past_key_values''']
      __snake_case	    =    {'''num_attention_heads''': '''encoder_attention_heads''', '''hidden_size''': '''d_model'''}







      def __init__( self :		Dict					,   __UpperCAmelCase :		Optional[int]=50_265					,   __UpperCAmelCase :		Optional[Any]=512					,   __UpperCAmelCase :		Dict=8					,   __UpperCAmelCase :		Tuple=2_048					,   __UpperCAmelCase :		Tuple=16					,   __UpperCAmelCase :		str=8					,   __UpperCAmelCase :		str=2_048					,   __UpperCAmelCase :		List[str]=16					,   __UpperCAmelCase :		List[str]=0.0					,   __UpperCAmelCase :		Optional[Any]=0.0					,   __UpperCAmelCase :		str=True					,   __UpperCAmelCase :		str=True					,   __UpperCAmelCase :		Tuple="gelu"					,   __UpperCAmelCase :		List[Any]=512					,   __UpperCAmelCase :		List[Any]=0.1					,   __UpperCAmelCase :		int=0.0					,   __UpperCAmelCase :		int=0.0					,   __UpperCAmelCase :		Any=0.02					,   __UpperCAmelCase :		Optional[int]=1					,   __UpperCAmelCase :		str=False					,   __UpperCAmelCase :		Optional[Any]=0					,   __UpperCAmelCase :		List[Any]=1					,   __UpperCAmelCase :		Optional[Any]=2					,   __UpperCAmelCase :		int=2					,   **__UpperCAmelCase :		Dict					,   )       ->List[Any]:




       """simple docstring"""


       a						    = vocab_size
       a						    = max_position_embeddings
       a						    = d_model
       a						    = encoder_ffn_dim
       a						    = encoder_layers
       a						    = encoder_attention_heads
       a						    = decoder_ffn_dim
       a						    = decoder_layers
       a						    = decoder_attention_heads
       a						    = dropout
       a						    = attention_dropout
       a						    = activation_dropout
       a						    = activation_function
       a						    = init_std
       a						    = encoder_layerdrop
       a						    = decoder_layerdrop
       a						    = use_cache
       a						    = encoder_layers
       a						    = scale_embedding  # scale factor will be sqrt(d_model) if True

       super().__init__(
           pad_token_id=__UpperCAmelCase					,   bos_token_id=__UpperCAmelCase					,   eos_token_id=__UpperCAmelCase					,   is_encoder_decoder=__UpperCAmelCase					,   decoder_start_token_id=__UpperCAmelCase					,   forced_eos_token_id=__UpperCAmelCase					,   **__UpperCAmelCase					,   )






class 						lowercase_   (      lowercase  ):






      '''simple docstring'''




      @property
      def  __lowerCAmelCase   ( self :		str    )       ->Mapping[str, Mapping[int, str]]:




       """simple docstring"""


       if self.task in ["default", "seq2seq-lm"]:
        a						    = OrderedDict(
            [
                ('''input_ids''', {0: '''batch''', 1: '''encoder_sequence'''}),
                ('''attention_mask''', {0: '''batch''', 1: '''encoder_sequence'''}),
            ]    )

        if self.use_past:
         a						    = {0: '''batch'''}
         a						    = {0: '''batch''', 1: '''past_decoder_sequence + sequence'''}
        else:
         a						    = {0: '''batch''', 1: '''decoder_sequence'''}
         a						    = {0: '''batch''', 1: '''decoder_sequence'''}

        if self.use_past:
         self.fill_with_past_key_values_(__UpperCAmelCase					,   direction='''inputs'''    )
       elif self.task == "causal-lm":
        # TODO: figure this case out.
        a						    = OrderedDict(
            [
                ('''input_ids''', {0: '''batch''', 1: '''encoder_sequence'''}),
                ('''attention_mask''', {0: '''batch''', 1: '''encoder_sequence'''}),
            ]    )
        if self.use_past:
         a    ,	a						    = self.num_layers
         for i in range(__UpperCAmelCase    ):
          a						    = {0: '''batch''', 2: '''past_sequence + sequence'''}
          a						    = {0: '''batch''', 2: '''past_sequence + sequence'''}
       else:
        a						    = OrderedDict(
            [
                ('''input_ids''', {0: '''batch''', 1: '''encoder_sequence'''}),
                ('''attention_mask''', {0: '''batch''', 1: '''encoder_sequence'''}),
                ('''decoder_input_ids''', {0: '''batch''', 1: '''decoder_sequence'''}),
                ('''decoder_attention_mask''', {0: '''batch''', 1: '''decoder_sequence'''}),
            ]    )

       return common_inputs







      @property
      def  __lowerCAmelCase   ( self :		str    )       ->Mapping[str, Mapping[int, str]]:




       """simple docstring"""


       if self.task in ["default", "seq2seq-lm"]:
        a						    = super().outputs
       else:
        a						    = super(__UpperCAmelCase					,   self    ).outputs
        if self.use_past:
         a    ,	a						    = self.num_layers
         for i in range(__UpperCAmelCase    ):
          a						    = {0: '''batch''', 2: '''past_sequence + sequence'''}
          a						    = {0: '''batch''', 2: '''past_sequence + sequence'''}
       return common_outputs







      def  __lowerCAmelCase   ( self :		List[str]					,   __UpperCAmelCase :		PreTrainedTokenizer					,   __UpperCAmelCase :		int = -1					,   __UpperCAmelCase :		int = -1					,   __UpperCAmelCase :		bool = False					,   __UpperCAmelCase :		Optional[TensorType] = None					,   )       ->Mapping[str, Any]:




       """simple docstring"""


       a						    = self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
           __UpperCAmelCase					,   __UpperCAmelCase					,   __UpperCAmelCase					,   __UpperCAmelCase					,   __UpperCAmelCase    )

       # Generate decoder inputs
       a						    = seq_length if not self.use_past else 1
       a						    = self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
           __UpperCAmelCase					,   __UpperCAmelCase					,   __UpperCAmelCase					,   __UpperCAmelCase					,   __UpperCAmelCase    )
       a						    = {F"""decoder_{name}""": tensor for name, tensor in decoder_inputs.items()}
       a						    = dict(**__UpperCAmelCase					,   **__UpperCAmelCase    )

       if self.use_past:
        if not is_torch_available():
         raise ValueError('''Cannot generate dummy past_keys inputs without PyTorch installed.'''    )
        else:
         import torch
        a    ,	a						    = common_inputs['''input_ids'''].shape
        a						    = common_inputs['''decoder_input_ids'''].shape[1]
        a    ,	a						    = self.num_attention_heads
        a						    = (
            batch,
            num_encoder_attention_heads,
            encoder_seq_length,
            self._config.hidden_size // num_encoder_attention_heads,
        )
        a						    = decoder_seq_length + 3
        a						    = (
            batch,
            num_decoder_attention_heads,
            decoder_past_length,
            self._config.hidden_size // num_decoder_attention_heads,
        )

        a						    = torch.cat(
            [common_inputs['''decoder_attention_mask'''], torch.ones(__UpperCAmelCase					,   __UpperCAmelCase    )]					,   dim=1    )

        a						    = []
        # If the number of encoder and decoder layers are present in the model configuration, both are considered
        a    ,	a						    = self.num_layers
        a						    = min(__UpperCAmelCase					,   __UpperCAmelCase    )
        a						    = max(__UpperCAmelCase					,   __UpperCAmelCase    ) - min_num_layers
        a						    = '''encoder''' if num_encoder_layers > num_decoder_layers else '''decoder'''

        for _ in range(__UpperCAmelCase    ):
         common_inputs["past_key_values"].append(
             (
                 torch.zeros(__UpperCAmelCase    ),
                 torch.zeros(__UpperCAmelCase    ),
                 torch.zeros(__UpperCAmelCase    ),
                 torch.zeros(__UpperCAmelCase    ),
             )    )
        # TODO: test this.
        a						    = encoder_shape if remaining_side_name == '''encoder''' else decoder_shape
        for _ in range(__UpperCAmelCase					,   __UpperCAmelCase    ):
         common_inputs["past_key_values"].append((torch.zeros(__UpperCAmelCase    ), torch.zeros(__UpperCAmelCase    ))    )
       return common_inputs







      def  __lowerCAmelCase   ( self :		Tuple					,   __UpperCAmelCase :		PreTrainedTokenizer					,   __UpperCAmelCase :		int = -1					,   __UpperCAmelCase :		int = -1					,   __UpperCAmelCase :		bool = False					,   __UpperCAmelCase :		Optional[TensorType] = None					,   )       ->Mapping[str, Any]:




       """simple docstring"""


       a						    = self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
           __UpperCAmelCase					,   __UpperCAmelCase					,   __UpperCAmelCase					,   __UpperCAmelCase					,   __UpperCAmelCase    )

       if self.use_past:
        if not is_torch_available():
         raise ValueError('''Cannot generate dummy past_keys inputs without PyTorch installed.'''    )
        else:
         import torch
        a    ,	a						    = common_inputs['''input_ids'''].shape
        # Not using the same length for past_key_values
        a						    = seqlen + 2
        a    ,	a						    = self.num_layers
        a    ,	a						    = self.num_attention_heads
        a						    = (
            batch,
            num_encoder_attention_heads,
            past_key_values_length,
            self._config.hidden_size // num_encoder_attention_heads,
        )

        a						    = common_inputs['''attention_mask'''].dtype
        a						    = torch.cat(
            [common_inputs['''attention_mask'''], torch.ones(__UpperCAmelCase					,   __UpperCAmelCase					,   dtype=__UpperCAmelCase    )]					,   dim=1    )
        a						    = [
            (torch.zeros(__UpperCAmelCase    ), torch.zeros(__UpperCAmelCase    )) for _ in range(__UpperCAmelCase    )
        ]
       return common_inputs







      def  __lowerCAmelCase   ( self :		Union[str, Any]					,   __UpperCAmelCase :		PreTrainedTokenizer					,   __UpperCAmelCase :		int = -1					,   __UpperCAmelCase :		int = -1					,   __UpperCAmelCase :		bool = False					,   __UpperCAmelCase :		Optional[TensorType] = None					,   )       ->Mapping[str, Any]:




       """simple docstring"""


       a						    = compute_effective_axis_dimension(
           __UpperCAmelCase					,   fixed_dimension=OnnxConfig.default_fixed_batch					,   num_token_to_add=0    )

       # If dynamic axis (-1) we forward with a fixed dimension of 8 tokens to avoid optimizations made by ONNX
       a						    = tokenizer.num_special_tokens_to_add(__UpperCAmelCase    )
       a						    = compute_effective_axis_dimension(
           __UpperCAmelCase					,   fixed_dimension=OnnxConfig.default_fixed_sequence					,   num_token_to_add=__UpperCAmelCase    )

       # Generate dummy inputs according to compute batch and sequence
       a						    = [''' '''.join([tokenizer.unk_token]    ) * seq_length] * batch_size
       a						    = dict(tokenizer(__UpperCAmelCase					,   return_tensors=__UpperCAmelCase    )    )
       return common_inputs







      def  __lowerCAmelCase   ( self :		List[str]					,   __UpperCAmelCase :		PreTrainedTokenizer					,   __UpperCAmelCase :		int = -1					,   __UpperCAmelCase :		int = -1					,   __UpperCAmelCase :		bool = False					,   __UpperCAmelCase :		Optional[TensorType] = None					,   )       ->Mapping[str, Any]:




       """simple docstring"""


       if self.task in ["default", "seq2seq-lm"]:
        a						    = self._generate_dummy_inputs_for_default_and_seqaseq_lm(
            __UpperCAmelCase					,   batch_size=__UpperCAmelCase					,   seq_length=__UpperCAmelCase					,   is_pair=__UpperCAmelCase					,   framework=__UpperCAmelCase    )

       elif self.task == "causal-lm":
        a						    = self._generate_dummy_inputs_for_causal_lm(
            __UpperCAmelCase					,   batch_size=__UpperCAmelCase					,   seq_length=__UpperCAmelCase					,   is_pair=__UpperCAmelCase					,   framework=__UpperCAmelCase    )
       else:
        a						    = self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
            __UpperCAmelCase					,   batch_size=__UpperCAmelCase					,   seq_length=__UpperCAmelCase					,   is_pair=__UpperCAmelCase					,   framework=__UpperCAmelCase    )

       return common_inputs







      def  __lowerCAmelCase   ( self :		int					,   __UpperCAmelCase :		Optional[int]					,   __UpperCAmelCase :		Optional[Any]					,   __UpperCAmelCase :		List[Any]					,   __UpperCAmelCase :		int    )       ->Union[str, Any]:




       """simple docstring"""


       if self.task in ["default", "seq2seq-lm"]:
        a						    = super()._flatten_past_key_values_(__UpperCAmelCase					,   __UpperCAmelCase					,   __UpperCAmelCase					,   __UpperCAmelCase    )
       else:
        a						    = super(__UpperCAmelCase					,   self    )._flatten_past_key_values_(
            __UpperCAmelCase					,   __UpperCAmelCase					,   __UpperCAmelCase					,   __UpperCAmelCase    )










import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
     from .tokenization_pegasus import PegasusTokenizer
else:
     UpperCAmelCase__							  =						None


UpperCAmelCase__							  =						logging.get_logger(__name__)


UpperCAmelCase__							  =						"▁"

UpperCAmelCase__							  =						{"vocab_file": "spiece.model", "tokenizer_file": "tokenizer.json"}

UpperCAmelCase__							  =						{
    "vocab_file": {"google/pegasus-xsum": "https://huggingface.co/google/pegasus-xsum/resolve/main/spiece.model"},
    "tokenizer_file": {
        "google/pegasus-xsum": "https://huggingface.co/google/pegasus-xsum/resolve/main/tokenizer.json"
    },
}

UpperCAmelCase__							  =						{
    "google/pegasus-xsum": 512,
}






class 						lowercase_   (      lowercase  ):






      '''simple docstring'''




      __snake_case	    =    VOCAB_FILES_NAMES
      __snake_case	    =    PRETRAINED_VOCAB_FILES_MAP
      __snake_case	    =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      __snake_case	    =    PegasusTokenizer
      __snake_case	    =    ['''input_ids''', '''attention_mask''']







      def __init__( self :		List[Any]					,   __UpperCAmelCase :		str=None					,   __UpperCAmelCase :		List[Any]=None					,   __UpperCAmelCase :		Any="<pad>"					,   __UpperCAmelCase :		str="</s>"					,   __UpperCAmelCase :		List[str]="<unk>"					,   __UpperCAmelCase :		List[str]="<mask_2>"					,   __UpperCAmelCase :		Tuple="<mask_1>"					,   __UpperCAmelCase :		int=None					,   __UpperCAmelCase :		str=103					,   **__UpperCAmelCase :		Optional[int]					,   )       ->Dict:




       """simple docstring"""


       a						    = offset

       if additional_special_tokens is not None:
        if not isinstance(__UpperCAmelCase					,   __UpperCAmelCase    ):
         raise TypeError(
             F"""additional_special_tokens should be of type {type(__UpperCAmelCase    )}, but is"""
             F""" {type(__UpperCAmelCase    )}"""    )

        a						    = (
            ([mask_token_sent] + additional_special_tokens)
            if mask_token_sent not in additional_special_tokens and mask_token_sent is not None
            else additional_special_tokens
        )
        # fill additional tokens with ..., <unk_token_102> in case not all additional tokens are already taken
        additional_special_tokens_extended += [
            F"""<unk_{i}>""" for i in range(len(__UpperCAmelCase    )					,   self.offset - 1    )
        ]

        if len(set(__UpperCAmelCase    )    ) != len(__UpperCAmelCase    ):
         raise ValueError(
             '''Please make sure that the provided additional_special_tokens do not contain an incorrectly'''
             F""" shifted list of <unk_x> tokens. Found {additional_special_tokens_extended}."""    )
        a						    = additional_special_tokens_extended
       else:
        a						    = [mask_token_sent] if mask_token_sent is not None else []
        additional_special_tokens += [F"""<unk_{i}>""" for i in range(2					,   self.offset    )]

       super().__init__(
           __UpperCAmelCase					,   tokenizer_file=__UpperCAmelCase					,   pad_token=__UpperCAmelCase					,   eos_token=__UpperCAmelCase					,   unk_token=__UpperCAmelCase					,   mask_token=__UpperCAmelCase					,   mask_token_sent=__UpperCAmelCase					,   offset=__UpperCAmelCase					,   additional_special_tokens=__UpperCAmelCase					,   **__UpperCAmelCase					,   )
       a						    = vocab_file
       a						    = False if not self.vocab_file else True







      def  __lowerCAmelCase   ( self :		Optional[Any]					,   __UpperCAmelCase :		int    )       ->int:




       """simple docstring"""


       a						    = set(self.all_special_ids    )  # call it once instead of inside list comp
       all_special_ids.remove(self.unk_token_id    )  # <unk> is only sometimes special

       if all_special_ids != set(range(len(self.additional_special_tokens    ) + 3    )    ):
        raise ValueError(
            '''There should be 3 special tokens: mask_token, pad_token, and eos_token +'''
            F""" {len(self.additional_special_tokens    )} additional_special_tokens, but got {all_special_ids}"""    )

       return [1 if x in all_special_ids else 0 for x in seq]







      def  __lowerCAmelCase   ( self :		Dict					,   __UpperCAmelCase :		List					,   __UpperCAmelCase :		Optional[List] = None					,   __UpperCAmelCase :		bool = False    )       ->List[int]:




       """simple docstring"""


       if already_has_special_tokens:
        return self._special_token_mask(__UpperCAmelCase    )
       elif token_ids_a is None:
        return self._special_token_mask(__UpperCAmelCase    ) + [1]
       else:
        return self._special_token_mask(token_ids_a + token_ids_a    ) + [1]







      def  __lowerCAmelCase   ( self :		Union[str, Any]					,   __UpperCAmelCase :		int					,   __UpperCAmelCase :		Optional[int]=None    )       ->List[int]:




       """simple docstring"""


       if token_ids_a is None:
        return token_ids_a + [self.eos_token_id]
       # We don't expect to process pairs, but leave the pair logic for API consistency
       return token_ids_a + token_ids_a + [self.eos_token_id]







      def  __lowerCAmelCase   ( self :		Dict					,   __UpperCAmelCase :		str					,   __UpperCAmelCase :		Optional[str] = None    )       ->Tuple[str]:




       """simple docstring"""


       if not self.can_save_slow_tokenizer:
        raise ValueError(
            '''Your fast tokenizer does not have the necessary information to save the vocabulary for a slow '''
            '''tokenizer.'''    )

       if not os.path.isdir(__UpperCAmelCase    ):
        logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""    )
        return
       a						    = os.path.join(
           __UpperCAmelCase					,   (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']    )

       if os.path.abspath(self.vocab_file    ) != os.path.abspath(__UpperCAmelCase    ):
        copyfile(self.vocab_file					,   __UpperCAmelCase    )

       return (out_vocab_file,)


