





import os
from shutil import copyfile
from typing import List, Optional, Tuple

from tokenizers import processors

from ...tokenization_utils import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
       from .tokenization_mbartaa import MBartaaTokenizer
else:
       UpperCamelCase__          =None


UpperCamelCase__          =logging.get_logger(__name__)

UpperCamelCase__          ={'vocab_file': 'sentencepiece.bpe.model', 'tokenizer_file': 'tokenizer.json'}

UpperCamelCase__          ={
    'vocab_file': {
        'facebook/mbart-large-50-one-to-many-mmt': (
            'https://huggingface.co/facebook/mbart-large-50-one-to-many-mmt/resolve/main/sentencepiece.bpe.model'
        ),
    },
    'tokenizer_file': {
        'facebook/mbart-large-50-one-to-many-mmt': (
            'https://huggingface.co/facebook/mbart-large-50-one-to-many-mmt/resolve/main/tokenizer.json'
        ),
    },
}

UpperCamelCase__          ={
    'facebook/mbart-large-50-one-to-many-mmt': 1024,
}

# fmt: off
UpperCamelCase__          =['ar_AR', 'cs_CZ', 'de_DE', 'en_XX', 'es_XX', 'et_EE', 'fi_FI', 'fr_XX', 'gu_IN', 'hi_IN', 'it_IT', 'ja_XX', 'kk_KZ', 'ko_KR', 'lt_LT', 'lv_LV', 'my_MM', 'ne_NP', 'nl_XX', 'ro_RO', 'ru_RU', 'si_LK', 'tr_TR', 'vi_VN', 'zh_CN', 'af_ZA', 'az_AZ', 'bn_IN', 'fa_IR', 'he_IL', 'hr_HR', 'id_ID', 'ka_GE', 'km_KH', 'mk_MK', 'ml_IN', 'mn_MN', 'mr_IN', 'pl_PL', 'ps_AF', 'pt_XX', 'sv_SE', 'sw_KE', 'ta_IN', 'te_IN', 'th_TH', 'tl_XX', 'uk_UA', 'ur_PK', 'xh_ZA', 'gl_ES', 'sl_SI']





class 	lowerCAmelCase__(       __lowercase			):






       '''simple docstring'''


       __snake_case       	=			VOCAB_FILES_NAMES
       __snake_case       	=			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
       __snake_case       	=			PRETRAINED_VOCAB_FILES_MAP
       __snake_case       	=			['input_ids', 'attention_mask']
       __snake_case       	=			MBartaaTokenizer

       __snake_case       	=			[]
       __snake_case       	=			[]




       def __init__(					self			,  __lowerCamelCase=None			,  __lowerCamelCase=None			,  __lowerCamelCase=None			,  __lowerCamelCase=None			,  __lowerCamelCase="</s>"			,  __lowerCamelCase="</s>"			,  __lowerCamelCase="<s>"			,  __lowerCamelCase="<unk>"			,  __lowerCamelCase="<pad>"			,  __lowerCamelCase="<mask>"			,  **__lowerCamelCase			,  )	->   List[Any]:
            # Mask token behave like a normal word, i.e. include the space before it
            _SCREAMING_SNAKE_CASE				:		Tuple  			=							AddedToken(__lowerCamelCase			,  lstrip=__lowerCamelCase			,  rstrip=__lowerCamelCase							) if isinstance(__lowerCamelCase			,  __lowerCamelCase							) else mask_token

            _SCREAMING_SNAKE_CASE				:		Tuple  			=							kwargs.get("additional_special_tokens"			,  []							)
            kwargs["additional_special_tokens"] += [
                code for code in FAIRSEQ_LANGUAGE_CODES if code not in kwargs["additional_special_tokens"]
            ]

            super().__init__(
                __lowerCamelCase			,  src_lang=__lowerCamelCase			,  tgt_lang=__lowerCamelCase			,  tokenizer_file=__lowerCamelCase			,  eos_token=__lowerCamelCase			,  sep_token=__lowerCamelCase			,  cls_token=__lowerCamelCase			,  unk_token=__lowerCamelCase			,  pad_token=__lowerCamelCase			,  mask_token=__lowerCamelCase			,  **__lowerCamelCase			,  )

            _SCREAMING_SNAKE_CASE				:		str  			=							vocab_file
            _SCREAMING_SNAKE_CASE				:		Union[str, Any]  			=							False if not self.vocab_file else True

            _SCREAMING_SNAKE_CASE				:		Union[str, Any]  			=							{
                lang_code: self.convert_tokens_to_ids(__lowerCamelCase							) for lang_code in FAIRSEQ_LANGUAGE_CODES
            }

            _SCREAMING_SNAKE_CASE				:		int  			=							src_lang if src_lang is not None else "en_XX"
            _SCREAMING_SNAKE_CASE				:		Optional[Any]  			=							tgt_lang
            _SCREAMING_SNAKE_CASE				:		Optional[int]  			=							self.lang_code_to_id[self._src_lang]
            self.set_src_lang_special_tokens(self._src_lang							)




       @property
       def 			UpperCamelCase_	(					self							)	->   str:
            return self._src_lang




       @src_lang.setter
       def 			UpperCamelCase_	(					self			,  __lowerCamelCase							)	->   None:
            _SCREAMING_SNAKE_CASE				:		List[str]  			=							new_src_lang
            self.set_src_lang_special_tokens(self._src_lang							)




       def 			UpperCamelCase_	(					self			,  __lowerCamelCase			,  __lowerCamelCase = None							)	->   List[int]:
            if token_ids_a is None:
                 return self.prefix_tokens + token_ids_a + self.suffix_tokens
            # We don't expect to process pairs, but leave the pair logic for API consistency
            return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens




       def 			UpperCamelCase_	(					self			,  __lowerCamelCase			,  __lowerCamelCase = "en_XX"			,  __lowerCamelCase = None			,  __lowerCamelCase = "ro_RO"			,  **__lowerCamelCase			,  )	->   BatchEncoding:
            _SCREAMING_SNAKE_CASE				:		str  			=							src_lang
            _SCREAMING_SNAKE_CASE				:		Optional[int]  			=							tgt_lang
            return super().prepare_seqaseq_batch(__lowerCamelCase			,  __lowerCamelCase			,  **__lowerCamelCase							)




       def 			UpperCamelCase_	(					self							)	->   Optional[Any]:
            return self.set_src_lang_special_tokens(self.src_lang							)




       def 			UpperCamelCase_	(					self							)	->   Optional[Any]:
            return self.set_tgt_lang_special_tokens(self.tgt_lang							)




       def 			UpperCamelCase_	(					self			,  __lowerCamelCase							)	->   None:
            _SCREAMING_SNAKE_CASE				:		List[Any]  			=							self.convert_tokens_to_ids(__lowerCamelCase							)
            _SCREAMING_SNAKE_CASE				:		Optional[Any]  			=							[self.cur_lang_code_id]
            _SCREAMING_SNAKE_CASE				:		List[Any]  			=							[self.eos_token_id]

            _SCREAMING_SNAKE_CASE				:		Any  			=							self.convert_ids_to_tokens(self.prefix_tokens							)
            _SCREAMING_SNAKE_CASE				:		int  			=							self.convert_ids_to_tokens(self.suffix_tokens							)

            _SCREAMING_SNAKE_CASE				:		Optional[Any]  			=							processors.TemplateProcessing(
                single=prefix_tokens_str + ["$A"] + suffix_tokens_str			,  pair=prefix_tokens_str + ["$A", "$B"] + suffix_tokens_str			,  special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str			,  self.prefix_tokens + self.suffix_tokens							)							)			,  )




       def 			UpperCamelCase_	(					self			,  __lowerCamelCase							)	->   None:
            _SCREAMING_SNAKE_CASE				:		Dict  			=							self.convert_tokens_to_ids(__lowerCamelCase							)
            _SCREAMING_SNAKE_CASE				:		Tuple  			=							[self.cur_lang_code_id]
            _SCREAMING_SNAKE_CASE				:		Optional[Any]  			=							[self.eos_token_id]

            _SCREAMING_SNAKE_CASE				:		Union[str, Any]  			=							self.convert_ids_to_tokens(self.prefix_tokens							)
            _SCREAMING_SNAKE_CASE				:		Tuple  			=							self.convert_ids_to_tokens(self.suffix_tokens							)

            _SCREAMING_SNAKE_CASE				:		List[str]  			=							processors.TemplateProcessing(
                single=prefix_tokens_str + ["$A"] + suffix_tokens_str			,  pair=prefix_tokens_str + ["$A", "$B"] + suffix_tokens_str			,  special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str			,  self.prefix_tokens + self.suffix_tokens							)							)			,  )




       def 			UpperCamelCase_	(					self			,  __lowerCamelCase			,  __lowerCamelCase			,  __lowerCamelCase			,  __lowerCamelCase			,  **__lowerCamelCase							)	->   Optional[int]:
            if src_lang is None or tgt_lang is None:
                 raise ValueError("Translation requires a `src_lang` and a `tgt_lang` for this model"							)
            _SCREAMING_SNAKE_CASE				:		int  			=							src_lang
            _SCREAMING_SNAKE_CASE				:		Tuple  			=							self(__lowerCamelCase			,  add_special_tokens=__lowerCamelCase			,  return_tensors=__lowerCamelCase			,  **__lowerCamelCase							)
            _SCREAMING_SNAKE_CASE				:		str  			=							self.convert_tokens_to_ids(__lowerCamelCase							)
            _SCREAMING_SNAKE_CASE				:		List[str]  			=							tgt_lang_id
            return inputs







       def 			UpperCamelCase_	(					self			,  __lowerCamelCase			,  __lowerCamelCase = None							)	->   Tuple[str]:
            if not self.can_save_slow_tokenizer:
                 raise ValueError(
                     "Your fast tokenizer does not have the necessary information to save the vocabulary for a slow "
                     "tokenizer."							)

            if not os.path.isdir(__lowerCamelCase							):
                 logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""							)
                 return
            _SCREAMING_SNAKE_CASE				:		Dict  			=							os.path.join(
                __lowerCamelCase			,  (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]							)

            if os.path.abspath(self.vocab_file							) != os.path.abspath(__lowerCamelCase							):
                 copyfile(self.vocab_file			,  __lowerCamelCase							)

            return (out_vocab_file,)

from collections import OrderedDict
from typing import Any, Mapping, Optional

from ... import PreTrainedTokenizer
from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig, OnnxConfigWithPast, OnnxSeqaSeqConfigWithPast
from ...onnx.utils import compute_effective_axis_dimension
from ...utils import TensorType, is_torch_available, logging


_A  					= logging.get_logger(__name__)

_A  					= {
    "google/bigbird-pegasus-large-arxiv": (
        "https://huggingface.co/google/bigbird-pegasus-large-arxiv/resolve/main/config.json"
    ),
    "google/bigbird-pegasus-large-pubmed": (
        "https://huggingface.co/google/bigbird-pegasus-large-pubmed/resolve/main/config.json"
    ),
    "google/bigbird-pegasus-large-bigpatent": (
        "https://huggingface.co/google/bigbird-pegasus-large-bigpatent/resolve/main/config.json"
    ),
    # See all BigBirdPegasus models at https://huggingface.co/models?filter=bigbird_pegasus
}







class      lowerCamelCase		(  A_  ):
     UpperCAmelCase__				:							List[Any]           =		"bigbird_pegasus"
     UpperCAmelCase__				:							Tuple           =		["past_key_values"]
     UpperCAmelCase__				:							Optional[Any]           =		{
         "num_attention_heads": "encoder_attention_heads",
         "hidden_size": "d_model",
         "attention_probs_dropout_prob": "attention_dropout",
     }





     def __init__(self      :    Any				,    _A      :    List[str]=9_6_1_0_3				,    _A      :    List[Any]=4_0_9_6				,    _A      :    str=1_6				,    _A      :    Tuple=4_0_9_6				,    _A      :    List[str]=1_6				,    _A      :    Optional[int]=1_6				,    _A      :    List[Any]=4_0_9_6				,    _A      :    Dict=1_6				,    _A      :    Tuple=0.0				,    _A      :    List[str]=0.0				,    _A      :    Dict=True				,    _A      :    int=True				,    _A      :    List[Any]="gelu_new"				,    _A      :    int=1_0_2_4				,    _A      :    int=0.1				,    _A      :    List[Any]=0.0				,    _A      :    Optional[int]=0.0				,    _A      :    Any=0.02				,    _A      :    Dict=2				,    _A      :    List[Any]=0.0				,    _A      :    Any=True				,    _A      :    str=0				,    _A      :    Dict=2				,    _A      :    Any=1				,    _A      :    List[str]="block_sparse"				,    _A      :    int=6_4				,    _A      :    Optional[int]=3				,    _A      :    List[Any]=False				,    **_A      :    Tuple				,    )						->						Any:
         snake_case      =					vocab_size
         snake_case      =					max_position_embeddings
         snake_case      =					d_model
         snake_case      =					encoder_ffn_dim
         snake_case      =					encoder_layers
         snake_case      =					encoder_attention_heads
         snake_case      =					decoder_ffn_dim
         snake_case      =					decoder_layers
         snake_case      =					decoder_attention_heads
         snake_case      =					dropout
         snake_case      =					attention_dropout
         snake_case      =					activation_dropout
         snake_case      =					activation_function
         snake_case      =					init_std
         snake_case      =					encoder_layerdrop
         snake_case      =					decoder_layerdrop
         snake_case      =					classifier_dropout
         snake_case      =					use_cache
         snake_case      =					encoder_layers
         snake_case      =					scale_embedding  # scale factor will be sqrt(d_model) if True

         # extra config
         snake_case      =					attention_type
         snake_case      =					block_size
         snake_case      =					num_random_blocks
         snake_case      =					use_bias

         super().__init__(
             pad_token_id=_A				,    bos_token_id=_A				,    eos_token_id=_A				,    is_encoder_decoder=_A				,    decoder_start_token_id=_A				,    **_A				,    )







class      lowerCamelCase		(  A_  ):





     @property
     def 		UpperCAmelCase(self      :    Any   )						->						Mapping[str, Mapping[int, str]]:
         if self.task in ["default", "seq2seq-lm"]:
             snake_case      =					OrderedDict(
                 [
                     ("input_ids", {0: "batch", 1: "encoder_sequence"}),
                     ("attention_mask", {0: "batch", 1: "encoder_sequence"}),
                 ]   )

             if self.use_past:
                 snake_case      =					{0: "batch"}
                 snake_case      =					{0: "batch", 1: "past_decoder_sequence + sequence"}
             else:
                 snake_case      =					{0: "batch", 1: "decoder_sequence"}
                 snake_case      =					{0: "batch", 1: "decoder_sequence"}

             if self.use_past:
                 self.fill_with_past_key_values_(_A				,    direction="inputs"   )
         elif self.task == "causal-lm":
             # TODO: figure this case out.
             snake_case      =					OrderedDict(
                 [
                     ("input_ids", {0: "batch", 1: "encoder_sequence"}),
                     ("attention_mask", {0: "batch", 1: "encoder_sequence"}),
                 ]   )
             if self.use_past:
                 snake_case     ,							snake_case      =					self.num_layers
                 for i in range(_A   ):
                     snake_case      =					{0: "batch", 2: "past_sequence + sequence"}
                     snake_case      =					{0: "batch", 2: "past_sequence + sequence"}
         else:
             snake_case      =					OrderedDict(
                 [
                     ("input_ids", {0: "batch", 1: "encoder_sequence"}),
                     ("attention_mask", {0: "batch", 1: "encoder_sequence"}),
                     ("decoder_input_ids", {0: "batch", 1: "decoder_sequence"}),
                     ("decoder_attention_mask", {0: "batch", 1: "decoder_sequence"}),
                 ]   )

         return common_inputs





     @property
     def 		UpperCAmelCase(self      :    str   )						->						Mapping[str, Mapping[int, str]]:
         if self.task in ["default", "seq2seq-lm"]:
             snake_case      =					super().outputs
         else:
             snake_case      =					super(_A				,    self   ).outputs
             if self.use_past:
                 snake_case     ,							snake_case      =					self.num_layers
                 for i in range(_A   ):
                     snake_case      =					{0: "batch", 2: "past_sequence + sequence"}
                     snake_case      =					{0: "batch", 2: "past_sequence + sequence"}
         return common_outputs





     def 		UpperCAmelCase(self      :    Any				,    _A      :    PreTrainedTokenizer				,    _A      :    int = -1				,    _A      :    int = -1				,    _A      :    bool = False				,    _A      :    Optional[TensorType] = None				,    )						->						Mapping[str, Any]:
         snake_case      =					self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
             _A				,    _A				,    _A				,    _A				,    _A   )

         # Generate decoder inputs
         snake_case      =					seq_length if not self.use_past else 1
         snake_case      =					self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
             _A				,    _A				,    _A				,    _A				,    _A   )
         snake_case      =					{f'decoder_{name}': tensor for name, tensor in decoder_inputs.items()}
         snake_case      =					dict(**_A				,    **_A   )

         if self.use_past:
             if not is_torch_available():
                 raise ValueError("Cannot generate dummy past_keys inputs without PyTorch installed."   )
             else:
                 import torch
             snake_case     ,							snake_case      =					common_inputs["input_ids"].shape
             snake_case      =					common_inputs["decoder_input_ids"].shape[1]
             snake_case     ,							snake_case      =					self.num_attention_heads
             snake_case      =					(
                 batch,
                 num_encoder_attention_heads,
                 encoder_seq_length,
                 self._config.hidden_size // num_encoder_attention_heads,
             )
             snake_case      =					decoder_seq_length + 3
             snake_case      =					(
                 batch,
                 num_decoder_attention_heads,
                 decoder_past_length,
                 self._config.hidden_size // num_decoder_attention_heads,
             )

             snake_case      =					torch.cat(
                 [common_inputs["decoder_attention_mask"], torch.ones(_A				,    _A   )]				,    dim=1   )

             snake_case      =					[]
             # If the number of encoder and decoder layers are present in the model configuration, both are considered
             snake_case     ,							snake_case      =					self.num_layers
             snake_case      =					min(_A				,    _A   )
             snake_case      =					max(_A				,    _A   ) - min_num_layers
             snake_case      =					"encoder" if num_encoder_layers > num_decoder_layers else "decoder"

             for _ in range(_A   ):
                 common_inputs["past_key_values"].append(
                     (
                         torch.zeros(_A   ),
                         torch.zeros(_A   ),
                         torch.zeros(_A   ),
                         torch.zeros(_A   ),
                     )   )
             # TODO: test this.
             snake_case      =					encoder_shape if remaining_side_name == "encoder" else decoder_shape
             for _ in range(_A				,    _A   ):
                 common_inputs["past_key_values"].append((torch.zeros(_A   ), torch.zeros(_A   ))   )
         return common_inputs





     def 		UpperCAmelCase(self      :    Tuple				,    _A      :    PreTrainedTokenizer				,    _A      :    int = -1				,    _A      :    int = -1				,    _A      :    bool = False				,    _A      :    Optional[TensorType] = None				,    )						->						Mapping[str, Any]:
         snake_case      =					self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
             _A				,    _A				,    _A				,    _A				,    _A   )

         if self.use_past:
             if not is_torch_available():
                 raise ValueError("Cannot generate dummy past_keys inputs without PyTorch installed."   )
             else:
                 import torch
             snake_case     ,							snake_case      =					common_inputs["input_ids"].shape
             # Not using the same length for past_key_values
             snake_case      =					seqlen + 2
             snake_case     ,							snake_case      =					self.num_layers
             snake_case     ,							snake_case      =					self.num_attention_heads
             snake_case      =					(
                 batch,
                 num_encoder_attention_heads,
                 past_key_values_length,
                 self._config.hidden_size // num_encoder_attention_heads,
             )

             snake_case      =					common_inputs["attention_mask"].dtype
             snake_case      =					torch.cat(
                 [common_inputs["attention_mask"], torch.ones(_A				,    _A				,    dtype=_A   )]				,    dim=1   )
             snake_case      =					[
                 (torch.zeros(_A   ), torch.zeros(_A   )) for _ in range(_A   )
             ]
         return common_inputs





     def 		UpperCAmelCase(self      :    Dict				,    _A      :    PreTrainedTokenizer				,    _A      :    int = -1				,    _A      :    int = -1				,    _A      :    bool = False				,    _A      :    Optional[TensorType] = None				,    )						->						Mapping[str, Any]:
         # Copied from OnnxConfig.generate_dummy_inputs
         # Did not use super(OnnxConfigWithPast, self).generate_dummy_inputs for code clarity.
         # If dynamic axis (-1) we forward with a fixed dimension of 2 samples to avoid optimizations made by ONNX
         snake_case      =					compute_effective_axis_dimension(
             _A				,    fixed_dimension=OnnxConfig.default_fixed_batch				,    num_token_to_add=0   )

         # If dynamic axis (-1) we forward with a fixed dimension of 8 tokens to avoid optimizations made by ONNX
         snake_case      =					tokenizer.num_special_tokens_to_add(_A   )
         snake_case      =					compute_effective_axis_dimension(
             _A				,    fixed_dimension=OnnxConfig.default_fixed_sequence				,    num_token_to_add=_A   )

         # Generate dummy inputs according to compute batch and sequence
         snake_case      =					[" ".join([tokenizer.unk_token]   ) * seq_length] * batch_size
         snake_case      =					dict(tokenizer(_A				,    return_tensors=_A   )   )
         return common_inputs





     def 		UpperCAmelCase(self      :    Any				,    _A      :    PreTrainedTokenizer				,    _A      :    int = -1				,    _A      :    int = -1				,    _A      :    bool = False				,    _A      :    Optional[TensorType] = None				,    )						->						Mapping[str, Any]:
         if self.task in ["default", "seq2seq-lm"]:
             snake_case      =					self._generate_dummy_inputs_for_default_and_seqaseq_lm(
                 _A				,    batch_size=_A				,    seq_length=_A				,    is_pair=_A				,    framework=_A   )

         elif self.task == "causal-lm":
             snake_case      =					self._generate_dummy_inputs_for_causal_lm(
                 _A				,    batch_size=_A				,    seq_length=_A				,    is_pair=_A				,    framework=_A   )
         else:
             snake_case      =					self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
                 _A				,    batch_size=_A				,    seq_length=_A				,    is_pair=_A				,    framework=_A   )

         return common_inputs





     def 		UpperCAmelCase(self      :    List[Any]				,    _A      :    str				,    _A      :    Union[str, Any]				,    _A      :    Tuple				,    _A      :    List[Any]   )						->						Any:
         if self.task in ["default", "seq2seq-lm"]:
             snake_case      =					super()._flatten_past_key_values_(_A				,    _A				,    _A				,    _A   )
         else:
             snake_case      =					super(_A				,    self   )._flatten_past_key_values_(
                 _A				,    _A				,    _A				,    _A   )

