



"""simple docstring"""
from collections import OrderedDict
from typing import Any, Mapping, Optional

from ... import PreTrainedTokenizer
from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig, OnnxConfigWithPast, OnnxSeqaSeqConfigWithPast
from ...onnx.utils import compute_effective_axis_dimension
from ...utils import TensorType, is_torch_available, logging


__SCREAMING_SNAKE_CASE     :  int										=					logging.get_logger(__name__)

__SCREAMING_SNAKE_CASE     :  Tuple										=					{
    '''facebook/mbart-large-cc25''': '''https://huggingface.co/facebook/mbart-large-cc25/resolve/main/config.json''',
    # See all MBART models at https://huggingface.co/models?filter=mbart
}


class 							lowerCamelCase_(      A__			):





					'''simple docstring'''



					lowercase__      :				Any        =       'mbart'
					lowercase__      :				List[str]        =       ['past_key_values']
					lowercase__      :				Union[str, Any]        =       {'num_attention_heads': 'encoder_attention_heads', 'hidden_size': 'd_model'}

					def __init__(			self ,				lowerCamelCase__=5_0_2_6_5 ,				lowerCamelCase__=1_0_2_4 ,				lowerCamelCase__=1_2 ,				lowerCamelCase__=4_0_9_6 ,				lowerCamelCase__=1_6 ,				lowerCamelCase__=1_2 ,				lowerCamelCase__=4_0_9_6 ,				lowerCamelCase__=1_6 ,				lowerCamelCase__=0.0 ,				lowerCamelCase__=0.0 ,				lowerCamelCase__=True ,				lowerCamelCase__=True ,				lowerCamelCase__="gelu" ,				lowerCamelCase__=1_0_2_4 ,				lowerCamelCase__=0.1 ,				lowerCamelCase__=0.0 ,				lowerCamelCase__=0.0 ,				lowerCamelCase__=0.0_2 ,				lowerCamelCase__=0.0 ,				lowerCamelCase__=False ,				lowerCamelCase__=1 ,				lowerCamelCase__=0 ,				lowerCamelCase__=2 ,				lowerCamelCase__=2 ,				**lowerCamelCase__ ,				):
									_lowerCamelCase					     = vocab_size
									_lowerCamelCase					     = max_position_embeddings
									_lowerCamelCase					     = d_model
									_lowerCamelCase					     = encoder_ffn_dim
									_lowerCamelCase					     = encoder_layers
									_lowerCamelCase					     = encoder_attention_heads
									_lowerCamelCase					     = decoder_ffn_dim
									_lowerCamelCase					     = decoder_layers
									_lowerCamelCase					     = decoder_attention_heads
									_lowerCamelCase					     = dropout
									_lowerCamelCase					     = attention_dropout
									_lowerCamelCase					     = activation_dropout
									_lowerCamelCase					     = activation_function
									_lowerCamelCase					     = init_std
									_lowerCamelCase					     = encoder_layerdrop
									_lowerCamelCase					     = decoder_layerdrop
									_lowerCamelCase					     = classifier_dropout
									_lowerCamelCase					     = use_cache
									_lowerCamelCase					     = encoder_layers
									_lowerCamelCase					     = scale_embedding  # scale factor will be sqrt(d_model) if True
									super().__init__(
									    pad_token_id=lowerCamelCase__ ,				bos_token_id=lowerCamelCase__ ,				eos_token_id=lowerCamelCase__ ,				is_encoder_decoder=lowerCamelCase__ ,				forced_eos_token_id=lowerCamelCase__ ,				**lowerCamelCase__ ,				)


class 							lowerCamelCase_(      A__			):





					'''simple docstring'''



					@property
					def 	snake_case__					(			self      ):
									if self.task in ["default", "seq2seq-lm"]:
													_lowerCamelCase					     = OrderedDict(
													    [
													        ('''input_ids''', {0: '''batch''', 1: '''encoder_sequence'''}),
													        ('''attention_mask''', {0: '''batch''', 1: '''encoder_sequence'''}),
													    ]      )

													if self.use_past:
																	_lowerCamelCase					     = {0: '''batch'''}
																	_lowerCamelCase					     = {0: '''batch''', 1: '''past_decoder_sequence + sequence'''}
													else:
																	_lowerCamelCase					     = {0: '''batch''', 1: '''decoder_sequence'''}
																	_lowerCamelCase					     = {0: '''batch''', 1: '''decoder_sequence'''}

													if self.use_past:
																	self.fill_with_past_key_values_(lowerCamelCase__ ,				direction='''inputs'''      )
									elif self.task == "causal-lm":
													# TODO: figure this case out.
													_lowerCamelCase					     = OrderedDict(
													    [
													        ('''input_ids''', {0: '''batch''', 1: '''encoder_sequence'''}),
													        ('''attention_mask''', {0: '''batch''', 1: '''encoder_sequence'''}),
													    ]      )
													if self.use_past:
																	_lowerCamelCase     ,   _lowerCamelCase					     = self.num_layers
																	for i in range(lowerCamelCase__      ):
																					_lowerCamelCase					     = {0: '''batch''', 2: '''past_sequence + sequence'''}
																					_lowerCamelCase					     = {0: '''batch''', 2: '''past_sequence + sequence'''}
									else:
													_lowerCamelCase					     = OrderedDict(
													    [
													        ('''input_ids''', {0: '''batch''', 1: '''encoder_sequence'''}),
													        ('''attention_mask''', {0: '''batch''', 1: '''encoder_sequence'''}),
													        ('''decoder_input_ids''', {0: '''batch''', 1: '''decoder_sequence'''}),
													        ('''decoder_attention_mask''', {0: '''batch''', 1: '''decoder_sequence'''}),
													    ]      )

									return common_inputs

					@property
					def 	snake_case__					(			self      ):
									if self.task in ["default", "seq2seq-lm"]:
													_lowerCamelCase					     = super().outputs
									else:
													_lowerCamelCase					     = super(lowerCamelCase__ ,				self      ).outputs
													if self.use_past:
																	_lowerCamelCase     ,   _lowerCamelCase					     = self.num_layers
																	for i in range(lowerCamelCase__      ):
																					_lowerCamelCase					     = {0: '''batch''', 2: '''past_sequence + sequence'''}
																					_lowerCamelCase					     = {0: '''batch''', 2: '''past_sequence + sequence'''}
									return common_outputs

					def 	snake_case__					(			self ,				lowerCamelCase__ ,				lowerCamelCase__ = -1 ,				lowerCamelCase__ = -1 ,				lowerCamelCase__ = False ,				lowerCamelCase__ = None ,				):
									_lowerCamelCase					     = self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
									    lowerCamelCase__ ,				lowerCamelCase__ ,				lowerCamelCase__ ,				lowerCamelCase__ ,				lowerCamelCase__      )

									# Generate decoder inputs
									_lowerCamelCase					     = seq_length if not self.use_past else 1
									_lowerCamelCase					     = self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
									    lowerCamelCase__ ,				lowerCamelCase__ ,				lowerCamelCase__ ,				lowerCamelCase__ ,				lowerCamelCase__      )
									_lowerCamelCase					     = {F"""decoder_{name}""": tensor for name, tensor in decoder_inputs.items()}
									_lowerCamelCase					     = dict(**lowerCamelCase__ ,				**lowerCamelCase__      )

									if self.use_past:
													if not is_torch_available():
																	raise ValueError('''Cannot generate dummy past_keys inputs without PyTorch installed.'''      )
													else:
																	import torch
													_lowerCamelCase     ,   _lowerCamelCase					     = common_inputs['''input_ids'''].shape
													_lowerCamelCase					     = common_inputs['''decoder_input_ids'''].shape[1]
													_lowerCamelCase     ,   _lowerCamelCase					     = self.num_attention_heads
													_lowerCamelCase					     = (
													    batch,
													    num_encoder_attention_heads,
													    encoder_seq_length,
													    self._config.hidden_size // num_encoder_attention_heads,
													)
													_lowerCamelCase					     = decoder_seq_length + 3
													_lowerCamelCase					     = (
													    batch,
													    num_decoder_attention_heads,
													    decoder_past_length,
													    self._config.hidden_size // num_decoder_attention_heads,
													)

													_lowerCamelCase					     = torch.cat(
													    [common_inputs['''decoder_attention_mask'''], torch.ones(lowerCamelCase__ ,				lowerCamelCase__      )] ,				dim=1      )

													_lowerCamelCase					     = []
													# If the number of encoder and decoder layers are present in the model configuration, both are considered
													_lowerCamelCase     ,   _lowerCamelCase					     = self.num_layers
													_lowerCamelCase					     = min(lowerCamelCase__ ,				lowerCamelCase__      )
													_lowerCamelCase					     = max(lowerCamelCase__ ,				lowerCamelCase__      ) - min_num_layers
													_lowerCamelCase					     = '''encoder''' if num_encoder_layers > num_decoder_layers else '''decoder'''

													for _ in range(lowerCamelCase__      ):
																	common_inputs["past_key_values"].append(
																	    (
																	        torch.zeros(lowerCamelCase__      ),
																	        torch.zeros(lowerCamelCase__      ),
																	        torch.zeros(lowerCamelCase__      ),
																	        torch.zeros(lowerCamelCase__      ),
																	    )      )
													# TODO: test this.
													_lowerCamelCase					     = encoder_shape if remaining_side_name == '''encoder''' else decoder_shape
													for _ in range(lowerCamelCase__ ,				lowerCamelCase__      ):
																	common_inputs["past_key_values"].append((torch.zeros(lowerCamelCase__      ), torch.zeros(lowerCamelCase__      ))      )
									return common_inputs

					def 	snake_case__					(			self ,				lowerCamelCase__ ,				lowerCamelCase__ = -1 ,				lowerCamelCase__ = -1 ,				lowerCamelCase__ = False ,				lowerCamelCase__ = None ,				):
									_lowerCamelCase					     = self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
									    lowerCamelCase__ ,				lowerCamelCase__ ,				lowerCamelCase__ ,				lowerCamelCase__ ,				lowerCamelCase__      )

									if self.use_past:
													if not is_torch_available():
																	raise ValueError('''Cannot generate dummy past_keys inputs without PyTorch installed.'''      )
													else:
																	import torch
													_lowerCamelCase     ,   _lowerCamelCase					     = common_inputs['''input_ids'''].shape
													# Not using the same length for past_key_values
													_lowerCamelCase					     = seqlen + 2
													_lowerCamelCase     ,   _lowerCamelCase					     = self.num_layers
													_lowerCamelCase     ,   _lowerCamelCase					     = self.num_attention_heads
													_lowerCamelCase					     = (
													    batch,
													    num_encoder_attention_heads,
													    past_key_values_length,
													    self._config.hidden_size // num_encoder_attention_heads,
													)

													_lowerCamelCase					     = common_inputs['''attention_mask'''].dtype
													_lowerCamelCase					     = torch.cat(
													    [common_inputs['''attention_mask'''], torch.ones(lowerCamelCase__ ,				lowerCamelCase__ ,				dtype=lowerCamelCase__      )] ,				dim=1      )
													_lowerCamelCase					     = [
													    (torch.zeros(lowerCamelCase__      ), torch.zeros(lowerCamelCase__      )) for _ in range(lowerCamelCase__      )
													]
									return common_inputs

					def 	snake_case__					(			self ,				lowerCamelCase__ ,				lowerCamelCase__ = -1 ,				lowerCamelCase__ = -1 ,				lowerCamelCase__ = False ,				lowerCamelCase__ = None ,				):
									# Copied from OnnxConfig.generate_dummy_inputs
									# Did not use super(OnnxConfigWithPast, self).generate_dummy_inputs for code clarity.
									# If dynamic axis (-1) we forward with a fixed dimension of 2 samples to avoid optimizations made by ONNX
									_lowerCamelCase					     = compute_effective_axis_dimension(
									    lowerCamelCase__ ,				fixed_dimension=OnnxConfig.default_fixed_batch ,				num_token_to_add=0      )

									# If dynamic axis (-1) we forward with a fixed dimension of 8 tokens to avoid optimizations made by ONNX
									_lowerCamelCase					     = tokenizer.num_special_tokens_to_add(lowerCamelCase__      )
									_lowerCamelCase					     = compute_effective_axis_dimension(
									    lowerCamelCase__ ,				fixed_dimension=OnnxConfig.default_fixed_sequence ,				num_token_to_add=lowerCamelCase__      )

									# Generate dummy inputs according to compute batch and sequence
									_lowerCamelCase					     = [''' '''.join([tokenizer.unk_token]      ) * seq_length] * batch_size
									_lowerCamelCase					     = dict(tokenizer(lowerCamelCase__ ,				return_tensors=lowerCamelCase__      )      )
									return common_inputs

					def 	snake_case__					(			self ,				lowerCamelCase__ ,				lowerCamelCase__ = -1 ,				lowerCamelCase__ = -1 ,				lowerCamelCase__ = False ,				lowerCamelCase__ = None ,				):
									if self.task in ["default", "seq2seq-lm"]:
													_lowerCamelCase					     = self._generate_dummy_inputs_for_default_and_seqaseq_lm(
													    lowerCamelCase__ ,				batch_size=lowerCamelCase__ ,				seq_length=lowerCamelCase__ ,				is_pair=lowerCamelCase__ ,				framework=lowerCamelCase__      )

									elif self.task == "causal-lm":
													_lowerCamelCase					     = self._generate_dummy_inputs_for_causal_lm(
													    lowerCamelCase__ ,				batch_size=lowerCamelCase__ ,				seq_length=lowerCamelCase__ ,				is_pair=lowerCamelCase__ ,				framework=lowerCamelCase__      )
									else:
													_lowerCamelCase					     = self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
													    lowerCamelCase__ ,				batch_size=lowerCamelCase__ ,				seq_length=lowerCamelCase__ ,				is_pair=lowerCamelCase__ ,				framework=lowerCamelCase__      )

									return common_inputs





					def 	snake_case__					(			self ,				lowerCamelCase__ ,				lowerCamelCase__ ,				lowerCamelCase__ ,				lowerCamelCase__      ):
									if self.task in ["default", "seq2seq-lm"]:
													_lowerCamelCase					     = super()._flatten_past_key_values_(lowerCamelCase__ ,				lowerCamelCase__ ,				lowerCamelCase__ ,				lowerCamelCase__      )
									else:
													_lowerCamelCase					     = super(lowerCamelCase__ ,				self      )._flatten_past_key_values_(
													    lowerCamelCase__ ,				lowerCamelCase__ ,				lowerCamelCase__ ,				lowerCamelCase__      )








"""simple docstring"""







import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
  from .tokenization_xglm import XGLMTokenizer
else:
  _UpperCamelCase     			=  None


_UpperCamelCase     			=  logging.get_logger(__name__)

_UpperCamelCase     			=  {"""vocab_file""": """sentencepiece.bpe.model""", """tokenizer_file""": """tokenizer.json"""}

_UpperCamelCase     			=  {
    """vocab_file""": {
        """facebook/xglm-564M""": """https://huggingface.co/facebook/xglm-564M/resolve/main/sentencepiece.bpe.model""",
    },
    """tokenizer_file""": {
        """facebook/xglm-564M""": """https://huggingface.co/facebook/xglm-564M/resolve/main/tokenizer.json""",
    },
}

_UpperCamelCase     			=  {
    """facebook/xglm-564M""": 2048,
}







class 					lowerCamelCase__					(       snake_case ):
    SCREAMING_SNAKE_CASE		=			VOCAB_FILES_NAMES
    SCREAMING_SNAKE_CASE		=			PRETRAINED_VOCAB_FILES_MAP
    SCREAMING_SNAKE_CASE		=			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    SCREAMING_SNAKE_CASE		=			['''input_ids''', '''attention_mask''']
    SCREAMING_SNAKE_CASE		=			XGLMTokenizer






    def __init__( self	,A=None	,A=None	,A="<s>"	,A="</s>"	,A="</s>"	,A="<s>"	,A="<unk>"	,A="<pad>"	,**A	,):
          # Compatibility with the original tokenizer
          UpperCAmelCase							=       7
          UpperCAmelCase							=       [F'''<madeupword{i}>''' for i in range(self.num_madeup_words					)]

          UpperCAmelCase							=       kwargs.get("""additional_special_tokens"""	,[]					)
          kwargs["additional_special_tokens"] += [
              word for word in madeup_words if word not in kwargs["additional_special_tokens"]
          ]

          super().__init__(
              A	,tokenizer_file=A	,bos_token=A	,eos_token=A	,sep_token=A	,cls_token=A	,unk_token=A	,pad_token=A	,**A	,)

          UpperCAmelCase							=       vocab_file
          UpperCAmelCase							=       False if not self.vocab_file else True






    def 				_UpperCamelCase							( self	,A	,A = None					):

          if token_ids_a is None:
                return [self.sep_token_id] + token_ids_a
          UpperCAmelCase							=       [self.sep_token_id]
          return sep + token_ids_a + sep + sep + token_ids_a






    def 				_UpperCamelCase							( self	,A	,A = None					):
          UpperCAmelCase							=       [self.sep_token_id]

          if token_ids_a is None:
                return len(sep + token_ids_a					) * [0]
          return len(sep + token_ids_a + sep + sep + token_ids_a					) * [0]






    def 				_UpperCamelCase							( self	,A	,A = None					):
          if not self.can_save_slow_tokenizer:
                raise ValueError(
                    """Your fast tokenizer does not have the necessary information to save the vocabulary for a slow """
                    """tokenizer."""					)

          if not os.path.isdir(A					):
                logger.error(F'''Vocabulary path ({save_directory}) should be a directory.'''					)
                return
          UpperCAmelCase							=       os.path.join(
              A	,(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]					)

          if os.path.abspath(self.vocab_file					) != os.path.abspath(A					):
                copyfile(self.vocab_file	,A					)

          return (out_vocab_file,)



