




import json
from typing import Dict, List, Optional, Tuple, Union

from tokenizers import pre_tokenizers, processors

from ...tokenization_utils_base import AddedToken, BatchEncoding, EncodedInput
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import PaddingStrategy, logging
from .tokenization_led import LEDTokenizer


__A								=       logging.get_logger(__name__)


__A								=       {"vocab_file": "vocab.json", "merges_file": "merges.txt", "tokenizer_file": "tokenizer.json"}

__A								=       {
    "vocab_file": {
        "allenai/led-base-16384": "https://huggingface.co/allenai/led-base-16384/resolve/main/vocab.json",
    },
    "merges_file": {
        "allenai/led-base-16384": "https://huggingface.co/allenai/led-base-16384/resolve/main/merges.txt",
    },
    "tokenizer_file": {
        "allenai/led-base-16384": "https://huggingface.co/allenai/led-base-16384/resolve/main/tokenizer.json",
    },
}

__A								=       {
    "allenai/led-base-16384": 1_63_84,
}







class        __lowerCAmelCase					(			__magic_name__					):

			"""simple docstring"""







			snake_case_  						=			VOCAB_FILES_NAMES
			snake_case_  						=			PRETRAINED_VOCAB_FILES_MAP
			snake_case_  						=			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			snake_case_  						=			LEDTokenizer
			snake_case_  						=			['''input_ids''', '''attention_mask''']


			def __init__( self     ,		lowerCamelCase__=None     ,		lowerCamelCase__=None     ,		lowerCamelCase__=None     ,		lowerCamelCase__="replace"     ,		lowerCamelCase__="<s>"     ,		lowerCamelCase__="</s>"     ,		lowerCamelCase__="</s>"     ,		lowerCamelCase__="<s>"     ,		lowerCamelCase__="<unk>"     ,		lowerCamelCase__="<pad>"     ,		lowerCamelCase__="<mask>"     ,		lowerCamelCase__=False     ,		lowerCamelCase__=True     ,		**lowerCamelCase__     ,		)       -> int:



							'''simple docstring'''
							super().__init__(
							    lowerCamelCase__     ,		lowerCamelCase__     ,		tokenizer_file=lowerCamelCase__     ,		errors=lowerCamelCase__     ,		bos_token=lowerCamelCase__     ,		eos_token=lowerCamelCase__     ,		sep_token=lowerCamelCase__     ,		cls_token=lowerCamelCase__     ,		unk_token=lowerCamelCase__     ,		pad_token=lowerCamelCase__     ,		mask_token=lowerCamelCase__     ,		add_prefix_space=lowerCamelCase__     ,		trim_offsets=lowerCamelCase__     ,		**lowerCamelCase__     ,		)

							__lowerCamelCase									=       json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()				)
							if pre_tok_state.get('add_prefix_space'     ,		lowerCamelCase__				) != add_prefix_space:
											__lowerCamelCase									=       getattr(lowerCamelCase__     ,		pre_tok_state.pop('type'				)				)
											__lowerCamelCase									=       add_prefix_space
											__lowerCamelCase									=       pre_tok_class(**lowerCamelCase__				)

							__lowerCamelCase									=       add_prefix_space

							# the pre_tokenizer is already updated in the GPT2TokenizerFast `__init__`
							__lowerCamelCase									=       'post_processor'
							__lowerCamelCase									=       getattr(self.backend_tokenizer     ,		lowerCamelCase__     ,		lowerCamelCase__				)
							if tokenizer_component_instance:
											__lowerCamelCase									=       json.loads(tokenizer_component_instance.__getstate__()				)

											# The lists 'sep' and 'cls' must be cased in tuples for the object `post_processor_class`
											if "sep" in state:
															__lowerCamelCase									=       tuple(state['sep']				)
											if "cls" in state:
															__lowerCamelCase									=       tuple(state['cls']				)

											__lowerCamelCase									=       False

											if state.get('add_prefix_space'     ,		lowerCamelCase__				) != add_prefix_space:
															__lowerCamelCase									=       add_prefix_space
															__lowerCamelCase									=       True

											if state.get('trim_offsets'     ,		lowerCamelCase__				) != trim_offsets:
															__lowerCamelCase									=       trim_offsets
															__lowerCamelCase									=       True

											if changes_to_apply:
															__lowerCamelCase									=       getattr(lowerCamelCase__     ,		state.pop('type'				)				)
															__lowerCamelCase									=       component_class(**lowerCamelCase__				)
															setattr(self.backend_tokenizer     ,		lowerCamelCase__     ,		lowerCamelCase__				)


			@property
			# Copied from transformers.models.bart.tokenization_bart_fast.BartTokenizerFast.mask_token with BART->LED
			def  lowercase_       ( self				)       -> str:



							'''simple docstring'''
							if self._mask_token is None:
											if self.verbose:
															logger.error('Using mask_token, but it is not set yet.'				)
											return None
							return str(self._mask_token				)


			@mask_token.setter
			def  lowercase_       ( self     ,		lowerCamelCase__				)       -> Optional[Any]:



							'''simple docstring'''
							__lowerCamelCase									=       AddedToken(lowerCamelCase__     ,		lstrip=lowerCamelCase__     ,		rstrip=lowerCamelCase__				) if isinstance(lowerCamelCase__     ,		lowerCamelCase__				) else value
							__lowerCamelCase									=       value


			def  lowercase_       ( self     ,		*lowerCamelCase__     ,		**lowerCamelCase__				)       -> BatchEncoding:



							'''simple docstring'''
							__lowerCamelCase									=       kwargs.get('is_split_into_words'     ,		lowerCamelCase__				)

							if is_split_into_words and not self.add_prefix_space:
											raise ValueError(
											    f"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
											    'to use it with pretokenized inputs.'				)

							return super()._batch_encode_plus(*lowerCamelCase__     ,		**lowerCamelCase__				)


			def  lowercase_       ( self     ,		*lowerCamelCase__     ,		**lowerCamelCase__				)       -> BatchEncoding:



							'''simple docstring'''
							__lowerCamelCase									=       kwargs.get('is_split_into_words'     ,		lowerCamelCase__				)

							if is_split_into_words and not self.add_prefix_space:
											raise ValueError(
											    f"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
											    'to use it with pretokenized inputs.'				)

							return super()._encode_plus(*lowerCamelCase__     ,		**lowerCamelCase__				)


			def  lowercase_       ( self     ,		lowerCamelCase__     ,		lowerCamelCase__ = None				)       -> Tuple[str]:



							'''simple docstring'''
							__lowerCamelCase									=       self._tokenizer.model.save(lowerCamelCase__     ,		name=lowerCamelCase__				)
							return tuple(lowerCamelCase__				)


			def  lowercase_       ( self     ,		lowerCamelCase__     ,		lowerCamelCase__=None				)       -> Dict:



							'''simple docstring'''
							__lowerCamelCase									=       [self.bos_token_id] + token_ids_a + [self.eos_token_id]
							if token_ids_a is None:
											return output

							return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]


			def  lowercase_       ( self     ,		lowerCamelCase__     ,		lowerCamelCase__ = None				)       -> List[int]:



							'''simple docstring'''
							__lowerCamelCase									=       [self.sep_token_id]
							__lowerCamelCase									=       [self.cls_token_id]

							if token_ids_a is None:
											return len(cls + token_ids_a + sep				) * [0]
							return len(cls + token_ids_a + sep + sep + token_ids_a + sep				) * [0]






			def  lowercase_       ( self     ,		lowerCamelCase__     ,		lowerCamelCase__ = None     ,		lowerCamelCase__ = PaddingStrategy.DO_NOT_PAD     ,		lowerCamelCase__ = None     ,		lowerCamelCase__ = None     ,		)       -> dict:



							'''simple docstring'''
							__lowerCamelCase									=       super()._pad(
							    encoded_inputs=lowerCamelCase__     ,		max_length=lowerCamelCase__     ,		padding_strategy=lowerCamelCase__     ,		pad_to_multiple_of=lowerCamelCase__     ,		return_attention_mask=lowerCamelCase__     ,		)

							# Load from model defaults
							if return_attention_mask is None:
											__lowerCamelCase									=       'attention_mask' in self.model_input_names

							if return_attention_mask and "global_attention_mask" in encoded_inputs:
											__lowerCamelCase									=       encoded_inputs[self.model_input_names[0]]
											# `global_attention_mask` need to have the same length as other (sequential) inputs.
											__lowerCamelCase									=       len(encoded_inputs['global_attention_mask']				) != len(lowerCamelCase__				)

											if needs_to_be_padded:
															__lowerCamelCase									=       len(lowerCamelCase__				) - len(encoded_inputs['global_attention_mask']				)

															if self.padding_side == "right":
																			# Use `-1` since `0` in `global_attention_mask` means `local attention` instead of `not to attend`
																			__lowerCamelCase									=       (
																			    encoded_inputs['global_attention_mask'] + [-1] * difference
																			)
															elif self.padding_side == "left":
																			__lowerCamelCase									=       [-1] * difference + encoded_inputs[
																			    'global_attention_mask'
																			]
															else:
																			raise ValueError('Invalid padding strategy:' + str(self.padding_side				)				)

							return encoded_inputs








from collections import OrderedDict
from typing import Mapping

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


__A								=       logging.get_logger(__name__)

__A								=       {
    "facebook/xlm-roberta-xl": "https://huggingface.co/facebook/xlm-roberta-xl/resolve/main/config.json",
    "facebook/xlm-roberta-xxl": "https://huggingface.co/facebook/xlm-roberta-xxl/resolve/main/config.json",
    # See all XLM-RoBERTa-XL models at https://huggingface.co/models?filter=xlm-roberta-xl
}







class        __lowerCAmelCase					(			__magic_name__					):

			"""simple docstring"""







			snake_case_  						=			'''xlm-roberta-xl'''


			def __init__( self     ,		lowerCamelCase__=250_880     ,		lowerCamelCase__=2_560     ,		lowerCamelCase__=36     ,		lowerCamelCase__=32     ,		lowerCamelCase__=10_240     ,		lowerCamelCase__="gelu"     ,		lowerCamelCase__=0.1     ,		lowerCamelCase__=0.1     ,		lowerCamelCase__=514     ,		lowerCamelCase__=1     ,		lowerCamelCase__=0.02     ,		lowerCamelCase__=1e-05     ,		lowerCamelCase__=1     ,		lowerCamelCase__=0     ,		lowerCamelCase__=2     ,		lowerCamelCase__="absolute"     ,		lowerCamelCase__=True     ,		lowerCamelCase__=None     ,		**lowerCamelCase__     ,		)       -> str:



							'''simple docstring'''
							super().__init__(pad_token_id=lowerCamelCase__     ,		bos_token_id=lowerCamelCase__     ,		eos_token_id=lowerCamelCase__     ,		**lowerCamelCase__				)
							__lowerCamelCase									=       vocab_size
							__lowerCamelCase									=       hidden_size
							__lowerCamelCase									=       num_hidden_layers
							__lowerCamelCase									=       num_attention_heads
							__lowerCamelCase									=       hidden_act
							__lowerCamelCase									=       intermediate_size
							__lowerCamelCase									=       hidden_dropout_prob
							__lowerCamelCase									=       attention_probs_dropout_prob
							__lowerCamelCase									=       max_position_embeddings
							__lowerCamelCase									=       type_vocab_size
							__lowerCamelCase									=       initializer_range
							__lowerCamelCase									=       layer_norm_eps
							__lowerCamelCase									=       position_embedding_type
							__lowerCamelCase									=       use_cache
							__lowerCamelCase									=       classifier_dropout







class        __lowerCAmelCase					(			__magic_name__					):

			"""simple docstring"""







			@property
			def  lowercase_       ( self				)       -> Mapping[str, Mapping[int, str]]:



							'''simple docstring'''
							if self.task == "multiple-choice":
											__lowerCamelCase									=       {0: 'batch', 1: 'choice', 2: 'sequence'}
							else:
											__lowerCamelCase									=       {0: 'batch', 1: 'sequence'}
							return OrderedDict(
							    [
							        ('input_ids', dynamic_axis),
							        ('attention_mask', dynamic_axis),
							    ]				)


