


'''simple docstring'''
import json
from typing import Dict, List, Optional, Tuple, Union

from tokenizers import pre_tokenizers, processors

from ...tokenization_utils_base import AddedToken, BatchEncoding, EncodedInput
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import PaddingStrategy, logging
from .tokenization_led import LEDTokenizer


__snake_case							   =  logging.get_logger(__name__)


__snake_case							   =  {'''vocab_file''': '''vocab.json''', '''merges_file''': '''merges.txt''', '''tokenizer_file''': '''tokenizer.json'''}

__snake_case							   =  {
    '''vocab_file''': {
        '''allenai/led-base-16384''': '''https://huggingface.co/allenai/led-base-16384/resolve/main/vocab.json''',
    },
    '''merges_file''': {
        '''allenai/led-base-16384''': '''https://huggingface.co/allenai/led-base-16384/resolve/main/merges.txt''',
    },
    '''tokenizer_file''': {
        '''allenai/led-base-16384''': '''https://huggingface.co/allenai/led-base-16384/resolve/main/tokenizer.json''',
    },
}

__snake_case							   =  {
    '''allenai/led-base-16384''': 16384,
}
class      lowercase			(							lowercase__      ):

		"""simple docstring"""

		_a									=       VOCAB_FILES_NAMES
		_a									=       PRETRAINED_VOCAB_FILES_MAP
		_a									=       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
		_a									=       LEDTokenizer
		_a									=       ["""input_ids""", """attention_mask"""]




		def __init__(	self					,     UpperCamelCase_=None					,     UpperCamelCase_=None					,     UpperCamelCase_=None					,     UpperCamelCase_="replace"					,     UpperCamelCase_="<s>"					,     UpperCamelCase_="</s>"					,     UpperCamelCase_="</s>"					,     UpperCamelCase_="<s>"					,     UpperCamelCase_="<unk>"					,     UpperCamelCase_="<pad>"					,     UpperCamelCase_="<mask>"					,     UpperCamelCase_=False					,     UpperCamelCase_=True					,     **UpperCamelCase_					,     ):

							'''simple docstring'''


							super().__init__(
							    __lowercase					,     __lowercase					,     tokenizer_file=__lowercase					,     errors=__lowercase					,     bos_token=__lowercase					,     eos_token=__lowercase					,     sep_token=__lowercase					,     cls_token=__lowercase					,     unk_token=__lowercase					,     pad_token=__lowercase					,     mask_token=__lowercase					,     add_prefix_space=__lowercase					,     trim_offsets=__lowercase					,     **__lowercase					,     )

							UpperCamelCase__     :List[str]							      =				json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()							)
							if pre_tok_state.get('''add_prefix_space'''					,     __lowercase							) != add_prefix_space:
												UpperCamelCase__     :Optional[int]							      =				getattr(__lowercase					,     pre_tok_state.pop('''type'''							)							)
												UpperCamelCase__     :Optional[int]							      =				add_prefix_space
												UpperCamelCase__     :Tuple							      =				pre_tok_class(**__lowercase							)

							UpperCamelCase__     :Union[str, Any]							      =				add_prefix_space

							# the pre_tokenizer is already updated in the GPT2TokenizerFast `__init__`
							UpperCamelCase__     :Optional[int]							      =				'''post_processor'''
							UpperCamelCase__     :List[Any]							      =				getattr(self.backend_tokenizer					,     __lowercase					,     __lowercase							)
							if tokenizer_component_instance:
												UpperCamelCase__     :Union[str, Any]							      =				json.loads(tokenizer_component_instance.__getstate__()							)

												# The lists 'sep' and 'cls' must be cased in tuples for the object `post_processor_class`
												if "sep" in state:
																	UpperCamelCase__     :Dict							      =				tuple(state['''sep''']							)
												if "cls" in state:
																	UpperCamelCase__     :Union[str, Any]							      =				tuple(state['''cls''']							)

												UpperCamelCase__     :Tuple							      =				False

												if state.get('''add_prefix_space'''					,     __lowercase							) != add_prefix_space:
																	UpperCamelCase__     :Union[str, Any]							      =				add_prefix_space
																	UpperCamelCase__     :List[str]							      =				True

												if state.get('''trim_offsets'''					,     __lowercase							) != trim_offsets:
																	UpperCamelCase__     :Any							      =				trim_offsets
																	UpperCamelCase__     :Optional[Any]							      =				True

												if changes_to_apply:
																	UpperCamelCase__     :Any							      =				getattr(__lowercase					,     state.pop('''type'''							)							)
																	UpperCamelCase__     :List[str]							      =				component_class(**__lowercase							)
																	setattr(self.backend_tokenizer					,     __lowercase					,     __lowercase							)




		@property
		# Copied from transformers.models.bart.tokenization_bart_fast.BartTokenizerFast.mask_token with BART->LED
		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							if self._mask_token is None:
												if self.verbose:
																	logger.error('''Using mask_token, but it is not set yet.'''							)
												return None
							return str(self._mask_token							)




		@mask_token.setter
		def 					lowerCAmelCase__       (	self					,     UpperCamelCase_							):

							'''simple docstring'''


							UpperCamelCase__     :Any							      =				AddedToken(__lowercase					,     lstrip=__lowercase					,     rstrip=__lowercase							) if isinstance(__lowercase					,     __lowercase							) else value
							UpperCamelCase__     :Optional[Any]							      =				value




		def 					lowerCAmelCase__       (	self					,     *UpperCamelCase_					,     **UpperCamelCase_							):

							'''simple docstring'''


							UpperCamelCase__     :Optional[Any]							      =				kwargs.get('''is_split_into_words'''					,     __lowercase							)

							if is_split_into_words and not self.add_prefix_space:
												raise ValueError(
												    F'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True '''
												    '''to use it with pretokenized inputs.'''							)

							return super()._batch_encode_plus(*__lowercase					,     **__lowercase							)




		def 					lowerCAmelCase__       (	self					,     *UpperCamelCase_					,     **UpperCamelCase_							):

							'''simple docstring'''


							UpperCamelCase__     :Tuple							      =				kwargs.get('''is_split_into_words'''					,     __lowercase							)

							if is_split_into_words and not self.add_prefix_space:
												raise ValueError(
												    F'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True '''
												    '''to use it with pretokenized inputs.'''							)

							return super()._encode_plus(*__lowercase					,     **__lowercase							)




		def 					lowerCAmelCase__       (	self					,     UpperCamelCase_					,     UpperCamelCase_ = None							):

							'''simple docstring'''


							UpperCamelCase__     :Optional[Any]							      =				self._tokenizer.model.save(__lowercase					,     name=__lowercase							)
							return tuple(__lowercase							)




		def 					lowerCAmelCase__       (	self					,     UpperCamelCase_					,     UpperCamelCase_=None							):

							'''simple docstring'''


							UpperCamelCase__     :Dict							      =				[self.bos_token_id] + token_ids_a + [self.eos_token_id]
							if token_ids_a is None:
												return output

							return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]




		def 					lowerCAmelCase__       (	self					,     UpperCamelCase_					,     UpperCamelCase_ = None							):

							'''simple docstring'''


							UpperCamelCase__     :Any							      =				[self.sep_token_id]
							UpperCamelCase__     :Any							      =				[self.cls_token_id]

							if token_ids_a is None:
												return len(cls + token_ids_a + sep							) * [0]
							return len(cls + token_ids_a + sep + sep + token_ids_a + sep							) * [0]




		def 					lowerCAmelCase__       (	self					,     UpperCamelCase_					,     UpperCamelCase_ = None					,     UpperCamelCase_ = PaddingStrategy.DO_NOT_PAD					,     UpperCamelCase_ = None					,     UpperCamelCase_ = None					,     ):

							'''simple docstring'''


							UpperCamelCase__     :Optional[Any]							      =				super()._pad(
							    encoded_inputs=__lowercase					,     max_length=__lowercase					,     padding_strategy=__lowercase					,     pad_to_multiple_of=__lowercase					,     return_attention_mask=__lowercase					,     )

							# Load from model defaults
							if return_attention_mask is None:
												UpperCamelCase__     :Dict							      =				'''attention_mask''' in self.model_input_names

							if return_attention_mask and "global_attention_mask" in encoded_inputs:
												UpperCamelCase__     :List[str]							      =				encoded_inputs[self.model_input_names[0]]
												# `global_attention_mask` need to have the same length as other (sequential) inputs.
												UpperCamelCase__     :List[str]							      =				len(encoded_inputs['''global_attention_mask''']							) != len(__lowercase							)

												if needs_to_be_padded:
																	UpperCamelCase__     :Optional[Any]							      =				len(__lowercase							) - len(encoded_inputs['''global_attention_mask''']							)

																	if self.padding_side == "right":
																						# Use `-1` since `0` in `global_attention_mask` means `local attention` instead of `not to attend`
																						UpperCamelCase__     :List[str]							      =				(
																						    encoded_inputs['''global_attention_mask'''] + [-1] * difference
																						)
																	elif self.padding_side == "left":
																						UpperCamelCase__     :Optional[Any]							      =				[-1] * difference + encoded_inputs[
																						    '''global_attention_mask'''
																						]
																	else:
																						raise ValueError('''Invalid padding strategy:''' + str(self.padding_side							)							)

							return encoded_inputs



'''simple docstring'''
from ...configuration_utils import PretrainedConfig
from ...utils import logging


__snake_case							   =  logging.get_logger(__name__)

__snake_case							   =  {
    '''transfo-xl-wt103''': '''https://huggingface.co/transfo-xl-wt103/resolve/main/config.json''',
}
class      lowercase			(							A__      ):

		"""simple docstring"""

		_a									=       'transfo-xl'
		_a									=       ['mems']
		_a									=       {
		    'n_token': 'vocab_size',
		    'hidden_size': 'd_model',
		    'num_attention_heads': 'n_head',
		    'num_hidden_layers': 'n_layer',
		}




		def __init__(	self					,     UpperCamelCase_=267735					,     UpperCamelCase_=[20000, 40000, 200000]					,     UpperCamelCase_=1024					,     UpperCamelCase_=1024					,     UpperCamelCase_=16					,     UpperCamelCase_=64					,     UpperCamelCase_=4096					,     UpperCamelCase_=4					,     UpperCamelCase_=False					,     UpperCamelCase_=18					,     UpperCamelCase_=1600					,     UpperCamelCase_=1000					,     UpperCamelCase_=True					,     UpperCamelCase_=True					,     UpperCamelCase_=0					,     UpperCamelCase_=-1					,     UpperCamelCase_=True					,     UpperCamelCase_=0.1					,     UpperCamelCase_=0.0					,     UpperCamelCase_=True					,     UpperCamelCase_="normal"					,     UpperCamelCase_=0.01					,     UpperCamelCase_=0.01					,     UpperCamelCase_=0.02					,     UpperCamelCase_=1e-5					,     UpperCamelCase_=0					,     **UpperCamelCase_					,     ):

							'''simple docstring'''


							UpperCamelCase__     :Dict							      =				vocab_size
							UpperCamelCase__     :Optional[int]							      =				[]
							self.cutoffs.extend(UpperCamelCase_							)
							if proj_share_all_but_first:
												UpperCamelCase__     :Any							      =				[False] + [True] * len(self.cutoffs							)
							else:
												UpperCamelCase__     :Dict							      =				[False] + [False] * len(self.cutoffs							)
							UpperCamelCase__     :Union[str, Any]							      =				d_model
							UpperCamelCase__     :List[Any]							      =				d_embed
							UpperCamelCase__     :Any							      =				d_head
							UpperCamelCase__     :Tuple							      =				d_inner
							UpperCamelCase__     :int							      =				div_val
							UpperCamelCase__     :int							      =				pre_lnorm
							UpperCamelCase__     :Any							      =				n_layer
							UpperCamelCase__     :int							      =				n_head
							UpperCamelCase__     :Tuple							      =				mem_len
							UpperCamelCase__     :List[str]							      =				same_length
							UpperCamelCase__     :Any							      =				attn_type
							UpperCamelCase__     :List[str]							      =				clamp_len
							UpperCamelCase__     :Union[str, Any]							      =				sample_softmax
							UpperCamelCase__     :List[Any]							      =				adaptive
							UpperCamelCase__     :List[Any]							      =				dropout
							UpperCamelCase__     :Optional[Any]							      =				dropatt
							UpperCamelCase__     :str							      =				untie_r
							UpperCamelCase__     :List[Any]							      =				init
							UpperCamelCase__     :Optional[Any]							      =				init_range
							UpperCamelCase__     :int							      =				proj_init_std
							UpperCamelCase__     :List[str]							      =				init_std
							UpperCamelCase__     :Optional[int]							      =				layer_norm_epsilon
							super().__init__(eos_token_id=UpperCamelCase_					,     **UpperCamelCase_							)




		@property
		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							logger.info(F'''The model {self.model_type} is one of the few models that has no sequence length limit.'''							)
							return -1




		@max_position_embeddings.setter
		def 					lowerCAmelCase__       (	self					,     UpperCamelCase_							):

							'''simple docstring'''


							raise NotImplementedError(
							    F'''The model {self.model_type} is one of the few models that has no sequence length limit.'''							)