






from collections import OrderedDict
from typing import Mapping

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


__A          = logging.get_logger(__name__)

__A          = {
    'mnaylor/mega-base-wikitext': 'https://huggingface.co/mnaylor/mega-base-wikitext/resolve/main/config.json',
}





class   SCREAMING_SNAKE_CASE (       snake_case    ):





							"""simple docstring"""

							A_       =    "mega"

							def __init__(     self:      Union[str, Any]						,   __A:      int=3_05_22						,   __A:      Tuple=1_28						,   __A:      Union[str, Any]=4						,   __A:      Union[str, Any]=2_56						,   __A:      int=16						,   __A:      Optional[Any]=True						,   __A:      Any=64						,   __A:      Dict=False						,   __A:      List[Any]=-1						,   __A:      Any=None						,   __A:      Dict=True						,   __A:      Tuple="scalenorm"						,   __A:      Tuple=True						,   __A:      int="silu"						,   __A:      Optional[int]="softmax"						,   __A:      Dict=0.1						,   __A:      Optional[int]=0.1						,   __A:      List[Any]=0.1						,   __A:      str=False						,   __A:      Optional[Any]=True						,   __A:      Optional[Any]=2_56						,   __A:      Optional[Any]=True						,   __A:      Optional[Any]=0.1						,   __A:      Optional[int]=20_48						,   __A:      Tuple=False						,   __A:      Any=2						,   __A:      Dict=0.02						,   __A:      Union[str, Any]=0.2						,   __A:      Tuple=0.02						,   __A:      Any=1.0						,   __A:      str=1						,   __A:      List[str]=0						,   __A:      Any=2						,   __A:      Tuple="rotary"						,   __A:      Optional[int]=None						,   __A:      Any=True						,   __A:      Optional[int]=True						,   **__A:      Union[str, Any]						,   )			-> int:
												super().__init__(pad_token_id=__A						,   bos_token_id=__A						,   eos_token_id=__A						,   **__A      )

												_A									=       vocab_size
												_A									=       hidden_size
												_A									=       num_hidden_layers
												_A									=       activation
												_A									=       attention_activation
												_A									=       intermediate_size
												_A									=       ema_projection_size
												_A									=       bidirectional
												_A									=       shared_representation_size
												_A									=       use_chunking
												_A									=       chunk_size
												_A									=       truncation
												_A									=       normalize_before_mega
												_A									=       normalization_type
												_A									=       norm_affine
												_A									=       dropout_prob
												_A									=       hidden_dropout_prob
												_A									=       attention_probs_dropout_prob
												_A									=       use_feature_dropout
												_A									=       use_normalized_ffn
												_A									=       nffn_hidden_size
												_A									=       normalize_before_ffn
												_A									=       nffn_activation_dropout_prob
												_A									=       max_positions
												_A									=       add_token_type_embeddings
												_A									=       type_vocab_size
												_A									=       initializer_range
												_A									=       ema_delta_alpha_range
												_A									=       ema_beta_range
												_A									=       ema_gamma_omega_range
												_A									=       relative_positional_bias
												_A									=       use_cache
												_A									=       classifier_dropout
												_A									=       add_lm_hidden_dense_layer
												_A									=       1  # not used but required by Hugging Face





class   SCREAMING_SNAKE_CASE (       snake_case    ):





							"""simple docstring"""

							@property
							def 					__A       (     self:      int      )			-> Mapping[str, Mapping[int, str]]:
												if self.task == "multiple-choice":
																	_A									=       {0: '''batch''', 1: '''choice''', 2: '''sequence'''}
												else:
																	_A									=       {0: '''batch''', 1: '''sequence'''}
												return OrderedDict(
												    [
												        ('''input_ids''', dynamic_axis),
												        ('''attention_mask''', dynamic_axis),
												    ]      )











import json
from typing import List, Optional, Tuple

from tokenizers import pre_tokenizers, processors

from ...tokenization_utils_base import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_longformer import LongformerTokenizer


__A          = logging.get_logger(__name__)

__A          = {'vocab_file': 'vocab.json', 'merges_file': 'merges.txt', 'tokenizer_file': 'tokenizer.json'}

__A          = {
    'vocab_file': {
        'allenai/longformer-base-4096': 'https://huggingface.co/allenai/longformer-base-4096/resolve/main/vocab.json',
        'allenai/longformer-large-4096': (
            'https://huggingface.co/allenai/longformer-large-4096/resolve/main/vocab.json'
        ),
        'allenai/longformer-large-4096-finetuned-triviaqa': (
            'https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/vocab.json'
        ),
        'allenai/longformer-base-4096-extra.pos.embd.only': (
            'https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/vocab.json'
        ),
        'allenai/longformer-large-4096-extra.pos.embd.only': (
            'https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/vocab.json'
        ),
    },
    'merges_file': {
        'allenai/longformer-base-4096': 'https://huggingface.co/allenai/longformer-base-4096/resolve/main/merges.txt',
        'allenai/longformer-large-4096': (
            'https://huggingface.co/allenai/longformer-large-4096/resolve/main/merges.txt'
        ),
        'allenai/longformer-large-4096-finetuned-triviaqa': (
            'https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/merges.txt'
        ),
        'allenai/longformer-base-4096-extra.pos.embd.only': (
            'https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/merges.txt'
        ),
        'allenai/longformer-large-4096-extra.pos.embd.only': (
            'https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/merges.txt'
        ),
    },
    'tokenizer_file': {
        'allenai/longformer-base-4096': (
            'https://huggingface.co/allenai/longformer-base-4096/resolve/main/tokenizer.json'
        ),
        'allenai/longformer-large-4096': (
            'https://huggingface.co/allenai/longformer-large-4096/resolve/main/tokenizer.json'
        ),
        'allenai/longformer-large-4096-finetuned-triviaqa': (
            'https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/tokenizer.json'
        ),
        'allenai/longformer-base-4096-extra.pos.embd.only': (
            'https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/tokenizer.json'
        ),
        'allenai/longformer-large-4096-extra.pos.embd.only': (
            'https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/tokenizer.json'
        ),
    },
}

__A          = {
    'allenai/longformer-base-4096': 4096,
    'allenai/longformer-large-4096': 4096,
    'allenai/longformer-large-4096-finetuned-triviaqa': 4096,
    'allenai/longformer-base-4096-extra.pos.embd.only': 4096,
    'allenai/longformer-large-4096-extra.pos.embd.only': 4096,
}





class   SCREAMING_SNAKE_CASE (       snake_case    ):





							"""simple docstring"""

							A_       =    VOCAB_FILES_NAMES
							A_       =    PRETRAINED_VOCAB_FILES_MAP
							A_       =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							A_       =    ["input_ids", "attention_mask"]
							A_       =    LongformerTokenizer

							def __init__(     self:      List[str]						,   __A:      Dict=None						,   __A:      int=None						,   __A:      Tuple=None						,   __A:      Any="replace"						,   __A:      List[str]="<s>"						,   __A:      int="</s>"						,   __A:      Dict="</s>"						,   __A:      Tuple="<s>"						,   __A:      Optional[int]="<unk>"						,   __A:      Optional[int]="<pad>"						,   __A:      List[Any]="<mask>"						,   __A:      Tuple=False						,   __A:      Dict=True						,   **__A:      List[Any]						,   )			-> List[Any]:
												super().__init__(
												    __A						,   __A						,   tokenizer_file=__A						,   errors=__A						,   bos_token=__A						,   eos_token=__A						,   sep_token=__A						,   cls_token=__A						,   unk_token=__A						,   pad_token=__A						,   mask_token=__A						,   add_prefix_space=__A						,   trim_offsets=__A						,   **__A						,   )

												_A									=       json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()      )
												if pre_tok_state.get('''add_prefix_space'''						,   __A      ) != add_prefix_space:
																	_A									=       getattr(__A						,   pre_tok_state.pop('''type'''      )      )
																	_A									=       add_prefix_space
																	_A									=       pre_tok_class(**__A      )

												_A									=       add_prefix_space

												_A									=       '''post_processor'''
												_A									=       getattr(self.backend_tokenizer						,   __A						,   __A      )
												if tokenizer_component_instance:
																	_A									=       json.loads(tokenizer_component_instance.__getstate__()      )

																	# The lists 'sep' and 'cls' must be cased in tuples for the object `post_processor_class`
																	if "sep" in state:
																						_A									=       tuple(state['''sep''']      )
																	if "cls" in state:
																						_A									=       tuple(state['''cls''']      )

																	_A									=       False

																	if state.get('''add_prefix_space'''						,   __A      ) != add_prefix_space:
																						_A									=       add_prefix_space
																						_A									=       True

																	if state.get('''trim_offsets'''						,   __A      ) != trim_offsets:
																						_A									=       trim_offsets
																						_A									=       True

																	if changes_to_apply:
																						_A									=       getattr(__A						,   state.pop('''type'''      )      )
																						_A									=       component_class(**__A      )
																						setattr(self.backend_tokenizer						,   __A						,   __A      )

							@property
							def 					__A       (     self:      Union[str, Any]      )			-> str:
												if self._mask_token is None:
																	if self.verbose:
																						logger.error('''Using mask_token, but it is not set yet.'''      )
																	return None
												return str(self._mask_token      )

							@mask_token.setter
							def 					__A       (     self:      Optional[int]						,   __A:      str      )			-> Union[str, Any]:
												_A									=       AddedToken(__A						,   lstrip=__A						,   rstrip=__A      ) if isinstance(__A						,   __A      ) else value
												_A									=       value

							def 					__A       (     self:      Optional[Any]						,   *__A:      Any						,   **__A:      List[Any]      )			-> BatchEncoding:
												_A									=       kwargs.get('''is_split_into_words'''						,   __A      )
												assert self.add_prefix_space or not is_split_into_words, (
												    f"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
												    "to use it with pretokenized inputs."
												)

												return super()._batch_encode_plus(*__A						,   **__A      )

							def 					__A       (     self:      Optional[Any]						,   *__A:      List[str]						,   **__A:      List[Any]      )			-> BatchEncoding:
												_A									=       kwargs.get('''is_split_into_words'''						,   __A      )

												assert self.add_prefix_space or not is_split_into_words, (
												    f"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
												    "to use it with pretokenized inputs."
												)

												return super()._encode_plus(*__A						,   **__A      )

							def 					__A       (     self:      List[str]						,   __A:      str						,   __A:      Optional[str] = None      )			-> Tuple[str]:
												_A									=       self._tokenizer.model.save(__A						,   name=__A      )
												return tuple(__A      )

							def 					__A       (     self:      Tuple						,   __A:      Dict						,   __A:      Optional[Any]=None      )			-> Optional[int]:
												_A									=       [self.bos_token_id] + token_ids_a + [self.eos_token_id]
												if token_ids_a is None:
																	return output

												return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]



							def 					__A       (     self:      Optional[int]						,   __A:      List[int]						,   __A:      Optional[List[int]] = None      )			-> List[int]:
												_A									=       [self.sep_token_id]
												_A									=       [self.cls_token_id]

												if token_ids_a is None:
																	return len(cls + token_ids_a + sep      ) * [0]
												return len(cls + token_ids_a + sep + sep + token_ids_a + sep      ) * [0]



