






"""simple docstring"""







import json
import os
from functools import lru_cache
from typing import List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


_lowerCAmelCase            =	logging.get_logger(__name__)


_lowerCAmelCase            =	{'vocab_file': 'vocab.json', 'merges_file': 'merges.txt'}

_lowerCAmelCase            =	{
    'vocab_file': {
        'allenai/longformer-base-4096': 'https://huggingface.co/allenai/longformer-base-4096/resolve/main/vocab.json',
        'allenai/longformer-large-4096': (
            'https://huggingface.co/allenai/longformer-large-4096/resolve/main/vocab.json'
        ),
        'allenai/longformer-large-4096-finetuned-triviaqa': (
            'https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/vocab.json'
        ),
        'allenai/longformer-base-4096-extra.pos.embd.only': (
            'https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/vocab.json'
        ),
        'allenai/longformer-large-4096-extra.pos.embd.only': (
            'https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/vocab.json'
        ),
    },
    'merges_file': {
        'allenai/longformer-base-4096': 'https://huggingface.co/allenai/longformer-base-4096/resolve/main/merges.txt',
        'allenai/longformer-large-4096': (
            'https://huggingface.co/allenai/longformer-large-4096/resolve/main/merges.txt'
        ),
        'allenai/longformer-large-4096-finetuned-triviaqa': (
            'https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/merges.txt'
        ),
        'allenai/longformer-base-4096-extra.pos.embd.only': (
            'https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/merges.txt'
        ),
        'allenai/longformer-large-4096-extra.pos.embd.only': (
            'https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/merges.txt'
        ),
    },
}

_lowerCAmelCase            =	{
    'allenai/longformer-base-4096': 40_96,
    'allenai/longformer-large-4096': 40_96,
    'allenai/longformer-large-4096-finetuned-triviaqa': 40_96,
    'allenai/longformer-base-4096-extra.pos.embd.only': 40_96,
    'allenai/longformer-large-4096-extra.pos.embd.only': 40_96,
}


@lru_cache()
# Copied from transformers.models.roberta.tokenization_roberta.bytes_to_unicode
def      UpperCamelCase       ( )     ->       Dict:
				lowercase    :      Optional[Any]       = (
				    list(range(ord("""!""" )  ,				ord("""~""" ) + 1 ) ) + list(range(ord("""¡""" )  ,				ord("""¬""" ) + 1 ) ) + list(range(ord("""®""" )  ,				ord("""ÿ""" ) + 1 ) )
				)
				lowercase    :      List[str]       = bs[:]
				lowercase    :      List[Any]       = 0
				for b in range(2**8 ):
								if b not in bs:
												bs.append(_lowerCAmelCase )
												cs.append(2**8 + n )
												n += 1
				lowercase    :      Tuple       = [chr(_lowerCAmelCase ) for n in cs]
				return dict(zip(_lowerCAmelCase  ,				_lowerCAmelCase ) )


def      UpperCamelCase       ( _A )     ->       int:
				lowercase    :      List[str]       = set()
				lowercase    :      Optional[Any]       = word[0]
				for char in word[1:]:
								pairs.add((prev_char, char) )
								lowercase    :      Any       = char
				return pairs


class      UpperCamelCase		(A__     ):
						_SCREAMING_SNAKE_CASE  : str												=		VOCAB_FILES_NAMES
						_SCREAMING_SNAKE_CASE  : Optional[Any]												=		PRETRAINED_VOCAB_FILES_MAP
						_SCREAMING_SNAKE_CASE  : Optional[Any]												=		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
						_SCREAMING_SNAKE_CASE  : Dict												=		["""input_ids""", """attention_mask"""]






						def __init__(		self  :List[str]			, __magic_name__  :List[Any]			, __magic_name__  :Union[str, Any]			, __magic_name__  :Union[str, Any]="replace"			, __magic_name__  :Tuple="<s>"			, __magic_name__  :Any="</s>"			, __magic_name__  :Tuple="</s>"			, __magic_name__  :List[str]="<s>"			, __magic_name__  :Dict="<unk>"			, __magic_name__  :Any="<pad>"			, __magic_name__  :Any="<mask>"			, __magic_name__  :Union[str, Any]=False			, **__magic_name__  :int			, )      ->Optional[int]:
										lowercase    :      Optional[int]       = AddedToken(snake_case_			, lstrip=snake_case_			, rstrip=snake_case_				) if isinstance(snake_case_			, snake_case_				) else bos_token
										lowercase    :      Optional[Any]       = AddedToken(snake_case_			, lstrip=snake_case_			, rstrip=snake_case_				) if isinstance(snake_case_			, snake_case_				) else eos_token
										lowercase    :      Any       = AddedToken(snake_case_			, lstrip=snake_case_			, rstrip=snake_case_				) if isinstance(snake_case_			, snake_case_				) else sep_token
										lowercase    :      int       = AddedToken(snake_case_			, lstrip=snake_case_			, rstrip=snake_case_				) if isinstance(snake_case_			, snake_case_				) else cls_token
										lowercase    :      Optional[int]       = AddedToken(snake_case_			, lstrip=snake_case_			, rstrip=snake_case_				) if isinstance(snake_case_			, snake_case_				) else unk_token
										lowercase    :      Union[str, Any]       = AddedToken(snake_case_			, lstrip=snake_case_			, rstrip=snake_case_				) if isinstance(snake_case_			, snake_case_				) else pad_token

										# Mask token behave like a normal word, i.e. include the space before it
										lowercase    :      Optional[Any]       = AddedToken(snake_case_			, lstrip=snake_case_			, rstrip=snake_case_				) if isinstance(snake_case_			, snake_case_				) else mask_token

										super().__init__(
										    errors=snake_case_			, bos_token=snake_case_			, eos_token=snake_case_			, unk_token=snake_case_			, sep_token=snake_case_			, cls_token=snake_case_			, pad_token=snake_case_			, mask_token=snake_case_			, add_prefix_space=snake_case_			, **snake_case_			, )

										with open(snake_case_			, encoding="""utf-8"""				) as vocab_handle:
														lowercase    :      Tuple       = json.load(snake_case_				)
										lowercase    :      Optional[int]       = {v: k for k, v in self.encoder.items()}
										lowercase    :      Optional[Any]       = errors  # how to handle errors in decoding
										lowercase    :      Any       = bytes_to_unicode()
										lowercase    :      Optional[int]       = {v: k for k, v in self.byte_encoder.items()}
										with open(snake_case_			, encoding="""utf-8"""				) as merges_handle:
														lowercase    :      Optional[int]       = merges_handle.read().split("""\n"""				)[1:-1]
										lowercase    :      List[Any]       = [tuple(merge.split()				) for merge in bpe_merges]
										lowercase    :      Union[str, Any]       = dict(zip(snake_case_			, range(len(snake_case_				)				)				)				)
										lowercase    :      Tuple       = {}
										lowercase    :      Dict       = add_prefix_space

										# Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
										lowercase    :      Optional[int]       = re.compile(r"""'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"""				)






						@property
						def      __snake_case      (		self  :Tuple				)      ->List[Any]:
										return len(self.encoder				)






						def      __snake_case      (		self  :Optional[int]				)      ->Optional[int]:
										return dict(self.encoder			, **self.added_tokens_encoder				)






						def      __snake_case      (		self  :List[str]			, __magic_name__  :Optional[int]				)      ->Optional[int]:
										if token in self.cache:
														return self.cache[token]
										lowercase    :      Optional[Any]       = tuple(snake_case_				)
										lowercase    :      str       = get_pairs(snake_case_				)

										if not pairs:
														return token

										while True:
														lowercase    :      str       = min(snake_case_			, key=lambda __magic_name__				: self.bpe_ranks.get(snake_case_			, float("""inf"""				)				)				)
														if bigram not in self.bpe_ranks:
																		break
														lowercase      ,      lowercase    :      List[Any]       = bigram
														lowercase    :      Dict       = []
														lowercase    :      str       = 0
														while i < len(snake_case_				):
																		try:
																						lowercase    :      Union[str, Any]       = word.index(snake_case_			, snake_case_				)
																		except ValueError:
																						new_word.extend(word[i:]				)
																						break
																		else:
																						new_word.extend(word[i:j]				)
																						lowercase    :      Union[str, Any]       = j

																		if word[i] == first and i < len(snake_case_				) - 1 and word[i + 1] == second:
																						new_word.append(first + second				)
																						i += 2
																		else:
																						new_word.append(word[i]				)
																						i += 1
														lowercase    :      List[str]       = tuple(snake_case_				)
														lowercase    :      List[str]       = new_word
														if len(snake_case_				) == 1:
																		break
														else:
																		lowercase    :      Any       = get_pairs(snake_case_				)
										lowercase    :      Union[str, Any]       = """ """.join(snake_case_				)
										lowercase    :      int       = word
										return word






						def      __snake_case      (		self  :Optional[Any]			, __magic_name__  :Any				)      ->Union[str, Any]:
										lowercase    :      List[str]       = []
										for token in re.findall(self.pat			, snake_case_				):
														lowercase    :      int       = """""".join(
														    self.byte_encoder[b] for b in token.encode("""utf-8"""				)				)  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
														bpe_tokens.extend(bpe_token for bpe_token in self.bpe(snake_case_				).split(""" """				)				)
										return bpe_tokens






						def      __snake_case      (		self  :int			, __magic_name__  :Optional[int]				)      ->Tuple:
										return self.encoder.get(snake_case_			, self.encoder.get(self.unk_token				)				)






						def      __snake_case      (		self  :Optional[int]			, __magic_name__  :int				)      ->Dict:
										return self.decoder.get(snake_case_				)






						def      __snake_case      (		self  :Optional[int]			, __magic_name__  :List[str]				)      ->int:
										lowercase    :      str       = """""".join(snake_case_				)
										lowercase    :      List[Any]       = bytearray([self.byte_decoder[c] for c in text]				).decode("""utf-8"""			, errors=self.errors				)
										return text






						def      __snake_case      (		self  :Any			, __magic_name__  :Any			, __magic_name__  :Any = None				)      ->Tuple[str]:
										if not os.path.isdir(snake_case_				):
														logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""				)
														return
										lowercase    :      Optional[Any]       = os.path.join(
										    snake_case_			, (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]				)
										lowercase    :      List[str]       = os.path.join(
										    snake_case_			, (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""merges_file"""]				)

										with open(snake_case_			, """w"""			, encoding="""utf-8"""				) as f:
														f.write(json.dumps(self.encoder			, indent=2			, sort_keys=snake_case_			, ensure_ascii=snake_case_				) + """\n"""				)

										lowercase    :      Union[str, Any]       = 0
										with open(snake_case_			, """w"""			, encoding="""utf-8"""				) as writer:
														writer.write("""#version: 0.2\n"""				)
														for bpe_tokens, token_index in sorted(self.bpe_ranks.items()			, key=lambda __magic_name__				: kv[1]				):
																		if index != token_index:
																						logger.warning(
																						    f"""Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."""
																						    """ Please check that the tokenizer is not corrupted!"""				)
																						lowercase    :      int       = token_index
																		writer.write(""" """.join(snake_case_				) + """\n"""				)
																		index += 1

										return vocab_file, merge_file






						def      __snake_case      (		self  :Optional[Any]			, __magic_name__  :Optional[Any]			, __magic_name__  :Union[str, Any] = None				)      ->List[int]:
										if token_ids_a is None:
														return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
										lowercase    :      List[Any]       = [self.cls_token_id]
										lowercase    :      Tuple       = [self.sep_token_id]
										return cls + token_ids_a + sep + sep + token_ids_a + sep






						def      __snake_case      (		self  :List[str]			, __magic_name__  :List[Any]			, __magic_name__  :Union[str, Any] = None			, __magic_name__  :Any = False				)      ->List[int]:
										if already_has_special_tokens:
														return super().get_special_tokens_mask(
														    token_ids_a=snake_case_			, token_ids_a=snake_case_			, already_has_special_tokens=snake_case_				)

										if token_ids_a is None:
														return [1] + ([0] * len(snake_case_				)) + [1]
										return [1] + ([0] * len(snake_case_				)) + [1, 1] + ([0] * len(snake_case_				)) + [1]






						def      __snake_case      (		self  :Optional[int]			, __magic_name__  :List[str]			, __magic_name__  :Tuple = None				)      ->List[int]:
										lowercase    :      List[str]       = [self.sep_token_id]
										lowercase    :      Optional[int]       = [self.cls_token_id]

										if token_ids_a is None:
														return len(cls + token_ids_a + sep				) * [0]
										return len(cls + token_ids_a + sep + sep + token_ids_a + sep				) * [0]






						def      __snake_case      (		self  :List[Any]			, __magic_name__  :Optional[Any]			, __magic_name__  :Any=False			, **__magic_name__  :Union[str, Any]				)      ->Optional[Any]:
										lowercase    :      List[str]       = kwargs.pop("""add_prefix_space"""			, self.add_prefix_space				)
										if (is_split_into_words or add_prefix_space) and (len(snake_case_				) > 0 and not text[0].isspace()):
														lowercase    :      Any       = """ """ + text
										return (text, kwargs)


"""simple docstring"""
from ...configuration_utils import PretrainedConfig
from ...utils import logging


SCREAMING_SNAKE_CASE_					     =	logging.get_logger(__name__)

SCREAMING_SNAKE_CASE_					     =	{
    '''alibaba-damo/mgp-str-base''': '''https://huggingface.co/alibaba-damo/mgp-str-base/resolve/main/config.json''',
}
class       lowerCAmelCase_		(				A__ ):



							'''simple docstring'''





							_snake_case						      =		'''mgp-str'''




							def __init__( self    ,				snake_case_=[32, 128]    ,				snake_case_=4    ,				snake_case_=3    ,				snake_case_=27    ,				snake_case_=38    ,				snake_case_=50_257    ,				snake_case_=30_522    ,				snake_case_=768    ,				snake_case_=12    ,				snake_case_=12    ,				snake_case_=4.0    ,				snake_case_=True    ,				snake_case_=False    ,				snake_case_=1e-5    ,				snake_case_=0.0    ,				snake_case_=0.0    ,				snake_case_=0.0    ,				snake_case_=False    ,				snake_case_=0.02    ,				**snake_case_    ,				)       ->  List[Any]:
													super().__init__(**snake_case_				)

													__lowerCAmelCase     =  image_size
													__lowerCAmelCase     =  patch_size
													__lowerCAmelCase     =  num_channels
													__lowerCAmelCase     =  max_token_length
													__lowerCAmelCase     =  num_character_labels
													__lowerCAmelCase     =  num_bpe_labels
													__lowerCAmelCase     =  num_wordpiece_labels
													__lowerCAmelCase     =  hidden_size
													__lowerCAmelCase     =  num_hidden_layers
													__lowerCAmelCase     =  num_attention_heads
													__lowerCAmelCase     =  mlp_ratio
													__lowerCAmelCase     =  distilled
													__lowerCAmelCase     =  layer_norm_eps
													__lowerCAmelCase     =  drop_rate
													__lowerCAmelCase     =  qkv_bias
													__lowerCAmelCase     =  attn_drop_rate
													__lowerCAmelCase     =  drop_path_rate
													__lowerCAmelCase     =  output_aa_attentions
													__lowerCAmelCase     =  initializer_range
