



'''simple docstring'''




import json
import os
from functools import lru_cache
from typing import List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


UpperCamelCase__ :			str												=				logging.get_logger(__name__)


UpperCamelCase__ :			Union[str, Any]												=				{'''vocab_file''': '''vocab.json''', '''merges_file''': '''merges.txt'''}

UpperCamelCase__ :			str												=				{
    '''vocab_file''': {
        '''allenai/longformer-base-4096''': '''https://huggingface.co/allenai/longformer-base-4096/resolve/main/vocab.json''',
        '''allenai/longformer-large-4096''': (
            '''https://huggingface.co/allenai/longformer-large-4096/resolve/main/vocab.json'''
        ),
        '''allenai/longformer-large-4096-finetuned-triviaqa''': (
            '''https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/vocab.json'''
        ),
        '''allenai/longformer-base-4096-extra.pos.embd.only''': (
            '''https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/vocab.json'''
        ),
        '''allenai/longformer-large-4096-extra.pos.embd.only''': (
            '''https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/vocab.json'''
        ),
    },
    '''merges_file''': {
        '''allenai/longformer-base-4096''': '''https://huggingface.co/allenai/longformer-base-4096/resolve/main/merges.txt''',
        '''allenai/longformer-large-4096''': (
            '''https://huggingface.co/allenai/longformer-large-4096/resolve/main/merges.txt'''
        ),
        '''allenai/longformer-large-4096-finetuned-triviaqa''': (
            '''https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/merges.txt'''
        ),
        '''allenai/longformer-base-4096-extra.pos.embd.only''': (
            '''https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/merges.txt'''
        ),
        '''allenai/longformer-large-4096-extra.pos.embd.only''': (
            '''https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/merges.txt'''
        ),
    },
}

UpperCamelCase__ :			List[str]												=				{
    '''allenai/longformer-base-4096''': 40_96,
    '''allenai/longformer-large-4096''': 40_96,
    '''allenai/longformer-large-4096-finetuned-triviaqa''': 40_96,
    '''allenai/longformer-base-4096-extra.pos.embd.only''': 40_96,
    '''allenai/longformer-large-4096-extra.pos.embd.only''': 40_96,
}


@lru_cache()
# Copied from transformers.models.roberta.tokenization_roberta.bytes_to_unicode
def        lowerCAmelCase_     (			):
		__SCREAMING_SNAKE_CASE :	Dict					 =	(
		    list(range(ord("""!"""      ) ,   ord("""~"""      ) + 1      )      ) + list(range(ord("""¡"""      ) ,   ord("""¬"""      ) + 1      )      ) + list(range(ord("""®"""      ) ,   ord("""ÿ"""      ) + 1      )      )
		)
		__SCREAMING_SNAKE_CASE :	Tuple					 =	bs[:]
		__SCREAMING_SNAKE_CASE :	str					 =	0
		for b in range(2**8      ):
				if b not in bs:
						bs.append(_lowerCamelCase      )
						cs.append(2**8 + n      )
						n += 1
		__SCREAMING_SNAKE_CASE :	Optional[Any]					 =	[chr(_lowerCamelCase      ) for n in cs]
		return dict(zip(_lowerCamelCase ,   _lowerCamelCase      )      )


def        lowerCAmelCase_     (			_lowerCamelCase:     List[str]      ):
		__SCREAMING_SNAKE_CASE :	Tuple					 =	set()
		__SCREAMING_SNAKE_CASE :	int					 =	word[0]
		for char in word[1:]:
				pairs.add((prev_char, char)      )
				__SCREAMING_SNAKE_CASE :	Tuple					 =	char
		return pairs






class 		_UpperCamelCase						(							lowerCamelCase__					):


			'''simple docstring'''

			_A   :	str    					=	VOCAB_FILES_NAMES
			_A   :	List[str]    					=	PRETRAINED_VOCAB_FILES_MAP
			_A   :	List[Any]    					=	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			_A   :	List[Any]    					=	['''input_ids''', '''attention_mask''']

			def __init__(							self  :				Tuple							,  lowerCAmelCase__  :				Optional[Any]							,  lowerCAmelCase__  :				Any							,  lowerCAmelCase__  :				Optional[int]="replace"							,  lowerCAmelCase__  :				int="<s>"							,  lowerCAmelCase__  :				Tuple="</s>"							,  lowerCAmelCase__  :				Tuple="</s>"							,  lowerCAmelCase__  :				Optional[int]="<s>"							,  lowerCAmelCase__  :				Optional[Any]="<unk>"							,  lowerCAmelCase__  :				List[Any]="<pad>"							,  lowerCAmelCase__  :				List[Any]="<mask>"							,  lowerCAmelCase__  :				Tuple=False							,  **lowerCAmelCase__  :				Optional[int]							,  ):





					"""simple docstring"""






					__SCREAMING_SNAKE_CASE :	Optional[Any]					 =	AddedToken(lowerCAmelCase__							,  lstrip=lowerCAmelCase__							,  rstrip=lowerCAmelCase__		) if isinstance(lowerCAmelCase__							,  lowerCAmelCase__		) else bos_token
					__SCREAMING_SNAKE_CASE :	Any					 =	AddedToken(lowerCAmelCase__							,  lstrip=lowerCAmelCase__							,  rstrip=lowerCAmelCase__		) if isinstance(lowerCAmelCase__							,  lowerCAmelCase__		) else eos_token
					__SCREAMING_SNAKE_CASE :	List[str]					 =	AddedToken(lowerCAmelCase__							,  lstrip=lowerCAmelCase__							,  rstrip=lowerCAmelCase__		) if isinstance(lowerCAmelCase__							,  lowerCAmelCase__		) else sep_token
					__SCREAMING_SNAKE_CASE :	Union[str, Any]					 =	AddedToken(lowerCAmelCase__							,  lstrip=lowerCAmelCase__							,  rstrip=lowerCAmelCase__		) if isinstance(lowerCAmelCase__							,  lowerCAmelCase__		) else cls_token
					__SCREAMING_SNAKE_CASE :	int					 =	AddedToken(lowerCAmelCase__							,  lstrip=lowerCAmelCase__							,  rstrip=lowerCAmelCase__		) if isinstance(lowerCAmelCase__							,  lowerCAmelCase__		) else unk_token
					__SCREAMING_SNAKE_CASE :	Union[str, Any]					 =	AddedToken(lowerCAmelCase__							,  lstrip=lowerCAmelCase__							,  rstrip=lowerCAmelCase__		) if isinstance(lowerCAmelCase__							,  lowerCAmelCase__		) else pad_token

					# Mask token behave like a normal word, i.e. include the space before it
					__SCREAMING_SNAKE_CASE :	Union[str, Any]					 =	AddedToken(lowerCAmelCase__							,  lstrip=lowerCAmelCase__							,  rstrip=lowerCAmelCase__		) if isinstance(lowerCAmelCase__							,  lowerCAmelCase__		) else mask_token

					super().__init__(
					    errors=lowerCAmelCase__							,  bos_token=lowerCAmelCase__							,  eos_token=lowerCAmelCase__							,  unk_token=lowerCAmelCase__							,  sep_token=lowerCAmelCase__							,  cls_token=lowerCAmelCase__							,  pad_token=lowerCAmelCase__							,  mask_token=lowerCAmelCase__							,  add_prefix_space=lowerCAmelCase__							,  **lowerCAmelCase__							,  )

					with open(lowerCAmelCase__							,  encoding="""utf-8"""		) as vocab_handle:
							__SCREAMING_SNAKE_CASE :	List[str]					 =	json.load(lowerCAmelCase__		)
					__SCREAMING_SNAKE_CASE :	Optional[Any]					 =	{v: k for k, v in self.encoder.items()}
					__SCREAMING_SNAKE_CASE :	Dict					 =	errors  # how to handle errors in decoding
					__SCREAMING_SNAKE_CASE :	Dict					 =	bytes_to_unicode()
					__SCREAMING_SNAKE_CASE :	Optional[Any]					 =	{v: k for k, v in self.byte_encoder.items()}
					with open(lowerCAmelCase__							,  encoding="""utf-8"""		) as merges_handle:
							__SCREAMING_SNAKE_CASE :	Tuple					 =	merges_handle.read().split("""\n"""		)[1:-1]
					__SCREAMING_SNAKE_CASE :	str					 =	[tuple(merge.split()		) for merge in bpe_merges]
					__SCREAMING_SNAKE_CASE :	List[str]					 =	dict(zip(lowerCAmelCase__							,  range(len(lowerCAmelCase__		)		)		)		)
					__SCREAMING_SNAKE_CASE :	Tuple					 =	{}
					__SCREAMING_SNAKE_CASE :	Optional[Any]					 =	add_prefix_space

					# Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
					__SCREAMING_SNAKE_CASE :	Optional[int]					 =	re.compile(r"""'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"""		)

			@property
			def UpperCamelCase__      (							self  :				Tuple		):





					"""simple docstring"""






					return len(self.encoder		)

			def UpperCamelCase__      (							self  :				List[str]		):





					"""simple docstring"""






					return dict(self.encoder							,  **self.added_tokens_encoder		)

			def UpperCamelCase__      (							self  :				Union[str, Any]							,  lowerCAmelCase__  :				Dict		):





					"""simple docstring"""






					if token in self.cache:
							return self.cache[token]
					__SCREAMING_SNAKE_CASE :	Dict					 =	tuple(lowerCAmelCase__		)
					__SCREAMING_SNAKE_CASE :	Union[str, Any]					 =	get_pairs(lowerCAmelCase__		)

					if not pairs:
							return token

					while True:
							__SCREAMING_SNAKE_CASE :	Tuple					 =	min(lowerCAmelCase__							,  key=lambda lowerCAmelCase__		: self.bpe_ranks.get(lowerCAmelCase__							,  float("""inf"""		)		)		)
							if bigram not in self.bpe_ranks:
									break
							__SCREAMING_SNAKE_CASE :	Dict					 =	bigram
							__SCREAMING_SNAKE_CASE :	List[str]					 =	[]
							__SCREAMING_SNAKE_CASE :	int					 =	0
							while i < len(lowerCAmelCase__		):
									try:
											__SCREAMING_SNAKE_CASE :	Optional[Any]					 =	word.index(lowerCAmelCase__							,  lowerCAmelCase__		)
									except ValueError:
											new_word.extend(word[i:]		)
											break
									else:
											new_word.extend(word[i:j]		)
											__SCREAMING_SNAKE_CASE :	Dict					 =	j

									if word[i] == first and i < len(lowerCAmelCase__		) - 1 and word[i + 1] == second:
											new_word.append(first + second		)
											i += 2
									else:
											new_word.append(word[i]		)
											i += 1
							__SCREAMING_SNAKE_CASE :	Union[str, Any]					 =	tuple(lowerCAmelCase__		)
							__SCREAMING_SNAKE_CASE :	Tuple					 =	new_word
							if len(lowerCAmelCase__		) == 1:
									break
							else:
									__SCREAMING_SNAKE_CASE :	Any					 =	get_pairs(lowerCAmelCase__		)
					__SCREAMING_SNAKE_CASE :	Any					 =	""" """.join(lowerCAmelCase__		)
					__SCREAMING_SNAKE_CASE :	Optional[Any]					 =	word
					return word

			def UpperCamelCase__      (							self  :				Any							,  lowerCAmelCase__  :				Tuple		):





					"""simple docstring"""






					__SCREAMING_SNAKE_CASE :	List[str]					 =	[]
					for token in re.findall(self.pat							,  lowerCAmelCase__		):
							__SCREAMING_SNAKE_CASE :	Tuple					 =	"""""".join(
							    self.byte_encoder[b] for b in token.encode("""utf-8"""		)		)  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
							bpe_tokens.extend(bpe_token for bpe_token in self.bpe(lowerCAmelCase__		).split(""" """		)		)
					return bpe_tokens

			def UpperCamelCase__      (							self  :				Optional[Any]							,  lowerCAmelCase__  :				int		):





					"""simple docstring"""






					return self.encoder.get(lowerCAmelCase__							,  self.encoder.get(self.unk_token		)		)

			def UpperCamelCase__      (							self  :				Optional[int]							,  lowerCAmelCase__  :				str		):





					"""simple docstring"""






					return self.decoder.get(lowerCAmelCase__		)

			def UpperCamelCase__      (							self  :				Any							,  lowerCAmelCase__  :				Tuple		):





					"""simple docstring"""






					__SCREAMING_SNAKE_CASE :	List[str]					 =	"""""".join(lowerCAmelCase__		)
					__SCREAMING_SNAKE_CASE :	List[str]					 =	bytearray([self.byte_decoder[c] for c in text]		).decode("""utf-8"""							,  errors=self.errors		)
					return text

			def UpperCamelCase__      (							self  :				Dict							,  lowerCAmelCase__  :				str							,  lowerCAmelCase__  :				Optional[str] = None		):





					"""simple docstring"""






					if not os.path.isdir(lowerCAmelCase__		):
							logger.error(F"Vocabulary path ({save_directory}) should be a directory"		)
							return
					__SCREAMING_SNAKE_CASE :	Any					 =	os.path.join(
					    lowerCAmelCase__							,  (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]		)
					__SCREAMING_SNAKE_CASE :	Union[str, Any]					 =	os.path.join(
					    lowerCAmelCase__							,  (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""merges_file"""]		)

					with open(lowerCAmelCase__							,  """w"""							,  encoding="""utf-8"""		) as f:
							f.write(json.dumps(self.encoder							,  indent=2							,  sort_keys=lowerCAmelCase__							,  ensure_ascii=lowerCAmelCase__		) + """\n"""		)

					__SCREAMING_SNAKE_CASE :	Any					 =	0
					with open(lowerCAmelCase__							,  """w"""							,  encoding="""utf-8"""		) as writer:
							writer.write("""#version: 0.2\n"""		)
							for bpe_tokens, token_index in sorted(self.bpe_ranks.items()							,  key=lambda lowerCAmelCase__		: kv[1]		):
									if index != token_index:
											logger.warning(
											    F"Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."
											    """ Please check that the tokenizer is not corrupted!"""		)
											__SCREAMING_SNAKE_CASE :	str					 =	token_index
									writer.write(""" """.join(lowerCAmelCase__		) + """\n"""		)
									index += 1

					return vocab_file, merge_file

			def UpperCamelCase__      (							self  :				int							,  lowerCAmelCase__  :				List[int]							,  lowerCAmelCase__  :				Optional[List[int]] = None		):





					"""simple docstring"""






					if token_ids_a is None:
							return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
					__SCREAMING_SNAKE_CASE :	List[Any]					 =	[self.cls_token_id]
					__SCREAMING_SNAKE_CASE :	Dict					 =	[self.sep_token_id]
					return cls + token_ids_a + sep + sep + token_ids_a + sep

			def UpperCamelCase__      (							self  :				Union[str, Any]							,  lowerCAmelCase__  :				List[int]							,  lowerCAmelCase__  :				Optional[List[int]] = None							,  lowerCAmelCase__  :				bool = False		):





					"""simple docstring"""






					if already_has_special_tokens:
							return super().get_special_tokens_mask(
							    token_ids_a=lowerCAmelCase__							,  token_ids_a=lowerCAmelCase__							,  already_has_special_tokens=lowerCAmelCase__		)

					if token_ids_a is None:
							return [1] + ([0] * len(lowerCAmelCase__		)) + [1]
					return [1] + ([0] * len(lowerCAmelCase__		)) + [1, 1] + ([0] * len(lowerCAmelCase__		)) + [1]

			def UpperCamelCase__      (							self  :				str							,  lowerCAmelCase__  :				List[int]							,  lowerCAmelCase__  :				Optional[List[int]] = None		):





					"""simple docstring"""






					__SCREAMING_SNAKE_CASE :	Any					 =	[self.sep_token_id]
					__SCREAMING_SNAKE_CASE :	List[str]					 =	[self.cls_token_id]

					if token_ids_a is None:
							return len(cls + token_ids_a + sep		) * [0]
					return len(cls + token_ids_a + sep + sep + token_ids_a + sep		) * [0]



			def UpperCamelCase__      (							self  :				str							,  lowerCAmelCase__  :				Tuple							,  lowerCAmelCase__  :				List[str]=False							,  **lowerCAmelCase__  :				Dict		):





					"""simple docstring"""






					__SCREAMING_SNAKE_CASE :	List[str]					 =	kwargs.pop("""add_prefix_space"""							,  self.add_prefix_space		)
					if (is_split_into_words or add_prefix_space) and (len(lowerCAmelCase__		) > 0 and not text[0].isspace()):
							__SCREAMING_SNAKE_CASE :	Optional[Any]					 =	""" """ + text
					return (text, kwargs)





'''simple docstring'''




from collections import OrderedDict
from typing import Mapping

from packaging import version

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


UpperCamelCase__ :			Any												=				logging.get_logger(__name__)

UpperCamelCase__ :			Dict												=				{
    '''microsoft/beit-base-patch16-224-pt22k''': (
        '''https://huggingface.co/microsoft/beit-base-patch16-224-pt22k/resolve/main/config.json'''
    ),
    # See all BEiT models at https://huggingface.co/models?filter=beit
}






class 		_UpperCamelCase						(							lowerCamelCase__					):


			'''simple docstring'''

			_A   :	Optional[int]    					=	'''beit'''

			def __init__(							self  :				Tuple							,  lowerCAmelCase__  :				List[Any]=8_1_9_2							,  lowerCAmelCase__  :				List[Any]=7_6_8							,  lowerCAmelCase__  :				List[Any]=1_2							,  lowerCAmelCase__  :				Union[str, Any]=1_2							,  lowerCAmelCase__  :				Dict=3_0_7_2							,  lowerCAmelCase__  :				List[Any]="gelu"							,  lowerCAmelCase__  :				List[Any]=0.0							,  lowerCAmelCase__  :				int=0.0							,  lowerCAmelCase__  :				Tuple=0.02							,  lowerCAmelCase__  :				Any=1E-12							,  lowerCAmelCase__  :				Dict=2_2_4							,  lowerCAmelCase__  :				List[str]=1_6							,  lowerCAmelCase__  :				int=3							,  lowerCAmelCase__  :				Union[str, Any]=False							,  lowerCAmelCase__  :				Union[str, Any]=False							,  lowerCAmelCase__  :				Any=False							,  lowerCAmelCase__  :				Union[str, Any]=False							,  lowerCAmelCase__  :				Optional[int]=0.1							,  lowerCAmelCase__  :				Dict=0.1							,  lowerCAmelCase__  :				Tuple=True							,  lowerCAmelCase__  :				str=[3, 5, 7, 1_1]							,  lowerCAmelCase__  :				str=[1, 2, 3, 6]							,  lowerCAmelCase__  :				Dict=True							,  lowerCAmelCase__  :				Dict=0.4							,  lowerCAmelCase__  :				int=2_5_6							,  lowerCAmelCase__  :				Any=1							,  lowerCAmelCase__  :				str=False							,  lowerCAmelCase__  :				Union[str, Any]=2_5_5							,  **lowerCAmelCase__  :				Optional[int]							,  ):





					"""simple docstring"""






					super().__init__(**lowerCAmelCase__		)

					__SCREAMING_SNAKE_CASE :	int					 =	vocab_size
					__SCREAMING_SNAKE_CASE :	int					 =	hidden_size
					__SCREAMING_SNAKE_CASE :	Optional[Any]					 =	num_hidden_layers
					__SCREAMING_SNAKE_CASE :	List[str]					 =	num_attention_heads
					__SCREAMING_SNAKE_CASE :	Union[str, Any]					 =	intermediate_size
					__SCREAMING_SNAKE_CASE :	Union[str, Any]					 =	hidden_act
					__SCREAMING_SNAKE_CASE :	Optional[Any]					 =	hidden_dropout_prob
					__SCREAMING_SNAKE_CASE :	Any					 =	attention_probs_dropout_prob
					__SCREAMING_SNAKE_CASE :	Dict					 =	initializer_range
					__SCREAMING_SNAKE_CASE :	Any					 =	layer_norm_eps

					__SCREAMING_SNAKE_CASE :	Dict					 =	image_size
					__SCREAMING_SNAKE_CASE :	Optional[int]					 =	patch_size
					__SCREAMING_SNAKE_CASE :	int					 =	num_channels
					__SCREAMING_SNAKE_CASE :	Tuple					 =	use_mask_token
					__SCREAMING_SNAKE_CASE :	Union[str, Any]					 =	use_absolute_position_embeddings
					__SCREAMING_SNAKE_CASE :	Optional[int]					 =	use_relative_position_bias
					__SCREAMING_SNAKE_CASE :	Optional[Any]					 =	use_shared_relative_position_bias
					__SCREAMING_SNAKE_CASE :	Optional[int]					 =	layer_scale_init_value
					__SCREAMING_SNAKE_CASE :	Any					 =	drop_path_rate
					__SCREAMING_SNAKE_CASE :	Union[str, Any]					 =	use_mean_pooling
					# decode head attributes (semantic segmentation)
					__SCREAMING_SNAKE_CASE :	Any					 =	out_indices
					__SCREAMING_SNAKE_CASE :	List[Any]					 =	pool_scales
					# auxiliary head attributes (semantic segmentation)
					__SCREAMING_SNAKE_CASE :	List[Any]					 =	use_auxiliary_head
					__SCREAMING_SNAKE_CASE :	str					 =	auxiliary_loss_weight
					__SCREAMING_SNAKE_CASE :	Union[str, Any]					 =	auxiliary_channels
					__SCREAMING_SNAKE_CASE :	str					 =	auxiliary_num_convs
					__SCREAMING_SNAKE_CASE :	int					 =	auxiliary_concat_input
					__SCREAMING_SNAKE_CASE :	Dict					 =	semantic_loss_ignore_index






class 		_UpperCamelCase						(							lowerCamelCase__					):


			'''simple docstring'''

			_A   :	List[str]    					=	version.parse('''1.11'''					)

			@property
			def UpperCamelCase__      (							self  :				List[str]		):





					"""simple docstring"""






					return OrderedDict(
					    [
					        ("""pixel_values""", {0: """batch""", 1: """num_channels""", 2: """height""", 3: """width"""}),
					    ]		)



			@property
			def UpperCamelCase__      (							self  :				Any		):





					"""simple docstring"""






					return 1E-4