
'''simple docstring'''







import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


_lowercase			:  Dict   							=	"""▁"""

_lowercase			:  List[Any]   							=	{"""vocab_file""": """spiece.model"""}

_lowercase			:  str   							=	{
    """vocab_file""": {"""google/pegasus-xsum""": """https://huggingface.co/google/pegasus-xsum/resolve/main/spiece.model"""}
}

_lowercase			:  str   							=	{
    """google/pegasus-xsum""": 5_1_2,
}


_lowercase			:  Any   							=	logging.get_logger(__name__)






class 			lowerCAmelCase__       (  lowerCamelCase__					):
					lowerCAmelCase_					       =					VOCAB_FILES_NAMES

					lowerCAmelCase_					       =					VOCAB_FILES_NAMES
					lowerCAmelCase_					       =					PRETRAINED_VOCAB_FILES_MAP
					lowerCAmelCase_					       =					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					lowerCAmelCase_					       =					['input_ids', 'attention_mask']






					def __init__(       self    ,							__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE="<pad>"    ,							__SCREAMING_SNAKE_CASE="</s>"    ,							__SCREAMING_SNAKE_CASE="<unk>"    ,							__SCREAMING_SNAKE_CASE="<mask_2>"    ,							__SCREAMING_SNAKE_CASE="<mask_1>"    ,							__SCREAMING_SNAKE_CASE=None    ,							__SCREAMING_SNAKE_CASE=1_03    ,							__SCREAMING_SNAKE_CASE = None    ,							**__SCREAMING_SNAKE_CASE    ,							):


											"""simple docstring"""





											lowercase_     :					Union[str, Any]													=	offset
											if additional_special_tokens is not None:
																	if not isinstance(__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE		):
																							raise TypeError(
																							    F'''additional_special_tokens should be of type {type(__SCREAMING_SNAKE_CASE		)}, but is'''
																							    F''' {type(__SCREAMING_SNAKE_CASE		)}'''		)

																	lowercase_     :					int													=	(
																	    ([mask_token_sent] + additional_special_tokens)
																	    if mask_token_sent not in additional_special_tokens and mask_token_sent is not None
																	    else additional_special_tokens
																	)
																	# fill additional tokens with ..., <unk_token_102> in case not all additional tokens are already taken
																	additional_special_tokens_extended += [
																	    F'''<unk_{i}>''' for i in range(len(__SCREAMING_SNAKE_CASE		)    ,							self.offset - 1		)
																	]

																	if len(set(__SCREAMING_SNAKE_CASE		)		) != len(__SCREAMING_SNAKE_CASE		):
																							raise ValueError(
																							    '''Please make sure that the provided additional_special_tokens do not contain an incorrectly'''
																							    F''' shifted list of <unk_x> tokens. Found {additional_special_tokens_extended}.'''		)
																	lowercase_     :					List[str]													=	additional_special_tokens_extended
											else:
																	lowercase_     :					int													=	[mask_token_sent] if mask_token_sent is not None else []
																	additional_special_tokens += [F'''<unk_{i}>''' for i in range(2    ,							self.offset		)]

											lowercase_     :					List[str]													=	{} if sp_model_kwargs is None else sp_model_kwargs

											super().__init__(
											    eos_token=__SCREAMING_SNAKE_CASE    ,							unk_token=__SCREAMING_SNAKE_CASE    ,							mask_token=__SCREAMING_SNAKE_CASE    ,							pad_token=__SCREAMING_SNAKE_CASE    ,							mask_token_sent=__SCREAMING_SNAKE_CASE    ,							offset=__SCREAMING_SNAKE_CASE    ,							additional_special_tokens=__SCREAMING_SNAKE_CASE    ,							sp_model_kwargs=self.sp_model_kwargs    ,							**__SCREAMING_SNAKE_CASE    ,							)
											lowercase_     :					List[Any]													=	mask_token_sent
											lowercase_     :					Tuple													=	vocab_file
											lowercase_     :					Any													=	spm.SentencePieceProcessor(**self.sp_model_kwargs		)
											self.sp_model.Load(__SCREAMING_SNAKE_CASE		)

											# add special tokens to encoder dict
											lowercase_     :					Optional[Any]													=	{
											    0: self.pad_token,
											    1: self.eos_token,
											}

											if self.mask_token_sent is not None:
																	self.encoder.update(
																	    {
																	        2: self.mask_token_sent,
																	        3: self.mask_token,
																	    }		)

											if self.offset > 0:
																	# entries 2-104 are only used for pretraining and called <mask_1>, <mask_2>, unk_2, ...unk_102
																	# mask_token_sent is already added to list -> so start at 1
																	self.encoder.update({i + 3: additional_special_tokens[i] for i in range(1    ,							self.offset - 1		)}		)

											lowercase_     :					str													=	{v: k for k, v in self.encoder.items()}






					@property
					def      _snake_case   (       self		):


											"""simple docstring"""





											return len(self.sp_model		) + self.offset






					def      _snake_case   (       self		):


											"""simple docstring"""





											lowercase_     :					int													=	{self.convert_ids_to_tokens(__SCREAMING_SNAKE_CASE		): i for i in range(self.vocab_size		)}
											vocab.update(self.added_tokens_encoder		)
											return vocab






					def __getstate__(       self		):


											"""simple docstring"""





											lowercase_     :					Optional[Any]													=	self.__dict__.copy()
											lowercase_     :					str													=	None
											return state






					def __setstate__(       self    ,							__SCREAMING_SNAKE_CASE		):


											"""simple docstring"""





											lowercase_     :					List[Any]													=	d

											# for backward compatibility
											if not hasattr(self    ,							'''sp_model_kwargs'''		):
																	lowercase_     :					Tuple													=	{}

											lowercase_     :					List[str]													=	spm.SentencePieceProcessor(**self.sp_model_kwargs		)
											self.sp_model.Load(self.vocab_file		)






					def      _snake_case   (       self    ,							__SCREAMING_SNAKE_CASE		):


											"""simple docstring"""





											return self.sp_model.encode(__SCREAMING_SNAKE_CASE    ,							out_type=__SCREAMING_SNAKE_CASE		)






					def      _snake_case   (       self    ,							__SCREAMING_SNAKE_CASE		):


											"""simple docstring"""





											if token in self.decoder:
																	return self.decoder[token]
											elif token in self.added_tokens_decoder:
																	return self.added_tokens_decoder[token]
											lowercase_     :					Dict													=	self.sp_model.piece_to_id(__SCREAMING_SNAKE_CASE		)
											return sp_id + self.offset






					def      _snake_case   (       self    ,							__SCREAMING_SNAKE_CASE		):


											"""simple docstring"""





											if index in self.encoder:
																	return self.encoder[index]
											elif index in self.added_tokens_encoder:
																	return self.added_tokens_encoder[index]
											else:
																	lowercase_     :					Any													=	self.sp_model.IdToPiece(index - self.offset		)
											return token






					def      _snake_case   (       self    ,							__SCREAMING_SNAKE_CASE		):


											"""simple docstring"""





											lowercase_     :					Optional[int]													=	[]
											lowercase_     :					Union[str, Any]													=	''''''
											for token in tokens:
																	# make sure that special tokens are not decoded using sentencepiece model
																	if token in self.all_special_tokens:
																							out_string += self.sp_model.decode(__SCREAMING_SNAKE_CASE		) + token
																							lowercase_     :					str													=	[]
																	else:
																							current_sub_tokens.append(__SCREAMING_SNAKE_CASE		)
											out_string += self.sp_model.decode(__SCREAMING_SNAKE_CASE		)
											return out_string.strip()






					def      _snake_case   (       self    ,							__SCREAMING_SNAKE_CASE=False		):


											"""simple docstring"""





											return 1






					def      _snake_case   (       self    ,							__SCREAMING_SNAKE_CASE		):


											"""simple docstring"""





											lowercase_     :					Optional[int]													=	set(self.all_special_ids		)  # call it once instead of inside list comp
											all_special_ids.remove(self.unk_token_id		)  # <unk> is only sometimes special

											return [1 if x in all_special_ids else 0 for x in seq]






					def      _snake_case   (       self    ,							__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE = None    ,							__SCREAMING_SNAKE_CASE = False		):


											"""simple docstring"""





											if already_has_special_tokens:
																	return self._special_token_mask(__SCREAMING_SNAKE_CASE		)
											elif token_ids_a is None:
																	return self._special_token_mask(__SCREAMING_SNAKE_CASE		) + [1]
											else:
																	return self._special_token_mask(token_ids_a + token_ids_a		) + [1]






					def      _snake_case   (       self    ,							__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE=None		):


											"""simple docstring"""





											if token_ids_a is None:
																	return token_ids_a + [self.eos_token_id]
											# We don't expect to process pairs, but leave the pair logic for API consistency
											return token_ids_a + token_ids_a + [self.eos_token_id]






					def      _snake_case   (       self    ,							__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE = None		):


											"""simple docstring"""





											if not os.path.isdir(__SCREAMING_SNAKE_CASE		):
																	logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''		)
																	return
											lowercase_     :					Tuple													=	os.path.join(
											    __SCREAMING_SNAKE_CASE    ,							(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']		)

											if os.path.abspath(self.vocab_file		) != os.path.abspath(__SCREAMING_SNAKE_CASE		) and os.path.isfile(self.vocab_file		):
																	copyfile(self.vocab_file    ,							__SCREAMING_SNAKE_CASE		)
											elif not os.path.isfile(self.vocab_file		):
																	with open(__SCREAMING_SNAKE_CASE    ,							'''wb'''		) as fi:
																							lowercase_     :					int													=	self.sp_model.serialized_model_proto()
																							fi.write(__SCREAMING_SNAKE_CASE		)

											return (out_vocab_file,)





'''simple docstring'''







from collections import OrderedDict
from typing import Mapping

from packaging import version

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


_lowercase			:  List[Any]   							=	logging.get_logger(__name__)

_lowercase			:  Optional[Any]   							=	{
    "google/vit-base-patch16-224": "https://huggingface.co/vit-base-patch16-224/resolve/main/config.json",
    # See all ViT models at https://huggingface.co/models?filter=vit
}






class 			lowerCAmelCase__       (  lowerCamelCase_					):
					lowerCAmelCase_					       =					'''vit'''






					def __init__(       self    ,							__SCREAMING_SNAKE_CASE=7_68    ,							__SCREAMING_SNAKE_CASE=12    ,							__SCREAMING_SNAKE_CASE=12    ,							__SCREAMING_SNAKE_CASE=30_72    ,							__SCREAMING_SNAKE_CASE="gelu"    ,							__SCREAMING_SNAKE_CASE=0.0    ,							__SCREAMING_SNAKE_CASE=0.0    ,							__SCREAMING_SNAKE_CASE=0.02    ,							__SCREAMING_SNAKE_CASE=1E-1_2    ,							__SCREAMING_SNAKE_CASE=2_24    ,							__SCREAMING_SNAKE_CASE=16    ,							__SCREAMING_SNAKE_CASE=3    ,							__SCREAMING_SNAKE_CASE=True    ,							__SCREAMING_SNAKE_CASE=16    ,							**__SCREAMING_SNAKE_CASE    ,							):


											"""simple docstring"""





											super().__init__(**__SCREAMING_SNAKE_CASE		)

											lowercase_     :					Union[str, Any]													=	hidden_size
											lowercase_     :					Dict													=	num_hidden_layers
											lowercase_     :					List[Any]													=	num_attention_heads
											lowercase_     :					Any													=	intermediate_size
											lowercase_     :					Union[str, Any]													=	hidden_act
											lowercase_     :					Dict													=	hidden_dropout_prob
											lowercase_     :					List[str]													=	attention_probs_dropout_prob
											lowercase_     :					Any													=	initializer_range
											lowercase_     :					Tuple													=	layer_norm_eps
											lowercase_     :					Union[str, Any]													=	image_size
											lowercase_     :					Tuple													=	patch_size
											lowercase_     :					Tuple													=	num_channels
											lowercase_     :					Union[str, Any]													=	qkv_bias
											lowercase_     :					List[Any]													=	encoder_stride







class 			lowerCAmelCase__       (  lowerCamelCase_					):
					lowerCAmelCase_					       =					version.parse('''1.11'''					)






					@property
					def      _snake_case   (       self		):


											"""simple docstring"""





											return OrderedDict(
											    [
											        ('''pixel_values''', {0: '''batch''', 1: '''num_channels''', 2: '''height''', 3: '''width'''}),
											    ]		)






					@property
					def      _snake_case   (       self		):


											"""simple docstring"""





											return 1E-4



