


from ...configuration_utils import PretrainedConfig


A			:						Optional[int]        =  {
    'google/tapas-base-finetuned-sqa': (
        'https://huggingface.co/google/tapas-base-finetuned-sqa/resolve/main/config.json'
    ),
    'google/tapas-base-finetuned-wtq': (
        'https://huggingface.co/google/tapas-base-finetuned-wtq/resolve/main/config.json'
    ),
    'google/tapas-base-finetuned-wikisql-supervised': (
        'https://huggingface.co/google/tapas-base-finetuned-wikisql-supervised/resolve/main/config.json'
    ),
    'google/tapas-base-finetuned-tabfact': (
        'https://huggingface.co/google/tapas-base-finetuned-tabfact/resolve/main/config.json'
    ),
}



class 				A       (			UpperCAmelCase__				):
					'''simple docstring'''






					A__     =					'''tapas'''

					def __init__(self       :		Optional[int] ,   _UpperCAmelCase       :		int=3_0522 ,   _UpperCAmelCase       :		str=768 ,   _UpperCAmelCase       :		Optional[int]=12 ,   _UpperCAmelCase       :		int=12 ,   _UpperCAmelCase       :		Tuple=3072 ,   _UpperCAmelCase       :		Dict="gelu" ,   _UpperCAmelCase       :		List[Any]=0.1 ,   _UpperCAmelCase       :		Optional[Any]=0.1 ,   _UpperCAmelCase       :		List[str]=1024 ,   _UpperCAmelCase       :		Any=[3, 256, 256, 2, 256, 256, 10] ,   _UpperCAmelCase       :		List[Any]=0.02 ,   _UpperCAmelCase       :		Union[str, Any]=1E-1_2 ,   _UpperCAmelCase       :		Optional[int]=0 ,   _UpperCAmelCase       :		List[str]=10.0 ,   _UpperCAmelCase       :		Optional[int]=0 ,   _UpperCAmelCase       :		List[Any]=1.0 ,   _UpperCAmelCase       :		Dict=None ,   _UpperCAmelCase       :		Dict=1.0 ,   _UpperCAmelCase       :		str=False ,   _UpperCAmelCase       :		int=None ,   _UpperCAmelCase       :		List[str]=1.0 ,   _UpperCAmelCase       :		Union[str, Any]=1.0 ,   _UpperCAmelCase       :		List[str]=False ,   _UpperCAmelCase       :		Dict=False ,   _UpperCAmelCase       :		Any="ratio" ,   _UpperCAmelCase       :		Optional[int]=None ,   _UpperCAmelCase       :		Dict=None ,   _UpperCAmelCase       :		List[Any]=64 ,   _UpperCAmelCase       :		Any=32 ,   _UpperCAmelCase       :		Optional[Any]=False ,   _UpperCAmelCase       :		List[str]=True ,   _UpperCAmelCase       :		List[str]=False ,   _UpperCAmelCase       :		Tuple=False ,   _UpperCAmelCase       :		Any=True ,   _UpperCAmelCase       :		List[Any]=False ,   _UpperCAmelCase       :		Tuple=None ,   _UpperCAmelCase       :		Any=None ,   **_UpperCAmelCase       :		Dict ,   )		->  Dict:




							"""simple docstring"""


							super().__init__(pad_token_id=_UpperCAmelCase ,   **_UpperCAmelCase						)

							# BERT hyperparameters (with updated max_position_embeddings and type_vocab_sizes)
							lowercase__	     =						vocab_size
							lowercase__	     =						hidden_size
							lowercase__	     =						num_hidden_layers
							lowercase__	     =						num_attention_heads
							lowercase__	     =						hidden_act
							lowercase__	     =						intermediate_size
							lowercase__	     =						hidden_dropout_prob
							lowercase__	     =						attention_probs_dropout_prob
							lowercase__	     =						max_position_embeddings
							lowercase__	     =						type_vocab_sizes
							lowercase__	     =						initializer_range
							lowercase__	     =						layer_norm_eps

							# Fine-tuning task hyperparameters
							lowercase__	     =						positive_label_weight
							lowercase__	     =						num_aggregation_labels
							lowercase__	     =						aggregation_loss_weight
							lowercase__	     =						use_answer_as_supervision
							lowercase__	     =						answer_loss_importance
							lowercase__	     =						use_normalized_answer_loss
							lowercase__	     =						huber_loss_delta
							lowercase__	     =						temperature
							lowercase__	     =						aggregation_temperature
							lowercase__	     =						use_gumbel_for_cells
							lowercase__	     =						use_gumbel_for_aggregation
							lowercase__	     =						average_approximation_function
							lowercase__	     =						cell_selection_preference
							lowercase__	     =						answer_loss_cutoff
							lowercase__	     =						max_num_rows
							lowercase__	     =						max_num_columns
							lowercase__	     =						average_logits_per_cell
							lowercase__	     =						select_one_column
							lowercase__	     =						allow_empty_column_selection
							lowercase__	     =						init_cell_selection_weights_to_zero
							lowercase__	     =						reset_position_index_per_cell
							lowercase__	     =						disable_per_token_loss

							# Aggregation hyperparameters
							lowercase__	     =						aggregation_labels
							lowercase__	     =						no_aggregation_label_index

							if isinstance(self.aggregation_labels ,   _UpperCAmelCase						):
									lowercase__	     =						{int(_UpperCAmelCase						): v for k, v in aggregation_labels.items()}





import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
	from .tokenization_pegasus import PegasusTokenizer
else:
	A			:						List[Any]        =  None


A			:						Optional[Any]        =  logging.get_logger(__name__)


A			:						str        =  '▁'

A			:						Optional[Any]        =  {'vocab_file': 'spiece.model', 'tokenizer_file': 'tokenizer.json'}

A			:						Any        =  {
    'vocab_file': {'google/pegasus-xsum': 'https://huggingface.co/google/pegasus-xsum/resolve/main/spiece.model'},
    'tokenizer_file': {
        'google/pegasus-xsum': 'https://huggingface.co/google/pegasus-xsum/resolve/main/tokenizer.json'
    },
}

A			:						Optional[int]        =  {
    'google/pegasus-xsum': 5_1_2,
}



class 				A       (			UpperCAmelCase__				):
					'''simple docstring'''






					A__     =					VOCAB_FILES_NAMES
					A__     =					PRETRAINED_VOCAB_FILES_MAP
					A__     =					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					A__     =					PegasusTokenizer
					A__     =					['''input_ids''', '''attention_mask''']

					def __init__(self       :		Tuple ,   _UpperCAmelCase       :		Union[str, Any]=None ,   _UpperCAmelCase       :		List[Any]=None ,   _UpperCAmelCase       :		Union[str, Any]="<pad>" ,   _UpperCAmelCase       :		int="</s>" ,   _UpperCAmelCase       :		int="<unk>" ,   _UpperCAmelCase       :		Optional[Any]="<mask_2>" ,   _UpperCAmelCase       :		Optional[Any]="<mask_1>" ,   _UpperCAmelCase       :		Optional[Any]=None ,   _UpperCAmelCase       :		Dict=103 ,   **_UpperCAmelCase       :		Dict ,   )		->  Dict:




							"""simple docstring"""


							lowercase__	     =						offset

							if additional_special_tokens is not None:
									if not isinstance(_UpperCAmelCase ,   _UpperCAmelCase						):
											raise TypeError(
											    f'''additional_special_tokens should be of type {type(_UpperCAmelCase						)}, but is'''
											    f''' {type(_UpperCAmelCase						)}'''						)

									lowercase__	     =						(
									    ([mask_token_sent] + additional_special_tokens)
									    if mask_token_sent not in additional_special_tokens and mask_token_sent is not None
									    else additional_special_tokens
									)
									# fill additional tokens with ..., <unk_token_102> in case not all additional tokens are already taken
									additional_special_tokens_extended += [
									    f'''<unk_{i}>''' for i in range(len(_UpperCAmelCase						) ,   self.offset - 1						)
									]

									if len(set(_UpperCAmelCase						)						) != len(_UpperCAmelCase						):
											raise ValueError(
											    """Please make sure that the provided additional_special_tokens do not contain an incorrectly"""
											    f''' shifted list of <unk_x> tokens. Found {additional_special_tokens_extended}.'''						)
									lowercase__	     =						additional_special_tokens_extended
							else:
									lowercase__	     =						[mask_token_sent] if mask_token_sent is not None else []
									additional_special_tokens += [f'''<unk_{i}>''' for i in range(2 ,   self.offset						)]

							super().__init__(
							    _UpperCAmelCase ,   tokenizer_file=_UpperCAmelCase ,   pad_token=_UpperCAmelCase ,   eos_token=_UpperCAmelCase ,   unk_token=_UpperCAmelCase ,   mask_token=_UpperCAmelCase ,   mask_token_sent=_UpperCAmelCase ,   offset=_UpperCAmelCase ,   additional_special_tokens=_UpperCAmelCase ,   **_UpperCAmelCase ,   )
							lowercase__	     =						vocab_file
							lowercase__	     =						False if not self.vocab_file else True

					def     lowerCamelCase__	(self       :		Optional[Any] ,   _UpperCAmelCase       :		Tuple						)		->  Union[str, Any]:




							"""simple docstring"""


							lowercase__	     =						set(self.all_special_ids						)  # call it once instead of inside list comp
							all_special_ids.remove(self.unk_token_id						)  # <unk> is only sometimes special

							if all_special_ids != set(range(len(self.additional_special_tokens						) + 3						)						):
									raise ValueError(
									    """There should be 3 special tokens: mask_token, pad_token, and eos_token +"""
									    f''' {len(self.additional_special_tokens						)} additional_special_tokens, but got {all_special_ids}'''						)

							return [1 if x in all_special_ids else 0 for x in seq]

					def     lowerCamelCase__	(self       :		Optional[Any] ,   _UpperCAmelCase       :		List ,   _UpperCAmelCase       :		Optional[List] = None ,   _UpperCAmelCase       :		bool = False						)		->  List[int]:




							"""simple docstring"""


							if already_has_special_tokens:
									return self._special_token_mask(_UpperCAmelCase						)
							elif token_ids_a is None:
									return self._special_token_mask(_UpperCAmelCase						) + [1]
							else:
									return self._special_token_mask(token_ids_a + token_ids_a						) + [1]

					def     lowerCamelCase__	(self       :		int ,   _UpperCAmelCase       :		Optional[int] ,   _UpperCAmelCase       :		Union[str, Any]=None						)		->  List[int]:




							"""simple docstring"""


							if token_ids_a is None:
									return token_ids_a + [self.eos_token_id]
							# We don't expect to process pairs, but leave the pair logic for API consistency
							return token_ids_a + token_ids_a + [self.eos_token_id]


					def     lowerCamelCase__	(self       :		str ,   _UpperCAmelCase       :		str ,   _UpperCAmelCase       :		Optional[str] = None						)		->  Tuple[str]:




							"""simple docstring"""


							if not self.can_save_slow_tokenizer:
									raise ValueError(
									    """Your fast tokenizer does not have the necessary information to save the vocabulary for a slow """
									    """tokenizer."""						)

							if not os.path.isdir(_UpperCAmelCase						):
									logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''						)
									return
							lowercase__	     =						os.path.join(
							    _UpperCAmelCase ,   (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]						)

							if os.path.abspath(self.vocab_file						) != os.path.abspath(_UpperCAmelCase						):
									copyfile(self.vocab_file ,   _UpperCAmelCase						)

							return (out_vocab_file,)

