'''simple docstring'''







import unittest

from transformers import AlbertConfig, is_torch_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
		import torch

		from transformers import (
		    MODEL_FOR_PRETRAINING_MAPPING,
		    AlbertForMaskedLM,
		    AlbertForMultipleChoice,
		    AlbertForPreTraining,
		    AlbertForQuestionAnswering,
		    AlbertForSequenceClassification,
		    AlbertForTokenClassification,
		    AlbertModel,
		)
		from transformers.models.albert.modeling_albert import ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST




class _SCREAMING_SNAKE_CASE:




						def __init__(    self :       Tuple							,							UpperCamelCase_ :       str							,							UpperCamelCase_ :       Union[str, Any]=13							,							UpperCamelCase_ :       Tuple=7							,							UpperCamelCase_ :       Union[str, Any]=True							,							UpperCamelCase_ :       int=True							,							UpperCamelCase_ :       Tuple=True							,							UpperCamelCase_ :       Optional[int]=True							,							UpperCamelCase_ :       int=99							,							UpperCamelCase_ :       Dict=16							,							UpperCamelCase_ :       Optional[Any]=36							,							UpperCamelCase_ :       Any=6							,							UpperCamelCase_ :       str=6							,							UpperCamelCase_ :       Dict=6							,							UpperCamelCase_ :       List[Any]=37							,							UpperCamelCase_ :       str="gelu"							,							UpperCamelCase_ :       int=0.1							,							UpperCamelCase_ :       Tuple=0.1							,							UpperCamelCase_ :       Dict=5_12							,							UpperCamelCase_ :       Dict=16							,							UpperCamelCase_ :       Union[str, Any]=2							,							UpperCamelCase_ :       Optional[int]=0.02							,							UpperCamelCase_ :       List[Any]=3							,							UpperCamelCase_ :       List[str]=4							,							UpperCamelCase_ :       str=None							,							)     ->				List[str]:
													SCREAMING_SNAKE_CASE__     :str						    = parent
													SCREAMING_SNAKE_CASE__     :str						    = batch_size
													SCREAMING_SNAKE_CASE__     :List[Any]						    = seq_length
													SCREAMING_SNAKE_CASE__     :str						    = is_training
													SCREAMING_SNAKE_CASE__     :List[str]						    = use_input_mask
													SCREAMING_SNAKE_CASE__     :str						    = use_token_type_ids
													SCREAMING_SNAKE_CASE__     :List[str]						    = use_labels
													SCREAMING_SNAKE_CASE__     :List[str]						    = vocab_size
													SCREAMING_SNAKE_CASE__     :Union[str, Any]						    = embedding_size
													SCREAMING_SNAKE_CASE__     :Any						    = hidden_size
													SCREAMING_SNAKE_CASE__     :Optional[Any]						    = num_hidden_layers
													SCREAMING_SNAKE_CASE__     :List[str]						    = num_hidden_groups
													SCREAMING_SNAKE_CASE__     :Optional[Any]						    = num_attention_heads
													SCREAMING_SNAKE_CASE__     :Union[str, Any]						    = intermediate_size
													SCREAMING_SNAKE_CASE__     :List[str]						    = hidden_act
													SCREAMING_SNAKE_CASE__     :Tuple						    = hidden_dropout_prob
													SCREAMING_SNAKE_CASE__     :Union[str, Any]						    = attention_probs_dropout_prob
													SCREAMING_SNAKE_CASE__     :str						    = max_position_embeddings
													SCREAMING_SNAKE_CASE__     :List[str]						    = type_vocab_size
													SCREAMING_SNAKE_CASE__     :Dict						    = type_sequence_label_size
													SCREAMING_SNAKE_CASE__     :Optional[int]						    = initializer_range
													SCREAMING_SNAKE_CASE__     :Tuple						    = num_labels
													SCREAMING_SNAKE_CASE__     :Dict						    = num_choices
													SCREAMING_SNAKE_CASE__     :int						    = scope




						def      __lowerCamelCase       (    self :       Tuple      )     ->				Dict:
													SCREAMING_SNAKE_CASE__     :int						    = ids_tensor([self.batch_size, self.seq_length]							,							self.vocab_size      )

													SCREAMING_SNAKE_CASE__     :Tuple						    = None
													if self.use_input_mask:
																				SCREAMING_SNAKE_CASE__     :Any						    = random_attention_mask([self.batch_size, self.seq_length]      )

													SCREAMING_SNAKE_CASE__     :List[Any]						    = None
													if self.use_token_type_ids:
																				SCREAMING_SNAKE_CASE__     :int						    = ids_tensor([self.batch_size, self.seq_length]							,							self.type_vocab_size      )

													SCREAMING_SNAKE_CASE__     :str						    = None
													SCREAMING_SNAKE_CASE__     :List[str]						    = None
													SCREAMING_SNAKE_CASE__     :int						    = None
													if self.use_labels:
																				SCREAMING_SNAKE_CASE__     :str						    = ids_tensor([self.batch_size]							,							self.type_sequence_label_size      )
																				SCREAMING_SNAKE_CASE__     :List[str]						    = ids_tensor([self.batch_size, self.seq_length]							,							self.num_labels      )
																				SCREAMING_SNAKE_CASE__     :Optional[int]						    = ids_tensor([self.batch_size]							,							self.num_choices      )

													SCREAMING_SNAKE_CASE__     :List[Any]						    = self.get_config()

													return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels




						def      __lowerCamelCase       (    self :       Dict      )     ->				Union[str, Any]:
													return AlbertConfig(
													    vocab_size=self.vocab_size							,							hidden_size=self.hidden_size							,							num_hidden_layers=self.num_hidden_layers							,							num_attention_heads=self.num_attention_heads							,							intermediate_size=self.intermediate_size							,							hidden_act=self.hidden_act							,							hidden_dropout_prob=self.hidden_dropout_prob							,							attention_probs_dropout_prob=self.attention_probs_dropout_prob							,							max_position_embeddings=self.max_position_embeddings							,							type_vocab_size=self.type_vocab_size							,							initializer_range=self.initializer_range							,							num_hidden_groups=self.num_hidden_groups							,							)




						def      __lowerCamelCase       (    self :       Dict							,							UpperCamelCase_ :       Union[str, Any]							,							UpperCamelCase_ :       List[Any]							,							UpperCamelCase_ :       List[str]							,							UpperCamelCase_ :       Dict							,							UpperCamelCase_ :       int							,							UpperCamelCase_ :       Union[str, Any]							,							UpperCamelCase_ :       List[Any]      )     ->				List[Any]:
													SCREAMING_SNAKE_CASE__     :List[str]						    = AlbertModel(config=UpperCamelCase_      )
													model.to(UpperCamelCase_      )
													model.eval()
													SCREAMING_SNAKE_CASE__     :Dict						    = model(UpperCamelCase_							,							attention_mask=UpperCamelCase_							,							token_type_ids=UpperCamelCase_      )
													SCREAMING_SNAKE_CASE__     :Any						    = model(UpperCamelCase_							,							token_type_ids=UpperCamelCase_      )
													SCREAMING_SNAKE_CASE__     :List[str]						    = model(UpperCamelCase_      )
													self.parent.assertEqual(result.last_hidden_state.shape							,							(self.batch_size, self.seq_length, self.hidden_size)      )
													self.parent.assertEqual(result.pooler_output.shape							,							(self.batch_size, self.hidden_size)      )




						def      __lowerCamelCase       (    self :       Tuple							,							UpperCamelCase_ :       Any							,							UpperCamelCase_ :       int							,							UpperCamelCase_ :       str							,							UpperCamelCase_ :       str							,							UpperCamelCase_ :       List[str]							,							UpperCamelCase_ :       Optional[int]							,							UpperCamelCase_ :       Optional[Any]      )     ->				str:
													SCREAMING_SNAKE_CASE__     :List[Any]						    = AlbertForPreTraining(config=UpperCamelCase_      )
													model.to(UpperCamelCase_      )
													model.eval()
													SCREAMING_SNAKE_CASE__     :Union[str, Any]						    = model(
													    UpperCamelCase_							,							attention_mask=UpperCamelCase_							,							token_type_ids=UpperCamelCase_							,							labels=UpperCamelCase_							,							sentence_order_label=UpperCamelCase_							,							)
													self.parent.assertEqual(result.prediction_logits.shape							,							(self.batch_size, self.seq_length, self.vocab_size)      )
													self.parent.assertEqual(result.sop_logits.shape							,							(self.batch_size, config.num_labels)      )




						def      __lowerCamelCase       (    self :       List[Any]							,							UpperCamelCase_ :       List[str]							,							UpperCamelCase_ :       Union[str, Any]							,							UpperCamelCase_ :       Union[str, Any]							,							UpperCamelCase_ :       Union[str, Any]							,							UpperCamelCase_ :       Optional[int]							,							UpperCamelCase_ :       Tuple							,							UpperCamelCase_ :       Tuple      )     ->				Optional[Any]:
													SCREAMING_SNAKE_CASE__     :Tuple						    = AlbertForMaskedLM(config=UpperCamelCase_      )
													model.to(UpperCamelCase_      )
													model.eval()
													SCREAMING_SNAKE_CASE__     :List[Any]						    = model(UpperCamelCase_							,							attention_mask=UpperCamelCase_							,							token_type_ids=UpperCamelCase_							,							labels=UpperCamelCase_      )
													self.parent.assertEqual(result.logits.shape							,							(self.batch_size, self.seq_length, self.vocab_size)      )




						def      __lowerCamelCase       (    self :       str							,							UpperCamelCase_ :       List[Any]							,							UpperCamelCase_ :       List[str]							,							UpperCamelCase_ :       List[str]							,							UpperCamelCase_ :       Tuple							,							UpperCamelCase_ :       Union[str, Any]							,							UpperCamelCase_ :       Any							,							UpperCamelCase_ :       List[Any]      )     ->				Tuple:
													SCREAMING_SNAKE_CASE__     :Union[str, Any]						    = AlbertForQuestionAnswering(config=UpperCamelCase_      )
													model.to(UpperCamelCase_      )
													model.eval()
													SCREAMING_SNAKE_CASE__     :Optional[int]						    = model(
													    UpperCamelCase_							,							attention_mask=UpperCamelCase_							,							token_type_ids=UpperCamelCase_							,							start_positions=UpperCamelCase_							,							end_positions=UpperCamelCase_							,							)
													self.parent.assertEqual(result.start_logits.shape							,							(self.batch_size, self.seq_length)      )
													self.parent.assertEqual(result.end_logits.shape							,							(self.batch_size, self.seq_length)      )




						def      __lowerCamelCase       (    self :       int							,							UpperCamelCase_ :       Optional[int]							,							UpperCamelCase_ :       List[Any]							,							UpperCamelCase_ :       Tuple							,							UpperCamelCase_ :       Tuple							,							UpperCamelCase_ :       Any							,							UpperCamelCase_ :       Optional[int]							,							UpperCamelCase_ :       List[str]      )     ->				Dict:
													SCREAMING_SNAKE_CASE__     :Dict						    = self.num_labels
													SCREAMING_SNAKE_CASE__     :List[str]						    = AlbertForSequenceClassification(UpperCamelCase_      )
													model.to(UpperCamelCase_      )
													model.eval()
													SCREAMING_SNAKE_CASE__     :Tuple						    = model(UpperCamelCase_							,							attention_mask=UpperCamelCase_							,							token_type_ids=UpperCamelCase_							,							labels=UpperCamelCase_      )
													self.parent.assertEqual(result.logits.shape							,							(self.batch_size, self.num_labels)      )




						def      __lowerCamelCase       (    self :       List[Any]							,							UpperCamelCase_ :       int							,							UpperCamelCase_ :       Optional[int]							,							UpperCamelCase_ :       str							,							UpperCamelCase_ :       Tuple							,							UpperCamelCase_ :       int							,							UpperCamelCase_ :       Union[str, Any]							,							UpperCamelCase_ :       Optional[int]      )     ->				Union[str, Any]:
													SCREAMING_SNAKE_CASE__     :List[str]						    = self.num_labels
													SCREAMING_SNAKE_CASE__     :Dict						    = AlbertForTokenClassification(config=UpperCamelCase_      )
													model.to(UpperCamelCase_      )
													model.eval()
													SCREAMING_SNAKE_CASE__     :int						    = model(UpperCamelCase_							,							attention_mask=UpperCamelCase_							,							token_type_ids=UpperCamelCase_							,							labels=UpperCamelCase_      )
													self.parent.assertEqual(result.logits.shape							,							(self.batch_size, self.seq_length, self.num_labels)      )




						def      __lowerCamelCase       (    self :       List[Any]							,							UpperCamelCase_ :       str							,							UpperCamelCase_ :       Union[str, Any]							,							UpperCamelCase_ :       Dict							,							UpperCamelCase_ :       int							,							UpperCamelCase_ :       Dict							,							UpperCamelCase_ :       str							,							UpperCamelCase_ :       Any      )     ->				Any:
													SCREAMING_SNAKE_CASE__     :Union[str, Any]						    = self.num_choices
													SCREAMING_SNAKE_CASE__     :Dict						    = AlbertForMultipleChoice(config=UpperCamelCase_      )
													model.to(UpperCamelCase_      )
													model.eval()
													SCREAMING_SNAKE_CASE__     :Any						    = input_ids.unsqueeze(1      ).expand(-1							,							self.num_choices							,							-1      ).contiguous()
													SCREAMING_SNAKE_CASE__     :Dict						    = token_type_ids.unsqueeze(1      ).expand(-1							,							self.num_choices							,							-1      ).contiguous()
													SCREAMING_SNAKE_CASE__     :Tuple						    = input_mask.unsqueeze(1      ).expand(-1							,							self.num_choices							,							-1      ).contiguous()
													SCREAMING_SNAKE_CASE__     :List[str]						    = model(
													    UpperCamelCase_							,							attention_mask=UpperCamelCase_							,							token_type_ids=UpperCamelCase_							,							labels=UpperCamelCase_							,							)
													self.parent.assertEqual(result.logits.shape							,							(self.batch_size, self.num_choices)      )




						def      __lowerCamelCase       (    self :       Dict      )     ->				Optional[int]:
													SCREAMING_SNAKE_CASE__     :Optional[int]						    = self.prepare_config_and_inputs()
													(
													    (
													    SCREAMING_SNAKE_CASE__
													)    ,      (
													    SCREAMING_SNAKE_CASE__
													)    ,      (
													    SCREAMING_SNAKE_CASE__
													)    ,      (
													    SCREAMING_SNAKE_CASE__
													)    ,      (
													    SCREAMING_SNAKE_CASE__
													)    ,      (
													    SCREAMING_SNAKE_CASE__
													)    ,      (
													    SCREAMING_SNAKE_CASE__
													)    ,      
													)     :str						    = config_and_inputs
													SCREAMING_SNAKE_CASE__     :Dict						    = {'input_ids': input_ids, 'token_type_ids': token_type_ids, 'attention_mask': input_mask}
													return config, inputs_dict




@require_torch
class _SCREAMING_SNAKE_CASE(							_SCREAMING_SNAKE_CASE						,						_SCREAMING_SNAKE_CASE						,						unittest.TestCase					):
						A_			:							str     		= (
						    (
						        AlbertModel,
						        AlbertForPreTraining,
						        AlbertForMaskedLM,
						        AlbertForMultipleChoice,
						        AlbertForSequenceClassification,
						        AlbertForTokenClassification,
						        AlbertForQuestionAnswering,
						    )
						    if is_torch_available()
						    else ()
						)
						A_			:							Optional[int]     		= (
						    {
						        'feature-extraction': AlbertModel,
						        'fill-mask': AlbertForMaskedLM,
						        'question-answering': AlbertForQuestionAnswering,
						        'text-classification': AlbertForSequenceClassification,
						        'token-classification': AlbertForTokenClassification,
						        'zero-shot': AlbertForSequenceClassification,
						    }
						    if is_torch_available()
						    else {}
						)
						A_			:							Tuple     		= True




						def      __lowerCamelCase       (    self :       int							,							UpperCamelCase_ :       int							,							UpperCamelCase_ :       List[Any]							,							UpperCamelCase_ :       str=False      )     ->				Dict:
													SCREAMING_SNAKE_CASE__     :Dict						    = super()._prepare_for_class(UpperCamelCase_							,							UpperCamelCase_							,							return_labels=UpperCamelCase_      )

													if return_labels:
																				if model_class in get_values(UpperCamelCase_      ):
																											SCREAMING_SNAKE_CASE__     :Tuple						    = torch.zeros(
																											    (self.model_tester.batch_size, self.model_tester.seq_length)							,							dtype=torch.long							,							device=UpperCamelCase_      )
																											SCREAMING_SNAKE_CASE__     :int						    = torch.zeros(
																											    self.model_tester.batch_size							,							dtype=torch.long							,							device=UpperCamelCase_      )
													return inputs_dict




						def      __lowerCamelCase       (    self :       List[Any]      )     ->				List[Any]:
													SCREAMING_SNAKE_CASE__     :Dict						    = AlbertModelTester(self      )
													SCREAMING_SNAKE_CASE__     :List[Any]						    = ConfigTester(self							,							config_class=UpperCamelCase_							,							hidden_size=37      )




						def      __lowerCamelCase       (    self :       Any      )     ->				int:
													self.config_tester.run_common_tests()




						def      __lowerCamelCase       (    self :       Optional[int]      )     ->				int:
													SCREAMING_SNAKE_CASE__     :List[str]						    = self.model_tester.prepare_config_and_inputs()
													self.model_tester.create_and_check_model(*UpperCamelCase_      )




						def      __lowerCamelCase       (    self :       Tuple      )     ->				Dict:
													SCREAMING_SNAKE_CASE__     :Union[str, Any]						    = self.model_tester.prepare_config_and_inputs()
													self.model_tester.create_and_check_for_pretraining(*UpperCamelCase_      )




						def      __lowerCamelCase       (    self :       List[str]      )     ->				Optional[Any]:
													SCREAMING_SNAKE_CASE__     :List[str]						    = self.model_tester.prepare_config_and_inputs()
													self.model_tester.create_and_check_for_masked_lm(*UpperCamelCase_      )




						def      __lowerCamelCase       (    self :       Any      )     ->				Union[str, Any]:
													SCREAMING_SNAKE_CASE__     :str						    = self.model_tester.prepare_config_and_inputs()
													self.model_tester.create_and_check_for_multiple_choice(*UpperCamelCase_      )




						def      __lowerCamelCase       (    self :       Union[str, Any]      )     ->				Dict:
													SCREAMING_SNAKE_CASE__     :Any						    = self.model_tester.prepare_config_and_inputs()
													self.model_tester.create_and_check_for_question_answering(*UpperCamelCase_      )




						def      __lowerCamelCase       (    self :       List[Any]      )     ->				Optional[int]:
													SCREAMING_SNAKE_CASE__     :List[Any]						    = self.model_tester.prepare_config_and_inputs()
													self.model_tester.create_and_check_for_sequence_classification(*UpperCamelCase_      )




						def      __lowerCamelCase       (    self :       Dict      )     ->				Tuple:
													SCREAMING_SNAKE_CASE__     :Tuple						    = self.model_tester.prepare_config_and_inputs()
													for type in ["absolute", "relative_key", "relative_key_query"]:
																				SCREAMING_SNAKE_CASE__     :List[Any]						    = type
																				self.model_tester.create_and_check_model(*UpperCamelCase_      )




						@slow
						def      __lowerCamelCase       (    self :       int      )     ->				Optional[Any]:
													for model_name in ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
																				SCREAMING_SNAKE_CASE__     :Union[str, Any]						    = AlbertModel.from_pretrained(UpperCamelCase_      )
																				self.assertIsNotNone(UpperCamelCase_      )




@require_torch
class _SCREAMING_SNAKE_CASE(							unittest.TestCase					):




						@slow
						def      __lowerCamelCase       (    self :       List[str]      )     ->				Tuple:
													SCREAMING_SNAKE_CASE__     :List[Any]						    = AlbertModel.from_pretrained('albert-base-v2'      )
													SCREAMING_SNAKE_CASE__     :Tuple						    = torch.tensor([[0, 3_45, 2_32, 3_28, 7_40, 1_40, 16_95, 69, 60_78, 15_88, 2]]      )
													SCREAMING_SNAKE_CASE__     :Dict						    = torch.tensor([[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]      )
													with torch.no_grad():
																				SCREAMING_SNAKE_CASE__     :Any						    = model(UpperCamelCase_							,							attention_mask=UpperCamelCase_      )[0]
													SCREAMING_SNAKE_CASE__     :Any						    = torch.Size((1, 11, 7_68)      )
													self.assertEqual(output.shape							,							UpperCamelCase_      )
													SCREAMING_SNAKE_CASE__     :Dict						    = torch.tensor(
													    [[[-0.6513, 1.5035, -0.2766], [-0.6515, 1.5046, -0.2780], [-0.6512, 1.5049, -0.2784]]]      )

													self.assertTrue(torch.allclose(output[:, 1:4, 1:4]							,							UpperCamelCase_							,							atol=1e-4      )      )







'''simple docstring'''







import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
		from .tokenization_pegasus import PegasusTokenizer
else:
		UpperCamelCase_      	=	None


UpperCamelCase_      	=	logging.get_logger(__name__)


UpperCamelCase_      	=	'''▁'''

UpperCamelCase_      	=	{'''vocab_file''': '''spiece.model''', '''tokenizer_file''': '''tokenizer.json'''}

UpperCamelCase_      	=	{
    '''vocab_file''': {'''google/pegasus-xsum''': '''https://huggingface.co/google/pegasus-xsum/resolve/main/spiece.model'''},
    '''tokenizer_file''': {
        '''google/pegasus-xsum''': '''https://huggingface.co/google/pegasus-xsum/resolve/main/tokenizer.json'''
    },
}

UpperCamelCase_      	=	{
    '''google/pegasus-xsum''': 5_12,
}




class _SCREAMING_SNAKE_CASE(							_SCREAMING_SNAKE_CASE					):
						A_			:							Union[str, Any]     		= VOCAB_FILES_NAMES
						A_			:							str     		= PRETRAINED_VOCAB_FILES_MAP
						A_			:							List[str]     		= PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
						A_			:							int     		= PegasusTokenizer
						A_			:							Dict     		= ['input_ids', 'attention_mask']




						def __init__(    self :       List[str]							,							UpperCamelCase_ :       str=None							,							UpperCamelCase_ :       str=None							,							UpperCamelCase_ :       Union[str, Any]="<pad>"							,							UpperCamelCase_ :       List[Any]="</s>"							,							UpperCamelCase_ :       Tuple="<unk>"							,							UpperCamelCase_ :       List[Any]="<mask_2>"							,							UpperCamelCase_ :       int="<mask_1>"							,							UpperCamelCase_ :       List[str]=None							,							UpperCamelCase_ :       Optional[Any]=1_03							,							**UpperCamelCase_ :       Optional[Any]							,							)     ->				Optional[int]:
													SCREAMING_SNAKE_CASE__     :Dict						    = offset

													if additional_special_tokens is not None:
																				if not isinstance(UpperCamelCase_							,							UpperCamelCase_      ):
																											raise TypeError(
																											    f'''additional_special_tokens should be of type {type(UpperCamelCase_      )}, but is'''
																											    f''' {type(UpperCamelCase_      )}'''      )

																				SCREAMING_SNAKE_CASE__     :List[str]						    = (
																				    ([mask_token_sent] + additional_special_tokens)
																				    if mask_token_sent not in additional_special_tokens and mask_token_sent is not None
																				    else additional_special_tokens
																				)
																				# fill additional tokens with ..., <unk_token_102> in case not all additional tokens are already taken
																				additional_special_tokens_extended += [
																				    f'''<unk_{i}>''' for i in range(len(UpperCamelCase_      )							,							self.offset - 1      )
																				]

																				if len(set(UpperCamelCase_      )      ) != len(UpperCamelCase_      ):
																											raise ValueError(
																											    'Please make sure that the provided additional_special_tokens do not contain an incorrectly'
																											    f''' shifted list of <unk_x> tokens. Found {additional_special_tokens_extended}.'''      )
																				SCREAMING_SNAKE_CASE__     :List[str]						    = additional_special_tokens_extended
													else:
																				SCREAMING_SNAKE_CASE__     :Tuple						    = [mask_token_sent] if mask_token_sent is not None else []
																				additional_special_tokens += [f'''<unk_{i}>''' for i in range(2							,							self.offset      )]

													super().__init__(
													    UpperCamelCase_							,							tokenizer_file=UpperCamelCase_							,							pad_token=UpperCamelCase_							,							eos_token=UpperCamelCase_							,							unk_token=UpperCamelCase_							,							mask_token=UpperCamelCase_							,							mask_token_sent=UpperCamelCase_							,							offset=UpperCamelCase_							,							additional_special_tokens=UpperCamelCase_							,							**UpperCamelCase_							,							)
													SCREAMING_SNAKE_CASE__     :int						    = vocab_file
													SCREAMING_SNAKE_CASE__     :Union[str, Any]						    = False if not self.vocab_file else True




						def      __lowerCamelCase       (    self :       int							,							UpperCamelCase_ :       Any      )     ->				Optional[int]:
													SCREAMING_SNAKE_CASE__     :int						    = set(self.all_special_ids      )  # call it once instead of inside list comp
													all_special_ids.remove(self.unk_token_id      )  # <unk> is only sometimes special

													if all_special_ids != set(range(len(self.additional_special_tokens      ) + 3      )      ):
																				raise ValueError(
																				    'There should be 3 special tokens: mask_token, pad_token, and eos_token +'
																				    f''' {len(self.additional_special_tokens      )} additional_special_tokens, but got {all_special_ids}'''      )

													return [1 if x in all_special_ids else 0 for x in seq]




						def      __lowerCamelCase       (    self :       Optional[int]							,							UpperCamelCase_ :       List							,							UpperCamelCase_ :       Optional[List] = None							,							UpperCamelCase_ :       bool = False      )     ->				List[int]:
													if already_has_special_tokens:
																				return self._special_token_mask(UpperCamelCase_      )
													elif token_ids_a is None:
																				return self._special_token_mask(UpperCamelCase_      ) + [1]
													else:
																				return self._special_token_mask(token_ids_a + token_ids_a      ) + [1]




						def      __lowerCamelCase       (    self :       Union[str, Any]							,							UpperCamelCase_ :       Any							,							UpperCamelCase_ :       Dict=None      )     ->				List[int]:
													if token_ids_a is None:
																				return token_ids_a + [self.eos_token_id]
													# We don't expect to process pairs, but leave the pair logic for API consistency
													return token_ids_a + token_ids_a + [self.eos_token_id]




						def      __lowerCamelCase       (    self :       Union[str, Any]							,							UpperCamelCase_ :       str							,							UpperCamelCase_ :       Optional[str] = None      )     ->				Tuple[str]:
													if not self.can_save_slow_tokenizer:
																				raise ValueError(
																				    'Your fast tokenizer does not have the necessary information to save the vocabulary for a slow '
																				    'tokenizer.'      )

													if not os.path.isdir(UpperCamelCase_      ):
																				logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''      )
																				return
													SCREAMING_SNAKE_CASE__     :Optional[int]						    = os.path.join(
													    UpperCamelCase_							,							(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']      )

													if os.path.abspath(self.vocab_file      ) != os.path.abspath(UpperCamelCase_      ):
																				copyfile(self.vocab_file							,							UpperCamelCase_      )

													return (out_vocab_file,)






