


import collections
import inspect
import unittest

from transformers import FocalNetConfig
from transformers.testing_utils import require_torch, require_vision, slow, torch_device
from transformers.utils import cached_property, is_torch_available, is_vision_available

from ...test_backbone_common import BackboneTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, _config_zero_init, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
	import torch
	from torch import nn

	from transformers import (
	    FocalNetBackbone,
	    FocalNetForImageClassification,
	    FocalNetForMaskedImageModeling,
	    FocalNetModel,
	)
	from transformers.models.focalnet.modeling_focalnet import FOCALNET_PRETRAINED_MODEL_ARCHIVE_LIST

if is_vision_available():
	from PIL import Image

	from transformers import AutoImageProcessor



class 				A       :
					'''simple docstring'''






					def __init__(self       :		Optional[int] ,   _UpperCAmelCase       :		Any ,   _UpperCAmelCase       :		Union[str, Any]=13 ,   _UpperCAmelCase       :		Optional[Any]=32 ,   _UpperCAmelCase       :		List[str]=2 ,   _UpperCAmelCase       :		Tuple=3 ,   _UpperCAmelCase       :		int=16 ,   _UpperCAmelCase       :		Optional[int]=[32, 64, 128] ,   _UpperCAmelCase       :		Any=[1, 2, 1] ,   _UpperCAmelCase       :		Dict=[2, 2, 4] ,   _UpperCAmelCase       :		Dict=2 ,   _UpperCAmelCase       :		int=2.0 ,   _UpperCAmelCase       :		Tuple=True ,   _UpperCAmelCase       :		Tuple=0.0 ,   _UpperCAmelCase       :		List[str]=0.0 ,   _UpperCAmelCase       :		List[Any]=0.1 ,   _UpperCAmelCase       :		Optional[int]="gelu" ,   _UpperCAmelCase       :		Tuple=False ,   _UpperCAmelCase       :		int=True ,   _UpperCAmelCase       :		Tuple=0.02 ,   _UpperCAmelCase       :		Optional[int]=1E-5 ,   _UpperCAmelCase       :		Dict=True ,   _UpperCAmelCase       :		Tuple=None ,   _UpperCAmelCase       :		Any=True ,   _UpperCAmelCase       :		Tuple=10 ,   _UpperCAmelCase       :		List[Any]=8 ,   _UpperCAmelCase       :		List[str]=["stage1", "stage2"] ,   _UpperCAmelCase       :		Optional[int]=[1, 2] ,   )		->  Optional[Any]:




							"""simple docstring"""


							lowercase__	     =						parent
							lowercase__	     =						batch_size
							lowercase__	     =						image_size
							lowercase__	     =						patch_size
							lowercase__	     =						num_channels
							lowercase__	     =						embed_dim
							lowercase__	     =						hidden_sizes
							lowercase__	     =						depths
							lowercase__	     =						num_heads
							lowercase__	     =						window_size
							lowercase__	     =						mlp_ratio
							lowercase__	     =						qkv_bias
							lowercase__	     =						hidden_dropout_prob
							lowercase__	     =						attention_probs_dropout_prob
							lowercase__	     =						drop_path_rate
							lowercase__	     =						hidden_act
							lowercase__	     =						use_absolute_embeddings
							lowercase__	     =						patch_norm
							lowercase__	     =						layer_norm_eps
							lowercase__	     =						initializer_range
							lowercase__	     =						is_training
							lowercase__	     =						scope
							lowercase__	     =						use_labels
							lowercase__	     =						type_sequence_label_size
							lowercase__	     =						encoder_stride
							lowercase__	     =						out_features
							lowercase__	     =						out_indices

					def     lowerCamelCase__	(self       :		Union[str, Any]						)		->  List[str]:




							"""simple docstring"""


							lowercase__	     =						floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size]						)

							lowercase__	     =						None
							if self.use_labels:
									lowercase__	     =						ids_tensor([self.batch_size] ,   self.type_sequence_label_size						)

							lowercase__	     =						self.get_config()

							return config, pixel_values, labels

					def     lowerCamelCase__	(self       :		Any						)		->  List[str]:




							"""simple docstring"""


							return FocalNetConfig(
							    image_size=self.image_size ,   patch_size=self.patch_size ,   num_channels=self.num_channels ,   embed_dim=self.embed_dim ,   hidden_sizes=self.hidden_sizes ,   depths=self.depths ,   num_heads=self.num_heads ,   window_size=self.window_size ,   mlp_ratio=self.mlp_ratio ,   qkv_bias=self.qkv_bias ,   hidden_dropout_prob=self.hidden_dropout_prob ,   attention_probs_dropout_prob=self.attention_probs_dropout_prob ,   drop_path_rate=self.drop_path_rate ,   hidden_act=self.hidden_act ,   use_absolute_embeddings=self.use_absolute_embeddings ,   path_norm=self.patch_norm ,   layer_norm_eps=self.layer_norm_eps ,   initializer_range=self.initializer_range ,   encoder_stride=self.encoder_stride ,   out_features=self.out_features ,   out_indices=self.out_indices ,   )

					def     lowerCamelCase__	(self       :		Any ,   _UpperCAmelCase       :		Optional[int] ,   _UpperCAmelCase       :		Tuple ,   _UpperCAmelCase       :		str						)		->  Dict:




							"""simple docstring"""


							lowercase__	     =						FocalNetModel(config=_UpperCAmelCase						)
							model.to(_UpperCAmelCase						)
							model.eval()
							lowercase__	     =						model(_UpperCAmelCase						)

							lowercase__	     =						((config.image_size // config.patch_size) ** 2) // (4 ** (len(config.depths						) - 1))
							lowercase__	     =						int(config.embed_dim * 2 ** (len(config.depths						) - 1)						)

							self.parent.assertEqual(result.last_hidden_state.shape ,   (self.batch_size, expected_seq_len, expected_dim)						)

					def     lowerCamelCase__	(self       :		str ,   _UpperCAmelCase       :		Any ,   _UpperCAmelCase       :		List[str] ,   _UpperCAmelCase       :		List[str]						)		->  Optional[Any]:




							"""simple docstring"""


							lowercase__	     =						FocalNetBackbone(config=_UpperCAmelCase						)
							model.to(_UpperCAmelCase						)
							model.eval()
							lowercase__	     =						model(_UpperCAmelCase						)

							# verify feature maps
							self.parent.assertEqual(len(result.feature_maps						) ,   len(config.out_features						)						)
							self.parent.assertListEqual(list(result.feature_maps[0].shape						) ,   [self.batch_size, self.image_size, 8, 8]						)

							# verify channels
							self.parent.assertEqual(len(model.channels						) ,   len(config.out_features						)						)
							self.parent.assertListEqual(model.channels ,   config.hidden_sizes[:-1]						)

							# verify backbone works with out_features=None
							lowercase__	     =						None
							lowercase__	     =						FocalNetBackbone(config=_UpperCAmelCase						)
							model.to(_UpperCAmelCase						)
							model.eval()
							lowercase__	     =						model(_UpperCAmelCase						)

							# verify feature maps
							self.parent.assertEqual(len(result.feature_maps						) ,   1						)
							self.parent.assertListEqual(list(result.feature_maps[0].shape						) ,   [self.batch_size, self.image_size * 2, 4, 4]						)

							# verify channels
							self.parent.assertEqual(len(model.channels						) ,   1						)
							self.parent.assertListEqual(model.channels ,   [config.hidden_sizes[-1]]						)

					def     lowerCamelCase__	(self       :		Any ,   _UpperCAmelCase       :		Optional[int] ,   _UpperCAmelCase       :		Any ,   _UpperCAmelCase       :		str						)		->  str:




							"""simple docstring"""


							lowercase__	     =						FocalNetForMaskedImageModeling(config=_UpperCAmelCase						)
							model.to(_UpperCAmelCase						)
							model.eval()
							lowercase__	     =						model(_UpperCAmelCase						)
							self.parent.assertEqual(
							    result.reconstruction.shape ,   (self.batch_size, self.num_channels, self.image_size, self.image_size)						)

							# test greyscale images
							lowercase__	     =						1
							lowercase__	     =						FocalNetForMaskedImageModeling(_UpperCAmelCase						)
							model.to(_UpperCAmelCase						)
							model.eval()

							lowercase__	     =						floats_tensor([self.batch_size, 1, self.image_size, self.image_size]						)
							lowercase__	     =						model(_UpperCAmelCase						)
							self.parent.assertEqual(result.reconstruction.shape ,   (self.batch_size, 1, self.image_size, self.image_size)						)

					def     lowerCamelCase__	(self       :		str ,   _UpperCAmelCase       :		str ,   _UpperCAmelCase       :		List[Any] ,   _UpperCAmelCase       :		int						)		->  List[str]:




							"""simple docstring"""


							lowercase__	     =						self.type_sequence_label_size
							lowercase__	     =						FocalNetForImageClassification(_UpperCAmelCase						)
							model.to(_UpperCAmelCase						)
							model.eval()
							lowercase__	     =						model(_UpperCAmelCase ,   labels=_UpperCAmelCase						)
							self.parent.assertEqual(result.logits.shape ,   (self.batch_size, self.type_sequence_label_size)						)

							# test greyscale images
							lowercase__	     =						1
							lowercase__	     =						FocalNetForImageClassification(_UpperCAmelCase						)
							model.to(_UpperCAmelCase						)
							model.eval()

							lowercase__	     =						floats_tensor([self.batch_size, 1, self.image_size, self.image_size]						)
							lowercase__	     =						model(_UpperCAmelCase						)
							self.parent.assertEqual(result.logits.shape ,   (self.batch_size, self.type_sequence_label_size)						)


					def     lowerCamelCase__	(self       :		Optional[int]						)		->  Optional[int]:




							"""simple docstring"""


							lowercase__	     =						self.prepare_config_and_inputs()

							lowercase__     ,      lowercase__     ,      lowercase__	     =						config_and_inputs
							lowercase__	     =						{"""pixel_values""": pixel_values}
							return config, inputs_dict






@require_torch
class 				A       (			UpperCAmelCase__					,    UpperCAmelCase__					,    unittest.TestCase				):
					'''simple docstring'''






					A__     =					(
					    (
					        FocalNetModel,
					        FocalNetForImageClassification,
					        FocalNetForMaskedImageModeling,
					        FocalNetBackbone,
					    )
					    if is_torch_available()
					    else ()
					)
					A__     =					(
					    {'''feature-extraction''': FocalNetModel, '''image-classification''': FocalNetForImageClassification}
					    if is_torch_available()
					    else {}
					)
					A__     =					False

					A__     =					False
					A__     =					False
					A__     =					False
					A__     =					False

					def     lowerCamelCase__	(self       :		str						)		->  Tuple:




							"""simple docstring"""


							lowercase__	     =						FocalNetModelTester(self						)
							lowercase__	     =						ConfigTester(self ,   config_class=_UpperCAmelCase ,   embed_dim=37 ,   has_text_modality=_UpperCAmelCase						)

					def     lowerCamelCase__	(self       :		List[str]						)		->  Optional[int]:




							"""simple docstring"""


							self.create_and_test_config_common_properties()
							self.config_tester.create_and_test_config_to_json_string()
							self.config_tester.create_and_test_config_to_json_file()
							self.config_tester.create_and_test_config_from_and_save_pretrained()
							self.config_tester.create_and_test_config_with_num_labels()
							self.config_tester.check_config_can_be_init_without_params()
							self.config_tester.check_config_arguments_init()

					def     lowerCamelCase__	(self       :		Optional[Any]						)		->  Optional[Any]:




							"""simple docstring"""


							return

					def     lowerCamelCase__	(self       :		List[Any]						)		->  Optional[int]:




							"""simple docstring"""


							lowercase__	     =						self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_model(*_UpperCAmelCase						)

					def     lowerCamelCase__	(self       :		Optional[Any]						)		->  List[str]:




							"""simple docstring"""


							lowercase__	     =						self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_backbone(*_UpperCAmelCase						)

					def     lowerCamelCase__	(self       :		Dict						)		->  int:




							"""simple docstring"""


							lowercase__	     =						self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_for_masked_image_modeling(*_UpperCAmelCase						)

					def     lowerCamelCase__	(self       :		int						)		->  Optional[Any]:




							"""simple docstring"""


							lowercase__	     =						self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_for_image_classification(*_UpperCAmelCase						)

					@unittest.skip(reason="""FocalNet does not use inputs_embeds"""						)
					def     lowerCamelCase__	(self       :		Union[str, Any]						)		->  Any:




							"""simple docstring"""


							pass

					@unittest.skip(reason="""FocalNet does not use feedforward chunking"""						)
					def     lowerCamelCase__	(self       :		Union[str, Any]						)		->  Any:




							"""simple docstring"""


							pass

					def     lowerCamelCase__	(self       :		Optional[int]						)		->  Tuple:




							"""simple docstring"""


							lowercase__     ,      lowercase__	     =						self.model_tester.prepare_config_and_inputs_for_common()

							for model_class in self.all_model_classes[:-1]:
									lowercase__	     =						model_class(_UpperCAmelCase						)
									self.assertIsInstance(model.get_input_embeddings() ,   (nn.Module)						)
									lowercase__	     =						model.get_output_embeddings()
									self.assertTrue(x is None or isinstance(_UpperCAmelCase ,   nn.Linear						)						)

					def     lowerCamelCase__	(self       :		Any						)		->  List[Any]:




							"""simple docstring"""


							lowercase__     ,      lowercase__	     =						self.model_tester.prepare_config_and_inputs_for_common()

							for model_class in self.all_model_classes[:-1]:
									lowercase__	     =						model_class(_UpperCAmelCase						)
									lowercase__	     =						inspect.signature(model.forward						)
									# signature.parameters is an OrderedDict => so arg_names order is deterministic
									lowercase__	     =						[*signature.parameters.keys()]

									lowercase__	     =						["""pixel_values"""]
									self.assertListEqual(arg_names[:1] ,   _UpperCAmelCase						)

					def     lowerCamelCase__	(self       :		Dict ,   _UpperCAmelCase       :		Union[str, Any] ,   _UpperCAmelCase       :		List[str] ,   _UpperCAmelCase       :		Union[str, Any] ,   _UpperCAmelCase       :		List[str]						)		->  Any:




							"""simple docstring"""


							lowercase__	     =						model_class(_UpperCAmelCase						)
							model.to(_UpperCAmelCase						)
							model.eval()

							with torch.no_grad():
									lowercase__	     =						model(**self._prepare_for_class(_UpperCAmelCase ,   _UpperCAmelCase						)						)

							lowercase__	     =						outputs.hidden_states

							lowercase__	     =						getattr(
							    self.model_tester ,   """expected_num_hidden_layers""" ,   len(self.model_tester.depths						) + 1						)
							self.assertEqual(len(_UpperCAmelCase						) ,   _UpperCAmelCase						)

							# FocalNet has a different seq_length
							lowercase__	     =						(
							    config.patch_size
							    if isinstance(config.patch_size ,   collections.abc.Iterable						)
							    else (config.patch_size, config.patch_size)
							)

							lowercase__	     =						(image_size[1] // patch_size[1]) * (image_size[0] // patch_size[0])

							self.assertListEqual(
							    list(hidden_states[0].shape[-2:]						) ,   [num_patches, self.model_tester.embed_dim] ,   )

							lowercase__	     =						outputs.reshaped_hidden_states
							self.assertEqual(len(_UpperCAmelCase						) ,   _UpperCAmelCase						)

							lowercase__     ,      lowercase__     ,      lowercase__     ,      lowercase__	     =						reshaped_hidden_states[0].shape
							lowercase__	     =						(
							    reshaped_hidden_states[0].view(_UpperCAmelCase ,   _UpperCAmelCase ,   height * width						).permute(0 ,   2 ,   1						)
							)
							self.assertListEqual(
							    list(reshaped_hidden_states.shape[-2:]						) ,   [num_patches, self.model_tester.embed_dim] ,   )

					def     lowerCamelCase__	(self       :		Tuple						)		->  List[Any]:




							"""simple docstring"""


							lowercase__     ,      lowercase__	     =						self.model_tester.prepare_config_and_inputs_for_common()

							lowercase__	     =						(
							    self.model_tester.image_size
							    if isinstance(self.model_tester.image_size ,   collections.abc.Iterable						)
							    else (self.model_tester.image_size, self.model_tester.image_size)
							)

							for model_class in self.all_model_classes[:-1]:
									lowercase__	     =						True
									self.check_hidden_states_output(_UpperCAmelCase ,   _UpperCAmelCase ,   _UpperCAmelCase ,   _UpperCAmelCase						)

									# check that output_hidden_states also work using config
									del inputs_dict["output_hidden_states"]
									lowercase__	     =						True

									self.check_hidden_states_output(_UpperCAmelCase ,   _UpperCAmelCase ,   _UpperCAmelCase ,   _UpperCAmelCase						)

					def     lowerCamelCase__	(self       :		Tuple						)		->  Optional[Any]:




							"""simple docstring"""


							lowercase__     ,      lowercase__	     =						self.model_tester.prepare_config_and_inputs_for_common()
							lowercase__	     =						3

							lowercase__	     =						(
							    self.model_tester.image_size
							    if isinstance(self.model_tester.image_size ,   collections.abc.Iterable						)
							    else (self.model_tester.image_size, self.model_tester.image_size)
							)
							lowercase__	     =						(
							    config.patch_size
							    if isinstance(config.patch_size ,   collections.abc.Iterable						)
							    else (config.patch_size, config.patch_size)
							)

							lowercase__	     =						image_size[0] + patch_size[0] - (image_size[0] % patch_size[0])
							lowercase__	     =						image_size[1] + patch_size[1] - (image_size[1] % patch_size[1])

							for model_class in self.all_model_classes[:-1]:
									lowercase__	     =						True
									self.check_hidden_states_output(_UpperCAmelCase ,   _UpperCAmelCase ,   _UpperCAmelCase ,   (padded_height, padded_width)						)

									# check that output_hidden_states also work using config
									del inputs_dict["output_hidden_states"]
									lowercase__	     =						True
									self.check_hidden_states_output(_UpperCAmelCase ,   _UpperCAmelCase ,   _UpperCAmelCase ,   (padded_height, padded_width)						)

					@slow
					def     lowerCamelCase__	(self       :		List[Any]						)		->  List[str]:




							"""simple docstring"""


							for model_name in FOCALNET_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
									lowercase__	     =						FocalNetModel.from_pretrained(_UpperCAmelCase						)
									self.assertIsNotNone(_UpperCAmelCase						)


					def     lowerCamelCase__	(self       :		int						)		->  Optional[Any]:




							"""simple docstring"""


							lowercase__     ,      lowercase__	     =						self.model_tester.prepare_config_and_inputs_for_common()

							lowercase__	     =						_config_zero_init(_UpperCAmelCase						)
							for model_class in self.all_model_classes:
									lowercase__	     =						model_class(config=_UpperCAmelCase						)
									for name, param in model.named_parameters():
											if "embeddings" not in name and param.requires_grad:
													self.assertIn(
													    ((param.data.mean() * 1E9).round() / 1E9).item() ,   [0.0, 1.0] ,   msg=f'''Parameter {name} of model {model_class} seems not properly initialized''' ,   )






@require_vision
@require_torch
class 				A       (			unittest.TestCase				):
					'''simple docstring'''






					@cached_property
					def     lowerCamelCase__	(self       :		str						)		->  Optional[Any]:




							"""simple docstring"""


							return AutoImageProcessor.from_pretrained("""microsoft/focalnet-tiny"""						) if is_vision_available() else None


					@slow
					def     lowerCamelCase__	(self       :		Optional[Any]						)		->  List[Any]:




							"""simple docstring"""


							lowercase__	     =						FocalNetForImageClassification.from_pretrained("""microsoft/focalnet-tiny"""						).to(_UpperCAmelCase						)
							lowercase__	     =						self.default_image_processor

							lowercase__	     =						Image.open("""./tests/fixtures/tests_samples/COCO/000000039769.png"""						)
							lowercase__	     =						image_processor(images=_UpperCAmelCase ,   return_tensors="""pt"""						).to(_UpperCAmelCase						)

							# forward pass
							with torch.no_grad():
									lowercase__	     =						model(**_UpperCAmelCase						)

							# verify the logits
							lowercase__	     =						torch.Size((1, 1000)						)
							self.assertEqual(outputs.logits.shape ,   _UpperCAmelCase						)
							lowercase__	     =						torch.tensor([0.2_166, -0.4_368, 0.2_191]						).to(_UpperCAmelCase						)
							self.assertTrue(torch.allclose(outputs.logits[0, :3] ,   _UpperCAmelCase ,   atol=1E-4						)						)
							self.assertTrue(outputs.logits.argmax(dim=-1						).item() ,   281						)






@require_torch
class 				A       (			UpperCAmelCase__					,    unittest.TestCase				):
					'''simple docstring'''






					A__     =					(FocalNetBackbone,) if is_torch_available() else ()
					A__     =					FocalNetConfig

					A__     =					False

					def     lowerCamelCase__	(self       :		Any						)		->  int:




							"""simple docstring"""


							lowercase__	     =						FocalNetModelTester(self						)





import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
	from .tokenization_pegasus import PegasusTokenizer
else:
	A			:						List[Any]        =  None


A			:						Optional[Any]        =  logging.get_logger(__name__)


A			:						str        =  '▁'

A			:						Optional[Any]        =  {'vocab_file': 'spiece.model', 'tokenizer_file': 'tokenizer.json'}

A			:						Any        =  {
    'vocab_file': {'google/pegasus-xsum': 'https://huggingface.co/google/pegasus-xsum/resolve/main/spiece.model'},
    'tokenizer_file': {
        'google/pegasus-xsum': 'https://huggingface.co/google/pegasus-xsum/resolve/main/tokenizer.json'
    },
}

A			:						Optional[int]        =  {
    'google/pegasus-xsum': 5_1_2,
}



class 				A       (			UpperCAmelCase__				):
					'''simple docstring'''






					A__     =					VOCAB_FILES_NAMES
					A__     =					PRETRAINED_VOCAB_FILES_MAP
					A__     =					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					A__     =					PegasusTokenizer
					A__     =					['''input_ids''', '''attention_mask''']

					def __init__(self       :		Tuple ,   _UpperCAmelCase       :		Union[str, Any]=None ,   _UpperCAmelCase       :		List[Any]=None ,   _UpperCAmelCase       :		Union[str, Any]="<pad>" ,   _UpperCAmelCase       :		int="</s>" ,   _UpperCAmelCase       :		int="<unk>" ,   _UpperCAmelCase       :		Optional[Any]="<mask_2>" ,   _UpperCAmelCase       :		Optional[Any]="<mask_1>" ,   _UpperCAmelCase       :		Optional[Any]=None ,   _UpperCAmelCase       :		Dict=103 ,   **_UpperCAmelCase       :		Dict ,   )		->  Dict:




							"""simple docstring"""


							lowercase__	     =						offset

							if additional_special_tokens is not None:
									if not isinstance(_UpperCAmelCase ,   _UpperCAmelCase						):
											raise TypeError(
											    f'''additional_special_tokens should be of type {type(_UpperCAmelCase						)}, but is'''
											    f''' {type(_UpperCAmelCase						)}'''						)

									lowercase__	     =						(
									    ([mask_token_sent] + additional_special_tokens)
									    if mask_token_sent not in additional_special_tokens and mask_token_sent is not None
									    else additional_special_tokens
									)
									# fill additional tokens with ..., <unk_token_102> in case not all additional tokens are already taken
									additional_special_tokens_extended += [
									    f'''<unk_{i}>''' for i in range(len(_UpperCAmelCase						) ,   self.offset - 1						)
									]

									if len(set(_UpperCAmelCase						)						) != len(_UpperCAmelCase						):
											raise ValueError(
											    """Please make sure that the provided additional_special_tokens do not contain an incorrectly"""
											    f''' shifted list of <unk_x> tokens. Found {additional_special_tokens_extended}.'''						)
									lowercase__	     =						additional_special_tokens_extended
							else:
									lowercase__	     =						[mask_token_sent] if mask_token_sent is not None else []
									additional_special_tokens += [f'''<unk_{i}>''' for i in range(2 ,   self.offset						)]

							super().__init__(
							    _UpperCAmelCase ,   tokenizer_file=_UpperCAmelCase ,   pad_token=_UpperCAmelCase ,   eos_token=_UpperCAmelCase ,   unk_token=_UpperCAmelCase ,   mask_token=_UpperCAmelCase ,   mask_token_sent=_UpperCAmelCase ,   offset=_UpperCAmelCase ,   additional_special_tokens=_UpperCAmelCase ,   **_UpperCAmelCase ,   )
							lowercase__	     =						vocab_file
							lowercase__	     =						False if not self.vocab_file else True

					def     lowerCamelCase__	(self       :		Optional[Any] ,   _UpperCAmelCase       :		Tuple						)		->  Union[str, Any]:




							"""simple docstring"""


							lowercase__	     =						set(self.all_special_ids						)  # call it once instead of inside list comp
							all_special_ids.remove(self.unk_token_id						)  # <unk> is only sometimes special

							if all_special_ids != set(range(len(self.additional_special_tokens						) + 3						)						):
									raise ValueError(
									    """There should be 3 special tokens: mask_token, pad_token, and eos_token +"""
									    f''' {len(self.additional_special_tokens						)} additional_special_tokens, but got {all_special_ids}'''						)

							return [1 if x in all_special_ids else 0 for x in seq]

					def     lowerCamelCase__	(self       :		Optional[Any] ,   _UpperCAmelCase       :		List ,   _UpperCAmelCase       :		Optional[List] = None ,   _UpperCAmelCase       :		bool = False						)		->  List[int]:




							"""simple docstring"""


							if already_has_special_tokens:
									return self._special_token_mask(_UpperCAmelCase						)
							elif token_ids_a is None:
									return self._special_token_mask(_UpperCAmelCase						) + [1]
							else:
									return self._special_token_mask(token_ids_a + token_ids_a						) + [1]

					def     lowerCamelCase__	(self       :		int ,   _UpperCAmelCase       :		Optional[int] ,   _UpperCAmelCase       :		Union[str, Any]=None						)		->  List[int]:




							"""simple docstring"""


							if token_ids_a is None:
									return token_ids_a + [self.eos_token_id]
							# We don't expect to process pairs, but leave the pair logic for API consistency
							return token_ids_a + token_ids_a + [self.eos_token_id]


					def     lowerCamelCase__	(self       :		str ,   _UpperCAmelCase       :		str ,   _UpperCAmelCase       :		Optional[str] = None						)		->  Tuple[str]:




							"""simple docstring"""


							if not self.can_save_slow_tokenizer:
									raise ValueError(
									    """Your fast tokenizer does not have the necessary information to save the vocabulary for a slow """
									    """tokenizer."""						)

							if not os.path.isdir(_UpperCAmelCase						):
									logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''						)
									return
							lowercase__	     =						os.path.join(
							    _UpperCAmelCase ,   (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]						)

							if os.path.abspath(self.vocab_file						) != os.path.abspath(_UpperCAmelCase						):
									copyfile(self.vocab_file ,   _UpperCAmelCase						)

							return (out_vocab_file,)

