






import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, BatchEncoding, PreTrainedTokenizer
from ...utils import logging


__UpperCAmelCase      				=		logging.get_logger(__name__)

__UpperCAmelCase      				=		'▁'

__UpperCAmelCase      				=		{'vocab_file': 'sentencepiece.bpe.model'}

__UpperCAmelCase      				=		{
    'vocab_file': {
        'facebook/mbart-large-50-one-to-many-mmt': (
            'https://huggingface.co/facebook/mbart-large-50-one-to-many-mmt/resolve/main/sentencepiece.bpe.model'
        ),
    }
}

__UpperCAmelCase      				=		{
    'facebook/mbart-large-50-one-to-many-mmt': 1024,
}

# fmt: off
__UpperCAmelCase      				=		['ar_AR', 'cs_CZ', 'de_DE', 'en_XX', 'es_XX', 'et_EE', 'fi_FI', 'fr_XX', 'gu_IN', 'hi_IN', 'it_IT', 'ja_XX', 'kk_KZ', 'ko_KR', 'lt_LT', 'lv_LV', 'my_MM', 'ne_NP', 'nl_XX', 'ro_RO', 'ru_RU', 'si_LK', 'tr_TR', 'vi_VN', 'zh_CN', 'af_ZA', 'az_AZ', 'bn_IN', 'fa_IR', 'he_IL', 'hr_HR', 'id_ID', 'ka_GE', 'km_KH', 'mk_MK', 'ml_IN', 'mn_MN', 'mr_IN', 'pl_PL', 'ps_AF', 'pt_XX', 'sv_SE', 'sw_KE', 'ta_IN', 'te_IN', 'th_TH', 'tl_XX', 'uk_UA', 'ur_PK', 'xh_ZA', 'gl_ES', 'sl_SI']



class     lowerCamelCase (__lowercase     ):







		'''simple docstring'''



		_snake_case       :       Optional[int]					=    VOCAB_FILES_NAMES
		_snake_case       :       Union[str, Any]					=    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
		_snake_case       :       Union[str, Any]					=    PRETRAINED_VOCAB_FILES_MAP
		_snake_case       :       Union[str, Any]					=    ['''input_ids''', '''attention_mask''']

		_snake_case       :       Optional[int]					=    []
		_snake_case       :       List[str]					=    []



		def __init__(				self							,						_UpperCamelCase							,						_UpperCamelCase=None							,						_UpperCamelCase=None							,						_UpperCamelCase="</s>"							,						_UpperCamelCase="</s>"							,						_UpperCamelCase="<s>"							,						_UpperCamelCase="<unk>"							,						_UpperCamelCase="<pad>"							,						_UpperCamelCase="<mask>"							,						_UpperCamelCase = None							,						**_UpperCamelCase							,						)      ->				None:
							# Mask token behave like a normal word, i.e. include the space before it
							UpperCAmelCase_       : Dict				=		AddedToken(_UpperCamelCase							,						lstrip=_UpperCamelCase							,						rstrip=_UpperCamelCase  ) if isinstance(_UpperCamelCase							,						_UpperCamelCase  ) else mask_token

							UpperCAmelCase_       : Dict				=		{} if sp_model_kwargs is None else sp_model_kwargs

							UpperCAmelCase_       : Any				=		kwargs.get('additional_special_tokens'							,						[]  )
							kwargs["additional_special_tokens"] += [
							    code for code in FAIRSEQ_LANGUAGE_CODES if code not in kwargs["additional_special_tokens"]
							]

							super().__init__(
							    src_lang=_UpperCamelCase							,						tgt_lang=_UpperCamelCase							,						eos_token=_UpperCamelCase							,						unk_token=_UpperCamelCase							,						sep_token=_UpperCamelCase							,						cls_token=_UpperCamelCase							,						pad_token=_UpperCamelCase							,						mask_token=_UpperCamelCase							,						sp_model_kwargs=self.sp_model_kwargs							,						**_UpperCamelCase							,						)

							UpperCAmelCase_       : List[str]				=		spm.SentencePieceProcessor(**self.sp_model_kwargs  )
							self.sp_model.Load(str(_UpperCamelCase  )  )
							UpperCAmelCase_       : Union[str, Any]				=		vocab_file

							# Original fairseq vocab and spm vocab must be "aligned":
							# Vocab    |    0    |    1    |   2    |    3    |  4  |  5  |  6  |   7   |   8   |  9
							# -------- | ------- | ------- | ------ | ------- | --- | --- | --- | ----- | ----- | ----
							# fairseq  | '<s>'   | '<pad>' | '</s>' | '<unk>' | ',' | '.' | '▁' | 's'   | '▁de' | '-'
							# spm      | '<unk>' | '<s>'   | '</s>' | ','     | '.' | '▁' | 's' | '▁de' | '-'   | '▁a'

							# Mimic fairseq token-to-id alignment for the first 4 token
							UpperCAmelCase_       : List[Any]				=		{"<s>": 0, "<pad>": 1, "</s>": 2, "<unk>": 3}

							# The first "real" token "," has position 4 in the original fairseq vocab and position 3 in the spm vocab
							UpperCAmelCase_       : Union[str, Any]				=		1

							UpperCAmelCase_       : Any				=		len(self.sp_model  )
							UpperCAmelCase_       : Dict				=		{
							    code: self.sp_model_size + i + self.fairseq_offset for i, code in enumerate(_UpperCamelCase  )
							}
							UpperCAmelCase_       : Union[str, Any]				=		{v: k for k, v in self.lang_code_to_id.items()}
							UpperCAmelCase_       : Tuple				=		len(self.sp_model  ) + len(self.lang_code_to_id  ) + self.fairseq_offset

							self.fairseq_tokens_to_ids.update(self.lang_code_to_id  )
							UpperCAmelCase_       : int				=		{v: k for k, v in self.fairseq_tokens_to_ids.items()}

							UpperCAmelCase_       : List[Any]				=		src_lang if src_lang is not None else "en_XX"
							UpperCAmelCase_       : str				=		self.lang_code_to_id[self._src_lang]
							UpperCAmelCase_       : Optional[Any]				=		tgt_lang
							self.set_src_lang_special_tokens(self._src_lang  )



		@property
		def    __UpperCAmelCase     (				self  )      ->				int:
							return len(self.sp_model  ) + len(self.lang_code_to_id  ) + self.fairseq_offset + 1  # Plus 1 for the mask token



		@property
		def    __UpperCAmelCase     (				self  )      ->				str:
							return self._src_lang



		@src_lang.setter
		def    __UpperCAmelCase     (				self							,						_UpperCamelCase  )      ->				None:
							UpperCAmelCase_       : Any				=		new_src_lang
							self.set_src_lang_special_tokens(self._src_lang  )



		def __getstate__(				self  )      ->				Dict:
							UpperCAmelCase_       : Optional[int]				=		self.__dict__.copy()
							UpperCAmelCase_       : List[str]				=		None
							return state



		def __setstate__(				self							,						_UpperCamelCase  )      ->				None:
							UpperCAmelCase_       : Tuple				=		d

							# for backward compatibility
							if not hasattr(self							,						'sp_model_kwargs'  ):
												UpperCAmelCase_       : Dict				=		{}

							UpperCAmelCase_       : Union[str, Any]				=		spm.SentencePieceProcessor(**self.sp_model_kwargs  )
							self.sp_model.Load(self.vocab_file  )



		def    __UpperCAmelCase     (				self  )      ->				Dict:
							UpperCAmelCase_       : str				=		{self.convert_ids_to_tokens(_UpperCamelCase  ): i for i in range(self.vocab_size  )}
							vocab.update(self.added_tokens_encoder  )
							return vocab



		def    __UpperCAmelCase     (				self							,						_UpperCamelCase  )      ->				List[str]:
							return self.sp_model.encode(_UpperCamelCase							,						out_type=_UpperCamelCase  )



		def    __UpperCAmelCase     (				self							,						_UpperCamelCase  )      ->				int:
							if token in self.fairseq_tokens_to_ids:
												return self.fairseq_tokens_to_ids[token]
							UpperCAmelCase_       : Any				=		self.sp_model.PieceToId(_UpperCamelCase  )

							# Need to return unknown token if the SP model returned 0
							return spm_id + self.fairseq_offset if spm_id else self.unk_token_id



		def    __UpperCAmelCase     (				self							,						_UpperCamelCase  )      ->				str:
							if index in self.fairseq_ids_to_tokens:
												return self.fairseq_ids_to_tokens[index]
							return self.sp_model.IdToPiece(index - self.fairseq_offset  )



		def    __UpperCAmelCase     (				self							,						_UpperCamelCase  )      ->				Tuple:
							UpperCAmelCase_       : Tuple				=		[]
							UpperCAmelCase_       : int				=		""
							UpperCAmelCase_       : Tuple				=		False
							for token in tokens:
												# make sure that special tokens are not decoded using sentencepiece model
												if token in self.all_special_tokens:
																	if not prev_is_special:
																						out_string += " "
																	out_string += self.sp_model.decode(_UpperCamelCase  ) + token
																	UpperCAmelCase_       : str				=		True
																	UpperCAmelCase_       : Any				=		[]
												else:
																	current_sub_tokens.append(_UpperCamelCase  )
																	UpperCAmelCase_       : List[str]				=		False
							out_string += self.sp_model.decode(_UpperCamelCase  )
							return out_string.strip()



		def    __UpperCAmelCase     (				self							,						_UpperCamelCase							,						_UpperCamelCase = None  )      ->				Tuple[str]:
							if not os.path.isdir(_UpperCamelCase  ):
												logger.error(f"Vocabulary path ({save_directory}) should be a directory"  )
												return
							UpperCAmelCase_       : Dict				=		os.path.join(
							    _UpperCamelCase							,						(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']  )

							if os.path.abspath(self.vocab_file  ) != os.path.abspath(_UpperCamelCase  ) and os.path.isfile(self.vocab_file  ):
												copyfile(self.vocab_file							,						_UpperCamelCase  )
							elif not os.path.isfile(self.vocab_file  ):
												with open(_UpperCamelCase							,						'wb'  ) as fi:
																	UpperCAmelCase_       : List[Any]				=		self.sp_model.serialized_model_proto()
																	fi.write(_UpperCamelCase  )

							return (out_vocab_file,)



		def    __UpperCAmelCase     (				self							,						_UpperCamelCase							,						_UpperCamelCase = None							,						_UpperCamelCase = False  )      ->				List[int]:

							if already_has_special_tokens:
												return super().get_special_tokens_mask(
												    token_ids_a=_UpperCamelCase							,						token_ids_a=_UpperCamelCase							,						already_has_special_tokens=_UpperCamelCase  )

							UpperCAmelCase_       : Optional[Any]				=		[1] * len(self.prefix_tokens  )
							UpperCAmelCase_       : Optional[int]				=		[1] * len(self.suffix_tokens  )
							if token_ids_a is None:
												return prefix_ones + ([0] * len(_UpperCamelCase  )) + suffix_ones
							return prefix_ones + ([0] * len(_UpperCamelCase  )) + ([0] * len(_UpperCamelCase  )) + suffix_ones



		def    __UpperCAmelCase     (				self							,						_UpperCamelCase							,						_UpperCamelCase = None  )      ->				List[int]:
							if token_ids_a is None:
												return self.prefix_tokens + token_ids_a + self.suffix_tokens
							# We don't expect to process pairs, but leave the pair logic for API consistency
							return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens



		def    __UpperCAmelCase     (				self							,						_UpperCamelCase							,						_UpperCamelCase							,						_UpperCamelCase							,						_UpperCamelCase							,						**_UpperCamelCase  )      ->				str:
							if src_lang is None or tgt_lang is None:
												raise ValueError('Translation requires a `src_lang` and a `tgt_lang` for this model'  )
							UpperCAmelCase_       : str				=		src_lang
							UpperCAmelCase_       : Dict				=		self(_UpperCamelCase							,						add_special_tokens=_UpperCamelCase							,						return_tensors=_UpperCamelCase							,						**_UpperCamelCase  )
							UpperCAmelCase_       : Union[str, Any]				=		self.convert_tokens_to_ids(_UpperCamelCase  )
							UpperCAmelCase_       : Tuple				=		tgt_lang_id
							return inputs



		def    __UpperCAmelCase     (				self							,						_UpperCamelCase							,						_UpperCamelCase = "en_XX"							,						_UpperCamelCase = None							,						_UpperCamelCase = "ro_RO"							,						**_UpperCamelCase							,						)      ->				BatchEncoding:
							UpperCAmelCase_       : Optional[Any]				=		src_lang
							UpperCAmelCase_       : int				=		tgt_lang
							return super().prepare_seqaseq_batch(_UpperCamelCase							,						_UpperCamelCase							,						**_UpperCamelCase  )



		def    __UpperCAmelCase     (				self  )      ->				List[str]:
							return self.set_src_lang_special_tokens(self.src_lang  )



		def    __UpperCAmelCase     (				self  )      ->				Any:
							return self.set_tgt_lang_special_tokens(self.tgt_lang  )



		def    __UpperCAmelCase     (				self							,						_UpperCamelCase  )      ->				None:
							UpperCAmelCase_       : Optional[Any]				=		self.lang_code_to_id[src_lang]
							UpperCAmelCase_       : List[Any]				=		[self.cur_lang_code_id]
							UpperCAmelCase_       : Optional[Any]				=		[self.eos_token_id]



		def    __UpperCAmelCase     (				self							,						_UpperCamelCase  )      ->				None:
							UpperCAmelCase_       : List[Any]				=		self.lang_code_to_id[tgt_lang]
							UpperCAmelCase_       : Any				=		[self.cur_lang_code_id]
							UpperCAmelCase_       : Any				=		[self.eos_token_id]








from typing import List

from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowercase    :   Union[str, Any]						=			logging.get_logger(__name__)

lowercase    :   Union[str, Any]						=			{
    'snap-research/efficientformer-l1-300': (
        'https://huggingface.co/snap-research/efficientformer-l1-300/resolve/main/config.json'
    ),
}




class 					lowerCamelCase__				(							__lowercase):




	'''simple docstring'''




	_A								=	'efficientformer'







	def __init__(   self						:Union[str, Any]	,  a						:List[int] = [3, 2, 6, 4]	,  a						:List[int] = [4_8, 9_6, 2_2_4, 4_4_8]	,  a						:List[bool] = [True, True, True, True]	,  a						:int = 4_4_8	,  a						:int = 3_2	,  a						:int = 4	,  a						:int = 7	,  a						:int = 5	,  a						:int = 8	,  a						:int = 4	,  a						:float = 0.0	,  a						:int = 1_6	,  a						:int = 3	,  a						:int = 3	,  a						:int = 3	,  a						:int = 2	,  a						:int = 1	,  a						:float = 0.0	,  a						:int = 1	,  a						:bool = True	,  a						:bool = True	,  a						:float = 1E-5	,  a						:str = "gelu"	,  a						:float = 0.02	,  a						:float = 1E-1_2	,  a						:int = 2_2_4	,  a						:float = 1E-0_5	,  **a						:str	,  )						->					None:
								super().__init__(**a		)

								__UpperCamelCase      :  Optional[Any]       =   hidden_act
								__UpperCamelCase      :  Any       =   hidden_dropout_prob
								__UpperCamelCase      :  str       =   hidden_sizes
								__UpperCamelCase      :  Dict       =   num_hidden_layers
								__UpperCamelCase      :  Optional[Any]       =   num_attention_heads
								__UpperCamelCase      :  str       =   initializer_range
								__UpperCamelCase      :  List[str]       =   layer_norm_eps
								__UpperCamelCase      :  str       =   patch_size
								__UpperCamelCase      :  str       =   num_channels
								__UpperCamelCase      :  Any       =   depths
								__UpperCamelCase      :  Tuple       =   mlp_expansion_ratio
								__UpperCamelCase      :  List[str]       =   downsamples
								__UpperCamelCase      :  Optional[int]       =   dim
								__UpperCamelCase      :  Dict       =   key_dim
								__UpperCamelCase      :  Optional[Any]       =   attention_ratio
								__UpperCamelCase      :  Dict       =   resolution
								__UpperCamelCase      :  Union[str, Any]       =   pool_size
								__UpperCamelCase      :  Tuple       =   downsample_patch_size
								__UpperCamelCase      :  Optional[int]       =   downsample_stride
								__UpperCamelCase      :  Optional[int]       =   downsample_pad
								__UpperCamelCase      :  Union[str, Any]       =   drop_path_rate
								__UpperCamelCase      :  Union[str, Any]       =   num_metaad_blocks
								__UpperCamelCase      :  List[str]       =   distillation
								__UpperCamelCase      :  str       =   use_layer_scale
								__UpperCamelCase      :  Tuple       =   layer_scale_init_value
								__UpperCamelCase      :  str       =   image_size
								__UpperCamelCase      :  int       =   batch_norm_eps