


'''simple docstring'''
import os
from shutil import copyfile
from typing import List, Optional, Tuple

from tokenizers import processors

from ...tokenization_utils import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
					from .tokenization_nllb import NllbTokenizer
else:
					lowerCAmelCase   : str             =				None


lowerCAmelCase   : Dict             =				logging.get_logger(__name__)


lowerCAmelCase   : List[str]             =				{"""vocab_file""": """sentencepiece.bpe.model""", """tokenizer_file""": """tokenizer.json"""}

lowerCAmelCase   : Union[str, Any]             =				{
    """vocab_file""": {
        """facebook/nllb-200-distilled-600M""": (
            """https://huggingface.co/facebook/nllb-200-distilled-600M/resolve/main/sentencepiece.bpe.model"""
        ),
    },
    """tokenizer_file""": {
        """facebook/nllb-200-distilled-600M""": (
            """https://huggingface.co/facebook/nllb-200-distilled-600M/resolve/main/tokenizer.json"""
        ),
    },
}

lowerCAmelCase   : int             =				{
    """facebook/nllb-large-en-ro""": 1_0_2_4,
    """facebook/nllb-200-distilled-600M""": 1_0_2_4,
}

# fmt: off
lowerCAmelCase   : Union[str, Any]             =				["""ace_Arab""", """ace_Latn""", """acm_Arab""", """acq_Arab""", """aeb_Arab""", """afr_Latn""", """ajp_Arab""", """aka_Latn""", """amh_Ethi""", """apc_Arab""", """arb_Arab""", """ars_Arab""", """ary_Arab""", """arz_Arab""", """asm_Beng""", """ast_Latn""", """awa_Deva""", """ayr_Latn""", """azb_Arab""", """azj_Latn""", """bak_Cyrl""", """bam_Latn""", """ban_Latn""", """bel_Cyrl""", """bem_Latn""", """ben_Beng""", """bho_Deva""", """bjn_Arab""", """bjn_Latn""", """bod_Tibt""", """bos_Latn""", """bug_Latn""", """bul_Cyrl""", """cat_Latn""", """ceb_Latn""", """ces_Latn""", """cjk_Latn""", """ckb_Arab""", """crh_Latn""", """cym_Latn""", """dan_Latn""", """deu_Latn""", """dik_Latn""", """dyu_Latn""", """dzo_Tibt""", """ell_Grek""", """eng_Latn""", """epo_Latn""", """est_Latn""", """eus_Latn""", """ewe_Latn""", """fao_Latn""", """pes_Arab""", """fij_Latn""", """fin_Latn""", """fon_Latn""", """fra_Latn""", """fur_Latn""", """fuv_Latn""", """gla_Latn""", """gle_Latn""", """glg_Latn""", """grn_Latn""", """guj_Gujr""", """hat_Latn""", """hau_Latn""", """heb_Hebr""", """hin_Deva""", """hne_Deva""", """hrv_Latn""", """hun_Latn""", """hye_Armn""", """ibo_Latn""", """ilo_Latn""", """ind_Latn""", """isl_Latn""", """ita_Latn""", """jav_Latn""", """jpn_Jpan""", """kab_Latn""", """kac_Latn""", """kam_Latn""", """kan_Knda""", """kas_Arab""", """kas_Deva""", """kat_Geor""", """knc_Arab""", """knc_Latn""", """kaz_Cyrl""", """kbp_Latn""", """kea_Latn""", """khm_Khmr""", """kik_Latn""", """kin_Latn""", """kir_Cyrl""", """kmb_Latn""", """kon_Latn""", """kor_Hang""", """kmr_Latn""", """lao_Laoo""", """lvs_Latn""", """lij_Latn""", """lim_Latn""", """lin_Latn""", """lit_Latn""", """lmo_Latn""", """ltg_Latn""", """ltz_Latn""", """lua_Latn""", """lug_Latn""", """luo_Latn""", """lus_Latn""", """mag_Deva""", """mai_Deva""", """mal_Mlym""", """mar_Deva""", """min_Latn""", """mkd_Cyrl""", """plt_Latn""", """mlt_Latn""", """mni_Beng""", """khk_Cyrl""", """mos_Latn""", """mri_Latn""", """zsm_Latn""", """mya_Mymr""", """nld_Latn""", """nno_Latn""", """nob_Latn""", """npi_Deva""", """nso_Latn""", """nus_Latn""", """nya_Latn""", """oci_Latn""", """gaz_Latn""", """ory_Orya""", """pag_Latn""", """pan_Guru""", """pap_Latn""", """pol_Latn""", """por_Latn""", """prs_Arab""", """pbt_Arab""", """quy_Latn""", """ron_Latn""", """run_Latn""", """rus_Cyrl""", """sag_Latn""", """san_Deva""", """sat_Beng""", """scn_Latn""", """shn_Mymr""", """sin_Sinh""", """slk_Latn""", """slv_Latn""", """smo_Latn""", """sna_Latn""", """snd_Arab""", """som_Latn""", """sot_Latn""", """spa_Latn""", """als_Latn""", """srd_Latn""", """srp_Cyrl""", """ssw_Latn""", """sun_Latn""", """swe_Latn""", """swh_Latn""", """szl_Latn""", """tam_Taml""", """tat_Cyrl""", """tel_Telu""", """tgk_Cyrl""", """tgl_Latn""", """tha_Thai""", """tir_Ethi""", """taq_Latn""", """taq_Tfng""", """tpi_Latn""", """tsn_Latn""", """tso_Latn""", """tuk_Latn""", """tum_Latn""", """tur_Latn""", """twi_Latn""", """tzm_Tfng""", """uig_Arab""", """ukr_Cyrl""", """umb_Latn""", """urd_Arab""", """uzn_Latn""", """vec_Latn""", """vie_Latn""", """war_Latn""", """wol_Latn""", """xho_Latn""", """ydd_Hebr""", """yor_Latn""", """yue_Hant""", """zho_Hans""", """zho_Hant""", """zul_Latn"""]




class        _UpperCamelCase	(    SCREAMING_SNAKE_CASE):


				'''simple docstring'''


				_snake_case						       =				VOCAB_FILES_NAMES
				_snake_case						       =				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				_snake_case						       =				PRETRAINED_VOCAB_FILES_MAP
				_snake_case						       =				['''input_ids''', '''attention_mask''']
				_snake_case						       =				NllbTokenizer

				_snake_case						       =				[]
				_snake_case						       =				[]

				def __init__(     self							,      a_=None							,      a_=None							,      a_="<s>"							,      a_="</s>"							,      a_="</s>"							,      a_="<s>"							,      a_="<unk>"							,      a_="<pad>"							,      a_="<mask>"							,      a_=None							,      a_=None							,      a_=None							,      a_=False							,      **a_							,      )    ->  Optional[int]:
					lowercase						:		Any   					=		AddedToken(snake_case__							,      lstrip=snake_case__							,      rstrip=snake_case__							) if isinstance(snake_case__							,      snake_case__							) else mask_token
					lowercase						:		Dict   					=		legacy_behaviour
					super().__init__(
					    vocab_file=snake_case__							,      tokenizer_file=snake_case__							,      bos_token=snake_case__							,      eos_token=snake_case__							,      sep_token=snake_case__							,      cls_token=snake_case__							,      unk_token=snake_case__							,      pad_token=snake_case__							,      mask_token=snake_case__							,      src_lang=snake_case__							,      tgt_lang=snake_case__							,      additional_special_tokens=snake_case__							,      legacy_behaviour=snake_case__							,      **snake_case__							,      )

					lowercase						:		Any   					=		vocab_file
					lowercase						:		int   					=		False if not self.vocab_file else True

					lowercase						:		Optional[Any]   					=		FAIRSEQ_LANGUAGE_CODES.copy()

					if additional_special_tokens is not None:
						# Only add those special tokens if they are not already there.
						_additional_special_tokens.extend(
						    [t for t in additional_special_tokens if t not in _additional_special_tokens]							)

					self.add_special_tokens({"additional_special_tokens": _additional_special_tokens}							)
					lowercase						:		List[str]   					=		{
					    lang_code: self.convert_tokens_to_ids(snake_case__							) for lang_code in FAIRSEQ_LANGUAGE_CODES
					}

					lowercase						:		List[Any]   					=		src_lang if src_lang is not None else "eng_Latn"
					lowercase						:		List[str]   					=		self.convert_tokens_to_ids(self._src_lang							)
					lowercase						:		List[Any]   					=		tgt_lang
					self.set_src_lang_special_tokens(self._src_lang							)

				@property
				def 	a__		(     self							)    ->  Optional[int]:
					return self._src_lang

				@src_lang.setter
				def 	a__		(     self							,      a_							)    ->  int:
					lowercase						:		List[str]   					=		new_src_lang
					self.set_src_lang_special_tokens(self._src_lang							)

				def 	a__		(     self							,      a_							,      a_ = None							)    ->  Tuple:
					if token_ids_a is None:
						return self.prefix_tokens + token_ids_a + self.suffix_tokens
					# We don't expect to process pairs, but leave the pair logic for API consistency
					return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens

				def 	a__		(     self							,      a_							,      a_ = None							)    ->  Optional[int]:
					lowercase						:		Optional[Any]   					=		[self.sep_token_id]
					lowercase						:		str   					=		[self.cls_token_id]

					if token_ids_a is None:
						return len(cls + token_ids_a + sep							) * [0]
					return len(cls + token_ids_a + sep + sep + token_ids_a + sep							) * [0]

				def 	a__		(     self							,      a_							,      a_							,      a_							,      a_							,      **a_							)    ->  Tuple:
					if src_lang is None or tgt_lang is None:
						raise ValueError("Translation requires a `src_lang` and a `tgt_lang` for this model"							)
					lowercase						:		int   					=		src_lang
					lowercase						:		Dict   					=		self(snake_case__							,      add_special_tokens=snake_case__							,      return_tensors=snake_case__							,      **snake_case__							)
					lowercase						:		Tuple   					=		self.convert_tokens_to_ids(snake_case__							)
					lowercase						:		Any   					=		tgt_lang_id
					return inputs

				def 	a__		(     self							,      a_							,      a_ = "eng_Latn"							,      a_ = None							,      a_ = "fra_Latn"							,      **a_							,      )    ->  Optional[Any]:
					lowercase						:		int   					=		src_lang
					lowercase						:		Optional[Any]   					=		tgt_lang
					return super().prepare_seqaseq_batch(snake_case__							,      snake_case__							,      **snake_case__							)

				def 	a__		(     self							)    ->  Union[str, Any]:
					return self.set_src_lang_special_tokens(self.src_lang							)

				def 	a__		(     self							)    ->  Union[str, Any]:
					return self.set_tgt_lang_special_tokens(self.tgt_lang							)

				def 	a__		(     self							,      a_							)    ->  Any:
					lowercase						:		Any   					=		self.convert_tokens_to_ids(snake_case__							)

					if self.legacy_behaviour:
						lowercase						:		Tuple   					=		[]
						lowercase						:		Optional[Any]   					=		[self.eos_token_id, self.cur_lang_code]
					else:
						lowercase						:		Tuple   					=		[self.cur_lang_code]
						lowercase						:		Any   					=		[self.eos_token_id]

					lowercase						:		Optional[int]   					=		self.convert_ids_to_tokens(self.prefix_tokens							)
					lowercase						:		Any   					=		self.convert_ids_to_tokens(self.suffix_tokens							)

					lowercase						:		Optional[Any]   					=		processors.TemplateProcessing(
					    single=prefix_tokens_str + ["$A"] + suffix_tokens_str							,      pair=prefix_tokens_str + ["$A", "$B"] + suffix_tokens_str							,      special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str							,      self.prefix_tokens + self.suffix_tokens							)							)							,      )

				def 	a__		(     self							,      a_							)    ->  List[Any]:
					lowercase						:		int   					=		self.convert_tokens_to_ids(snake_case__							)
					if self.legacy_behaviour:
						lowercase						:		Union[str, Any]   					=		[]
						lowercase						:		Any   					=		[self.eos_token_id, self.cur_lang_code]
					else:
						lowercase						:		Tuple   					=		[self.cur_lang_code]
						lowercase						:		Optional[Any]   					=		[self.eos_token_id]

					lowercase						:		Optional[Any]   					=		self.convert_ids_to_tokens(self.prefix_tokens							)
					lowercase						:		Optional[int]   					=		self.convert_ids_to_tokens(self.suffix_tokens							)

					lowercase						:		Optional[int]   					=		processors.TemplateProcessing(
					    single=prefix_tokens_str + ["$A"] + suffix_tokens_str							,      pair=prefix_tokens_str + ["$A", "$B"] + suffix_tokens_str							,      special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str							,      self.prefix_tokens + self.suffix_tokens							)							)							,      )


				def 	a__		(     self							,      a_							,      a_ = None							)    ->  List[str]:
					if not self.can_save_slow_tokenizer:
						raise ValueError(
						    "Your fast tokenizer does not have the necessary information to save the vocabulary for a slow "
						    "tokenizer."							)

					if not os.path.isdir(snake_case__							):
						logger.error(F'''Vocabulary path ({save_directory}) should be a directory.'''							)
						return
					lowercase						:		Union[str, Any]   					=		os.path.join(
					    snake_case__							,      (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]							)

					if os.path.abspath(self.vocab_file							) != os.path.abspath(snake_case__							):
						copyfile(self.vocab_file							,      snake_case__							)

					return (out_vocab_file,)










"""simple docstring"""



from ...configuration_utils import PretrainedConfig
from ...utils import logging


_lowerCAmelCase	:	Optional[int]				=      logging.get_logger(__name__)

_lowerCAmelCase	:	Tuple				=      {
    """uw-madison/mra-base-512-4""": """https://huggingface.co/uw-madison/mra-base-512-4/resolve/main/config.json""",
}




class 			lowerCAmelCase__			(    __magic_name__   ):
   SCREAMING_SNAKE_CASE_    ='''mra'''
   def __init__(		self      :	Any							,			snake_case__      :	List[str]=5_0_2_6_5							,			snake_case__      :	Any=7_6_8							,			snake_case__      :	Union[str, Any]=1_2							,			snake_case__      :	Optional[Any]=1_2							,			snake_case__      :	Tuple=3_0_7_2							,			snake_case__      :	str="gelu"							,			snake_case__      :	Any=0.1							,			snake_case__      :	Tuple=0.1							,			snake_case__      :	Tuple=5_1_2							,			snake_case__      :	Union[str, Any]=1							,			snake_case__      :	List[Any]=0.02							,			snake_case__      :	str=1e-5							,			snake_case__      :	List[Any]="absolute"							,			snake_case__      :	str=4							,			snake_case__      :	List[str]="full"							,			snake_case__      :	Tuple=0							,			snake_case__      :	Any=0							,			snake_case__      :	Union[str, Any]=1							,			snake_case__      :	int=0							,			snake_case__      :	int=2							,			**snake_case__      :	List[Any]							,			):


        '''simple docstring'''
        super().__init__(pad_token_id=snake_case__							,			bos_token_id=snake_case__							,			eos_token_id=snake_case__							,			**snake_case__							)

        UpperCAmelCase__						:    List[Any]							=							vocab_size
        UpperCAmelCase__						:    str							=							max_position_embeddings
        UpperCAmelCase__						:    Any							=							hidden_size
        UpperCAmelCase__						:    Union[str, Any]							=							num_hidden_layers
        UpperCAmelCase__						:    str							=							num_attention_heads
        UpperCAmelCase__						:    int							=							intermediate_size
        UpperCAmelCase__						:    int							=							hidden_act
        UpperCAmelCase__						:    List[str]							=							hidden_dropout_prob
        UpperCAmelCase__						:    List[str]							=							attention_probs_dropout_prob
        UpperCAmelCase__						:    Any							=							initializer_range
        UpperCAmelCase__						:    Any							=							type_vocab_size
        UpperCAmelCase__						:    Dict							=							layer_norm_eps
        UpperCAmelCase__						:    Tuple							=							position_embedding_type
        UpperCAmelCase__						:    List[str]							=							block_per_row
        UpperCAmelCase__						:    Optional[Any]							=							approx_mode
        UpperCAmelCase__						:    Any							=							initial_prior_first_n_blocks
        UpperCAmelCase__						:    List[Any]							=							initial_prior_diagonal_n_blocks


