
'''simple docstring'''

import os
from shutil import copyfile
from typing import List, Optional, Tuple

from tokenizers import processors

from ...tokenization_utils import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
		from .tokenization_nllb import NllbTokenizer
else:
		UpperCAmelCase__ :			Optional[int]											=		None


UpperCAmelCase__ :			int											=		logging.get_logger(__name__)


UpperCAmelCase__ :			List[str]											=		{"vocab_file": "sentencepiece.bpe.model", "tokenizer_file": "tokenizer.json"}

UpperCAmelCase__ :			List[Any]											=		{
    "vocab_file": {
        "facebook/nllb-200-distilled-600M": (
            "https://huggingface.co/facebook/nllb-200-distilled-600M/resolve/main/sentencepiece.bpe.model"
        ),
    },
    "tokenizer_file": {
        "facebook/nllb-200-distilled-600M": (
            "https://huggingface.co/facebook/nllb-200-distilled-600M/resolve/main/tokenizer.json"
        ),
    },
}

UpperCAmelCase__ :			List[str]											=		{
    "facebook/nllb-large-en-ro": 10_24,
    "facebook/nllb-200-distilled-600M": 10_24,
}

# fmt: off
UpperCAmelCase__ :			Union[str, Any]											=		["ace_Arab", "ace_Latn", "acm_Arab", "acq_Arab", "aeb_Arab", "afr_Latn", "ajp_Arab", "aka_Latn", "amh_Ethi", "apc_Arab", "arb_Arab", "ars_Arab", "ary_Arab", "arz_Arab", "asm_Beng", "ast_Latn", "awa_Deva", "ayr_Latn", "azb_Arab", "azj_Latn", "bak_Cyrl", "bam_Latn", "ban_Latn", "bel_Cyrl", "bem_Latn", "ben_Beng", "bho_Deva", "bjn_Arab", "bjn_Latn", "bod_Tibt", "bos_Latn", "bug_Latn", "bul_Cyrl", "cat_Latn", "ceb_Latn", "ces_Latn", "cjk_Latn", "ckb_Arab", "crh_Latn", "cym_Latn", "dan_Latn", "deu_Latn", "dik_Latn", "dyu_Latn", "dzo_Tibt", "ell_Grek", "eng_Latn", "epo_Latn", "est_Latn", "eus_Latn", "ewe_Latn", "fao_Latn", "pes_Arab", "fij_Latn", "fin_Latn", "fon_Latn", "fra_Latn", "fur_Latn", "fuv_Latn", "gla_Latn", "gle_Latn", "glg_Latn", "grn_Latn", "guj_Gujr", "hat_Latn", "hau_Latn", "heb_Hebr", "hin_Deva", "hne_Deva", "hrv_Latn", "hun_Latn", "hye_Armn", "ibo_Latn", "ilo_Latn", "ind_Latn", "isl_Latn", "ita_Latn", "jav_Latn", "jpn_Jpan", "kab_Latn", "kac_Latn", "kam_Latn", "kan_Knda", "kas_Arab", "kas_Deva", "kat_Geor", "knc_Arab", "knc_Latn", "kaz_Cyrl", "kbp_Latn", "kea_Latn", "khm_Khmr", "kik_Latn", "kin_Latn", "kir_Cyrl", "kmb_Latn", "kon_Latn", "kor_Hang", "kmr_Latn", "lao_Laoo", "lvs_Latn", "lij_Latn", "lim_Latn", "lin_Latn", "lit_Latn", "lmo_Latn", "ltg_Latn", "ltz_Latn", "lua_Latn", "lug_Latn", "luo_Latn", "lus_Latn", "mag_Deva", "mai_Deva", "mal_Mlym", "mar_Deva", "min_Latn", "mkd_Cyrl", "plt_Latn", "mlt_Latn", "mni_Beng", "khk_Cyrl", "mos_Latn", "mri_Latn", "zsm_Latn", "mya_Mymr", "nld_Latn", "nno_Latn", "nob_Latn", "npi_Deva", "nso_Latn", "nus_Latn", "nya_Latn", "oci_Latn", "gaz_Latn", "ory_Orya", "pag_Latn", "pan_Guru", "pap_Latn", "pol_Latn", "por_Latn", "prs_Arab", "pbt_Arab", "quy_Latn", "ron_Latn", "run_Latn", "rus_Cyrl", "sag_Latn", "san_Deva", "sat_Beng", "scn_Latn", "shn_Mymr", "sin_Sinh", "slk_Latn", "slv_Latn", "smo_Latn", "sna_Latn", "snd_Arab", "som_Latn", "sot_Latn", "spa_Latn", "als_Latn", "srd_Latn", "srp_Cyrl", "ssw_Latn", "sun_Latn", "swe_Latn", "swh_Latn", "szl_Latn", "tam_Taml", "tat_Cyrl", "tel_Telu", "tgk_Cyrl", "tgl_Latn", "tha_Thai", "tir_Ethi", "taq_Latn", "taq_Tfng", "tpi_Latn", "tsn_Latn", "tso_Latn", "tuk_Latn", "tum_Latn", "tur_Latn", "twi_Latn", "tzm_Tfng", "uig_Arab", "ukr_Cyrl", "umb_Latn", "urd_Arab", "uzn_Latn", "vec_Latn", "vie_Latn", "war_Latn", "wol_Latn", "xho_Latn", "ydd_Hebr", "yor_Latn", "yue_Hant", "zho_Hans", "zho_Hant", "zul_Latn"]






class  A  (						SCREAMING_SNAKE_CASE__     ):
	snake_case__   :Tuple   =  VOCAB_FILES_NAMES
	snake_case__   :Tuple   =  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	snake_case__   :List[Any]   =  PRETRAINED_VOCAB_FILES_MAP
	snake_case__   :Tuple   =  ['input_ids', 'attention_mask']
	snake_case__   :str   =  NllbTokenizer

	snake_case__   :List[int]   =  []
	snake_case__   :List[int]   =  []





	def __init__(       self      :							str   ,				__magic_name__      :							Dict=None   ,				__magic_name__      :							List[str]=None   ,				__magic_name__      :							List[Any]="<s>"   ,				__magic_name__      :							Any="</s>"   ,				__magic_name__      :							Tuple="</s>"   ,				__magic_name__      :							str="<s>"   ,				__magic_name__      :							Optional[int]="<unk>"   ,				__magic_name__      :							str="<pad>"   ,				__magic_name__      :							int="<mask>"   ,				__magic_name__      :							List[Any]=None   ,				__magic_name__      :							Optional[int]=None   ,				__magic_name__      :							Any=None   ,				__magic_name__      :							Optional[Any]=False   ,				**__magic_name__      :							Union[str, Any]   ,				):


				"""simple docstring"""

				lowerCAmelCase__										=				AddedToken(__magic_name__   ,				lstrip=__magic_name__   ,				rstrip=__magic_name__				) if isinstance(__magic_name__   ,				__magic_name__				) else mask_token
				lowerCAmelCase__										=				legacy_behaviour
				super().__init__(
				    vocab_file=__magic_name__   ,				tokenizer_file=__magic_name__   ,				bos_token=__magic_name__   ,				eos_token=__magic_name__   ,				sep_token=__magic_name__   ,				cls_token=__magic_name__   ,				unk_token=__magic_name__   ,				pad_token=__magic_name__   ,				mask_token=__magic_name__   ,				src_lang=__magic_name__   ,				tgt_lang=__magic_name__   ,				additional_special_tokens=__magic_name__   ,				legacy_behaviour=__magic_name__   ,				**__magic_name__   ,				)

				lowerCAmelCase__										=				vocab_file
				lowerCAmelCase__										=				False if not self.vocab_file else True

				lowerCAmelCase__										=				FAIRSEQ_LANGUAGE_CODES.copy()

				if additional_special_tokens is not None:
							# Only add those special tokens if they are not already there.
							_additional_special_tokens.extend(
							    [t for t in additional_special_tokens if t not in _additional_special_tokens]				)

				self.add_special_tokens({"additional_special_tokens": _additional_special_tokens}				)
				lowerCAmelCase__										=				{
				    lang_code: self.convert_tokens_to_ids(__magic_name__				) for lang_code in FAIRSEQ_LANGUAGE_CODES
				}

				lowerCAmelCase__										=				src_lang if src_lang is not None else "eng_Latn"
				lowerCAmelCase__										=				self.convert_tokens_to_ids(self._src_lang				)
				lowerCAmelCase__										=				tgt_lang
				self.set_src_lang_special_tokens(self._src_lang				)





	@property
	def 				__SCREAMING_SNAKE_CASE	(       self      :							Tuple				):


				"""simple docstring"""

				return self._src_lang





	@src_lang.setter
	def 				__SCREAMING_SNAKE_CASE	(       self      :							Optional[Any]   ,				__magic_name__      :							str				):


				"""simple docstring"""

				lowerCAmelCase__										=				new_src_lang
				self.set_src_lang_special_tokens(self._src_lang				)





	def 				__SCREAMING_SNAKE_CASE	(       self      :							Tuple   ,				__magic_name__      :							List[int]   ,				__magic_name__      :							Optional[List[int]] = None				):


				"""simple docstring"""

				if token_ids_a is None:
							return self.prefix_tokens + token_ids_a + self.suffix_tokens
				# We don't expect to process pairs, but leave the pair logic for API consistency
				return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens





	def 				__SCREAMING_SNAKE_CASE	(       self      :							Any   ,				__magic_name__      :							List[int]   ,				__magic_name__      :							Optional[List[int]] = None				):


				"""simple docstring"""

				lowerCAmelCase__										=				[self.sep_token_id]
				lowerCAmelCase__										=				[self.cls_token_id]

				if token_ids_a is None:
							return len(cls + token_ids_a + sep				) * [0]
				return len(cls + token_ids_a + sep + sep + token_ids_a + sep				) * [0]





	def 				__SCREAMING_SNAKE_CASE	(       self      :							Any   ,				__magic_name__      :							Tuple   ,				__magic_name__      :							str   ,				__magic_name__      :							Optional[str]   ,				__magic_name__      :							Optional[str]   ,				**__magic_name__      :							int				):


				"""simple docstring"""

				if src_lang is None or tgt_lang is None:
							raise ValueError("Translation requires a `src_lang` and a `tgt_lang` for this model"				)
				lowerCAmelCase__										=				src_lang
				lowerCAmelCase__										=				self(__magic_name__   ,				add_special_tokens=__magic_name__   ,				return_tensors=__magic_name__   ,				**__magic_name__				)
				lowerCAmelCase__										=				self.convert_tokens_to_ids(__magic_name__				)
				lowerCAmelCase__										=				tgt_lang_id
				return inputs





	def 				__SCREAMING_SNAKE_CASE	(       self      :							Tuple   ,				__magic_name__      :							List[str]   ,				__magic_name__      :							str = "eng_Latn"   ,				__magic_name__      :							Optional[List[str]] = None   ,				__magic_name__      :							str = "fra_Latn"   ,				**__magic_name__      :							int   ,				):


				"""simple docstring"""

				lowerCAmelCase__										=				src_lang
				lowerCAmelCase__										=				tgt_lang
				return super().prepare_seqaseq_batch(__magic_name__   ,				__magic_name__   ,				**__magic_name__				)





	def 				__SCREAMING_SNAKE_CASE	(       self      :							Union[str, Any]				):


				"""simple docstring"""

				return self.set_src_lang_special_tokens(self.src_lang				)





	def 				__SCREAMING_SNAKE_CASE	(       self      :							Dict				):


				"""simple docstring"""

				return self.set_tgt_lang_special_tokens(self.tgt_lang				)





	def 				__SCREAMING_SNAKE_CASE	(       self      :							int   ,				__magic_name__      :							List[str]				):


				"""simple docstring"""

				lowerCAmelCase__										=				self.convert_tokens_to_ids(__magic_name__				)

				if self.legacy_behaviour:
							lowerCAmelCase__										=				[]
							lowerCAmelCase__										=				[self.eos_token_id, self.cur_lang_code]
				else:
							lowerCAmelCase__										=				[self.cur_lang_code]
							lowerCAmelCase__										=				[self.eos_token_id]

				lowerCAmelCase__										=				self.convert_ids_to_tokens(self.prefix_tokens				)
				lowerCAmelCase__										=				self.convert_ids_to_tokens(self.suffix_tokens				)

				lowerCAmelCase__										=				processors.TemplateProcessing(
				    single=prefix_tokens_str + ["$A"] + suffix_tokens_str   ,				pair=prefix_tokens_str + ["$A", "$B"] + suffix_tokens_str   ,				special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str   ,				self.prefix_tokens + self.suffix_tokens				)				)   ,				)





	def 				__SCREAMING_SNAKE_CASE	(       self      :							Any   ,				__magic_name__      :							str				):


				"""simple docstring"""

				lowerCAmelCase__										=				self.convert_tokens_to_ids(__magic_name__				)
				if self.legacy_behaviour:
							lowerCAmelCase__										=				[]
							lowerCAmelCase__										=				[self.eos_token_id, self.cur_lang_code]
				else:
							lowerCAmelCase__										=				[self.cur_lang_code]
							lowerCAmelCase__										=				[self.eos_token_id]

				lowerCAmelCase__										=				self.convert_ids_to_tokens(self.prefix_tokens				)
				lowerCAmelCase__										=				self.convert_ids_to_tokens(self.suffix_tokens				)

				lowerCAmelCase__										=				processors.TemplateProcessing(
				    single=prefix_tokens_str + ["$A"] + suffix_tokens_str   ,				pair=prefix_tokens_str + ["$A", "$B"] + suffix_tokens_str   ,				special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str   ,				self.prefix_tokens + self.suffix_tokens				)				)   ,				)







	def 				__SCREAMING_SNAKE_CASE	(       self      :							Union[str, Any]   ,				__magic_name__      :							str   ,				__magic_name__      :							Optional[str] = None				):


				"""simple docstring"""

				if not self.can_save_slow_tokenizer:
							raise ValueError(
							    "Your fast tokenizer does not have the necessary information to save the vocabulary for a slow "
							    "tokenizer."				)

				if not os.path.isdir(__magic_name__				):
							logger.error(f"""Vocabulary path ({save_directory}) should be a directory."""				)
							return
				lowerCAmelCase__										=				os.path.join(
				    __magic_name__   ,				(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]				)

				if os.path.abspath(self.vocab_file				) != os.path.abspath(__magic_name__				):
							copyfile(self.vocab_file   ,				__magic_name__				)

				return (out_vocab_file,)






from __future__ import annotations

import unittest

import numpy as np

from transformers import BlipTextConfig
from transformers.testing_utils import require_tf, slow
from transformers.utils import is_tf_available

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask


if is_tf_available():
				import tensorflow as tf

				from transformers import TFBlipTextModel
				from transformers.models.blip.modeling_tf_blip import TF_BLIP_PRETRAINED_MODEL_ARCHIVE_LIST



class        lowerCamelCase__				:





			def __init__(       self	:	Optional[Any]  ,  lowercase__	:	int  ,  lowercase__	:	List[str]=12  ,  lowercase__	:	Optional[int]=7  ,  lowercase__	:	List[Any]=True  ,  lowercase__	:	str=True  ,  lowercase__	:	Optional[Any]=True  ,  lowercase__	:	List[str]=99  ,  lowercase__	:	Optional[int]=32  ,  lowercase__	:	Tuple=32  ,  lowercase__	:	Dict=2  ,  lowercase__	:	Optional[int]=4  ,  lowercase__	:	str=37  ,  lowercase__	:	Optional[Any]=0.1  ,  lowercase__	:	List[str]=0.1  ,  lowercase__	:	Union[str, Any]=5_12  ,  lowercase__	:	List[str]=0.0_2  ,  lowercase__	:	Tuple=0  ,  lowercase__	:	List[Any]=None  ,  ):
										_lowerCAmelCase							=							parent
										_lowerCAmelCase							=							batch_size
										_lowerCAmelCase							=							seq_length
										_lowerCAmelCase							=							is_training
										_lowerCAmelCase							=							use_input_mask
										_lowerCAmelCase							=							use_labels
										_lowerCAmelCase							=							vocab_size
										_lowerCAmelCase							=							hidden_size
										_lowerCAmelCase							=							projection_dim
										_lowerCAmelCase							=							num_hidden_layers
										_lowerCAmelCase							=							num_attention_heads
										_lowerCAmelCase							=							intermediate_size
										_lowerCAmelCase							=							dropout
										_lowerCAmelCase							=							attention_dropout
										_lowerCAmelCase							=							max_position_embeddings
										_lowerCAmelCase							=							initializer_range
										_lowerCAmelCase							=							scope
										_lowerCAmelCase							=							bos_token_id





			def   SCREAMING_SNAKE_CASE__     (       self	:	int						):
										_lowerCAmelCase							=							ids_tensor([self.batch_size, self.seq_length]  ,  self.vocab_size						)

										_lowerCAmelCase							=							None
										if self.use_input_mask:
																	_lowerCAmelCase							=							random_attention_mask([self.batch_size, self.seq_length]						)

										if input_mask is not None:
																	_lowerCAmelCase							=							input_mask.numpy()
																	_lowerCAmelCase			,		_lowerCAmelCase							=							input_mask.shape
																	_lowerCAmelCase							=							np.random.randint(1  ,  seq_length - 1  ,  size=(batch_size,)						)
																	for batch_idx, start_index in enumerate(lowercase__						):
																								_lowerCAmelCase							=							1
																								_lowerCAmelCase							=							0

										_lowerCAmelCase							=							self.get_config()

										return config, input_ids, tf.convert_to_tensor(lowercase__						)





			def   SCREAMING_SNAKE_CASE__     (       self	:	List[Any]						):
										return BlipTextConfig(
										    vocab_size=self.vocab_size  ,  hidden_size=self.hidden_size  ,  projection_dim=self.projection_dim  ,  num_hidden_layers=self.num_hidden_layers  ,  num_attention_heads=self.num_attention_heads  ,  intermediate_size=self.intermediate_size  ,  dropout=self.dropout  ,  attention_dropout=self.attention_dropout  ,  max_position_embeddings=self.max_position_embeddings  ,  initializer_range=self.initializer_range  ,  bos_token_id=self.bos_token_id  ,  )





			def   SCREAMING_SNAKE_CASE__     (       self	:	Tuple  ,  lowercase__	:	int  ,  lowercase__	:	List[Any]  ,  lowercase__	:	Union[str, Any]						):
										_lowerCAmelCase							=							TFBlipTextModel(config=lowercase__						)
										_lowerCAmelCase							=							model(lowercase__  ,  attention_mask=lowercase__  ,  training=lowercase__						)
										_lowerCAmelCase							=							model(lowercase__  ,  training=lowercase__						)
										self.parent.assertEqual(result.last_hidden_state.shape  ,  (self.batch_size, self.seq_length, self.hidden_size)						)
										self.parent.assertEqual(result.pooler_output.shape  ,  (self.batch_size, self.hidden_size)						)





			def   SCREAMING_SNAKE_CASE__     (       self	:	int						):
										_lowerCAmelCase							=							self.prepare_config_and_inputs()
										_lowerCAmelCase			,		_lowerCAmelCase			,		_lowerCAmelCase							=							config_and_inputs
										_lowerCAmelCase							=							{'input_ids': input_ids, 'attention_mask': input_mask}
										return config, inputs_dict







@require_tf
class        lowerCamelCase__				(   UpperCAmelCase       ,unittest.TestCase				):
			UpperCamelCase__    							=(TFBlipTextModel,) if is_tf_available() else ()
			UpperCamelCase__    							=False
			UpperCamelCase__    							=False
			UpperCamelCase__    							=False





			def   SCREAMING_SNAKE_CASE__     (       self	:	Optional[Any]						):
										_lowerCAmelCase							=							BlipTextModelTester(self						)
										_lowerCAmelCase							=							ConfigTester(self  ,  config_class=lowercase__  ,  hidden_size=37						)





			def   SCREAMING_SNAKE_CASE__     (       self	:	Union[str, Any]						):
										self.config_tester.run_common_tests()





			def   SCREAMING_SNAKE_CASE__     (       self	:	Tuple						):
										_lowerCAmelCase							=							self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_model(*lowercase__						)





			def   SCREAMING_SNAKE_CASE__     (       self	:	Dict						):
										pass





			def   SCREAMING_SNAKE_CASE__     (       self	:	Dict						):
										pass





			@unittest.skip(reason='Blip does not use inputs_embeds'						)
			def   SCREAMING_SNAKE_CASE__     (       self	:	Optional[Any]						):
										pass





			@unittest.skip(reason='BlipTextModel has no base class and is not available in MODEL_MAPPING'						)
			def   SCREAMING_SNAKE_CASE__     (       self	:	Optional[Any]						):
										pass





			@unittest.skip(reason='BlipTextModel has no base class and is not available in MODEL_MAPPING'						)
			def   SCREAMING_SNAKE_CASE__     (       self	:	List[Any]						):
										pass





			@slow
			def   SCREAMING_SNAKE_CASE__     (       self	:	List[Any]						):
										for model_name in TF_BLIP_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
																	_lowerCAmelCase							=							TFBlipTextModel.from_pretrained(lowercase__						)
																	self.assertIsNotNone(lowercase__						)





			def   SCREAMING_SNAKE_CASE__     (       self	:	Any  ,  lowercase__	:	Any=True						):
										super().test_pt_tf_model_equivalence(allow_missing_keys=lowercase__						)






