





'''simple docstring'''


import inspect
import unittest

import numpy as np

from tests.test_modeling_common import floats_tensor
from transformers import DetrConfig, MaskFormerConfig, SwinConfig, is_torch_available, is_vision_available
from transformers.testing_utils import require_torch, require_torch_multi_gpu, require_vision, slow, torch_device
from transformers.utils import cached_property

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
		import torch

		from transformers import MaskFormerForInstanceSegmentation, MaskFormerModel

		if is_vision_available():
				from transformers import MaskFormerImageProcessor

if is_vision_available():
		from PIL import Image






class       UpperCamelCase__  :







	def __init__(				self			:   Optional[int] , lowerCamelCase			:   Optional[int] , lowerCamelCase			:   Union[str, Any]=2 , lowerCamelCase			:   str=True , lowerCamelCase			:   Optional[Any]=False , lowerCamelCase			:   Optional[Any]=1_0 , lowerCamelCase			:   Dict=3 , lowerCamelCase			:   Tuple=3_2 * 4 , lowerCamelCase			:   Optional[int]=3_2 * 6 , lowerCamelCase			:   List[str]=4 , lowerCamelCase			:   int=3_2 , ):




				'''simple docstring'''
				a__     = parent
				a__     = batch_size
				a__     = is_training
				a__     = use_auxiliary_loss
				a__     = num_queries
				a__     = num_channels
				a__     = min_size
				a__     = max_size
				a__     = num_labels
				a__     = mask_feature_size







	def 						__a   (				self			:   int		):




				'''simple docstring'''
				a__     = floats_tensor([self.batch_size, self.num_channels, self.min_size, self.max_size]		).to(
				    lowerCamelCase		)

				a__     = torch.ones([self.batch_size, self.min_size, self.max_size] , device=lowerCamelCase		)

				a__     = (
				    torch.rand([self.batch_size, self.num_labels, self.min_size, self.max_size] , device=lowerCamelCase		) > 0.5
				).float()
				a__     = (torch.rand((self.batch_size, self.num_labels) , device=lowerCamelCase		) > 0.5).long()

				a__     = self.get_config()
				return config, pixel_values, pixel_mask, mask_labels, class_labels







	def 						__a   (				self			:   Union[str, Any]		):




				'''simple docstring'''
				return MaskFormerConfig.from_backbone_and_decoder_configs(
				    backbone_config=SwinConfig(
				        depths=[1, 1, 1, 1] , ) , decoder_config=DetrConfig(
				        decoder_ffn_dim=1_2_8 , num_queries=self.num_queries , decoder_attention_heads=2 , d_model=self.mask_feature_size , ) , mask_feature_size=self.mask_feature_size , fpn_feature_size=self.mask_feature_size , num_channels=self.num_channels , num_labels=self.num_labels , )







	def 						__a   (				self			:   Union[str, Any]		):




				'''simple docstring'''
				a__   ,				a__   ,				a__   ,				a__   ,				a__     = self.prepare_config_and_inputs()
				a__     = {"pixel_values": pixel_values, "pixel_mask": pixel_mask}
				return config, inputs_dict







	def 						__a   (				self			:   Union[str, Any] , lowerCamelCase			:   Optional[int] , lowerCamelCase			:   int		):




				'''simple docstring'''
				a__     = output.encoder_hidden_states
				a__     = output.pixel_decoder_hidden_states
				a__     = output.transformer_decoder_hidden_states

				self.parent.assertTrue(len(lowerCamelCase		) , len(config.backbone_config.depths		)		)
				self.parent.assertTrue(len(lowerCamelCase		) , len(config.backbone_config.depths		)		)
				self.parent.assertTrue(len(lowerCamelCase		) , config.decoder_config.decoder_layers		)







	def 						__a   (				self			:   Optional[int] , lowerCamelCase			:   Optional[Any] , lowerCamelCase			:   Dict , lowerCamelCase			:   List[Any] , lowerCamelCase			:   Any=False		):




				'''simple docstring'''
				with torch.no_grad():
							a__     = MaskFormerModel(config=lowerCamelCase		)
							model.to(lowerCamelCase		)
							model.eval()

							a__     = model(pixel_values=lowerCamelCase , pixel_mask=lowerCamelCase		)
							a__     = model(lowerCamelCase , output_hidden_states=lowerCamelCase		)
				# the correct shape of output.transformer_decoder_hidden_states ensure the correcteness of the
				# encoder and pixel decoder
				self.parent.assertEqual(
				    output.transformer_decoder_last_hidden_state.shape , (self.batch_size, self.num_queries, self.mask_feature_size) , )
				# let's ensure the other two hidden state exists
				self.parent.assertTrue(output.pixel_decoder_last_hidden_state is not None		)
				self.parent.assertTrue(output.encoder_last_hidden_state is not None		)

				if output_hidden_states:
							self.check_output_hidden_state(lowerCamelCase , lowerCamelCase		)







	def 						__a   (				self			:   Any , lowerCamelCase			:   Union[str, Any] , lowerCamelCase			:   List[Any] , lowerCamelCase			:   Union[str, Any] , lowerCamelCase			:   int , lowerCamelCase			:   int		):




				'''simple docstring'''
				a__     = MaskFormerForInstanceSegmentation(config=lowerCamelCase		)
				model.to(lowerCamelCase		)
				model.eval()

				def comm_check_on_output(lowerCamelCase			:   Tuple		):
							# let's still check that all the required stuff is there
							self.parent.assertTrue(result.transformer_decoder_last_hidden_state is not None		)
							self.parent.assertTrue(result.pixel_decoder_last_hidden_state is not None		)
							self.parent.assertTrue(result.encoder_last_hidden_state is not None		)
							# okay, now we need to check the logits shape
							# due to the encoder compression, masks have a //4 spatial size
							self.parent.assertEqual(
							    result.masks_queries_logits.shape , (self.batch_size, self.num_queries, self.min_size // 4, self.max_size // 4) , )
							# + 1 for null class
							self.parent.assertEqual(
							    result.class_queries_logits.shape , (self.batch_size, self.num_queries, self.num_labels + 1)		)

				with torch.no_grad():
							a__     = model(pixel_values=lowerCamelCase , pixel_mask=lowerCamelCase		)
							a__     = model(lowerCamelCase		)

							comm_check_on_output(lowerCamelCase		)

							a__     = model(
							    pixel_values=lowerCamelCase , pixel_mask=lowerCamelCase , mask_labels=lowerCamelCase , class_labels=lowerCamelCase		)

				comm_check_on_output(lowerCamelCase		)

				self.parent.assertTrue(result.loss is not None		)
				self.parent.assertEqual(result.loss.shape , torch.Size([1]		)		)






@require_torch
class       UpperCamelCase__  (		__lowerCAmelCase	,__lowerCAmelCase	,unittest.TestCase     ):
	lowerCAmelCase__					:			int								=						(MaskFormerModel, MaskFormerForInstanceSegmentation) if is_torch_available() else ()
	lowerCAmelCase__					:			Dict								=						(
	    {"feature-extraction": MaskFormerModel, "image-segmentation": MaskFormerForInstanceSegmentation}
	    if is_torch_available()
	    else {}
	)

	lowerCAmelCase__					:			Optional[int]								=						False
	lowerCAmelCase__					:			int								=						False
	lowerCAmelCase__					:			Tuple								=						False
	lowerCAmelCase__					:			Dict								=						False







	def 						__a   (				self			:   List[Any]		):




				'''simple docstring'''
				a__     = MaskFormerModelTester(self		)
				a__     = ConfigTester(self , config_class=lowerCamelCase , has_text_modality=lowerCamelCase		)







	def 						__a   (				self			:   Tuple		):




				'''simple docstring'''
				self.config_tester.run_common_tests()







	def 						__a   (				self			:   Any		):




				'''simple docstring'''
				a__   ,				a__     = self.model_tester.prepare_config_and_inputs_for_common()
				self.model_tester.create_and_check_maskformer_model(lowerCamelCase , **lowerCamelCase , output_hidden_states=lowerCamelCase		)







	def 						__a   (				self			:   Dict		):




				'''simple docstring'''
				a__     = self.model_tester.prepare_config_and_inputs()
				self.model_tester.create_and_check_maskformer_instance_segmentation_head_model(*lowerCamelCase		)







	@unittest.skip(reason="MaskFormer does not use inputs_embeds"		)
	def 						__a   (				self			:   Optional[int]		):




				'''simple docstring'''
				pass







	@unittest.skip(reason="MaskFormer does not have a get_input_embeddings method"		)
	def 						__a   (				self			:   Optional[int]		):




				'''simple docstring'''
				pass







	@unittest.skip(reason="MaskFormer is not a generative model"		)
	def 						__a   (				self			:   Optional[int]		):




				'''simple docstring'''
				pass







	@unittest.skip(reason="MaskFormer does not use token embeddings"		)
	def 						__a   (				self			:   int		):




				'''simple docstring'''
				pass







	@require_torch_multi_gpu
	@unittest.skip(
	    reason="MaskFormer has some layers using `add_module` which doesn't work well with `nn.DataParallel`"		)
	def 						__a   (				self			:   Dict		):




				'''simple docstring'''
				pass







	@unittest.skip("Will be fixed soon by reducing the size of the model used for common tests."		)
	def 						__a   (				self			:   Dict		):




				'''simple docstring'''
				pass







	def 						__a   (				self			:   Any		):




				'''simple docstring'''
				a__   ,				a__     = self.model_tester.prepare_config_and_inputs_for_common()

				for model_class in self.all_model_classes:
							a__     = model_class(lowerCamelCase		)
							a__     = inspect.signature(model.forward		)
							# signature.parameters is an OrderedDict => so arg_names order is deterministic
							a__     = [*signature.parameters.keys()]

							a__     = ["pixel_values"]
							self.assertListEqual(arg_names[:1] , lowerCamelCase		)







	@slow
	def 						__a   (				self			:   List[str]		):




				'''simple docstring'''
				for model_name in ["facebook/maskformer-swin-small-coco"]:
							a__     = MaskFormerModel.from_pretrained(lowerCamelCase		)
							self.assertIsNotNone(lowerCamelCase		)







	def 						__a   (				self			:   Any		):




				'''simple docstring'''
				a__     = (self.model_tester.min_size,) * 2
				a__     = {
				    "pixel_values": torch.randn((2, 3, *size) , device=lowerCamelCase		),
				    "mask_labels": torch.randn((2, 1_0, *size) , device=lowerCamelCase		),
				    "class_labels": torch.zeros(2 , 1_0 , device=lowerCamelCase		).long(),
				}

				a__     = MaskFormerForInstanceSegmentation(MaskFormerConfig()		).to(lowerCamelCase		)
				a__     = model(**lowerCamelCase		)
				self.assertTrue(outputs.loss is not None		)







	def 						__a   (				self			:   Optional[int]		):




				'''simple docstring'''
				a__   ,				a__     = self.model_tester.prepare_config_and_inputs_for_common()
				self.model_tester.create_and_check_maskformer_model(lowerCamelCase , **lowerCamelCase , output_hidden_states=lowerCamelCase		)







	def 						__a   (				self			:   Union[str, Any]		):




				'''simple docstring'''
				a__   ,				a__     = self.model_tester.prepare_config_and_inputs_for_common()

				for model_class in self.all_model_classes:
							a__     = model_class(lowerCamelCase		).to(lowerCamelCase		)
							a__     = model(**lowerCamelCase , output_attentions=lowerCamelCase		)
							self.assertTrue(outputs.attentions is not None		)







	def 						__a   (				self			:   List[Any]		):




				'''simple docstring'''
				if not self.model_tester.is_training:
							return
				# only MaskFormerForInstanceSegmentation has the loss
				a__     = self.all_model_classes[1]
				a__   ,				a__   ,				a__   ,				a__   ,				a__     = self.model_tester.prepare_config_and_inputs()

				a__     = model_class(lowerCamelCase		)
				model.to(lowerCamelCase		)
				model.train()

				a__     = model(lowerCamelCase , mask_labels=lowerCamelCase , class_labels=lowerCamelCase		).loss
				loss.backward()







	def 						__a   (				self			:   Optional[Any]		):




				'''simple docstring'''
				# only MaskFormerForInstanceSegmentation has the loss
				a__     = self.all_model_classes[1]
				a__   ,				a__   ,				a__   ,				a__   ,				a__     = self.model_tester.prepare_config_and_inputs()
				a__     = True
				a__     = True

				a__     = model_class(lowerCamelCase		)
				model.to(lowerCamelCase		)
				model.train()

				a__     = model(lowerCamelCase , mask_labels=lowerCamelCase , class_labels=lowerCamelCase		)

				a__     = outputs.encoder_hidden_states[0]
				encoder_hidden_states.retain_grad()

				a__     = outputs.pixel_decoder_hidden_states[0]
				pixel_decoder_hidden_states.retain_grad()
				# we requires_grad=True in inputs_embeds (line 2152), the original implementation don't
				a__     = outputs.transformer_decoder_hidden_states[0]
				transformer_decoder_hidden_states.retain_grad()

				a__     = outputs.attentions[0]
				attentions.retain_grad()

				outputs.loss.backward(retain_graph=lowerCamelCase		)

				self.assertIsNotNone(encoder_hidden_states.grad		)
				self.assertIsNotNone(pixel_decoder_hidden_states.grad		)
				self.assertIsNotNone(transformer_decoder_hidden_states.grad		)
				self.assertIsNotNone(attentions.grad		)





lowerCAmelCase_   :   int       =					1E-4







def    _lowerCamelCase  ()   ->    int:
			a__     = Image.open("./tests/fixtures/tests_samples/COCO/000000039769.png"     )
			return image






@require_vision
@slow
class       UpperCamelCase__  (		unittest.TestCase     ):







	@cached_property
	def 						__a   (				self			:   str		):




				'''simple docstring'''
				return (
				    MaskFormerImageProcessor.from_pretrained("facebook/maskformer-swin-small-coco"		)
				    if is_vision_available()
				    else None
				)







	def 						__a   (				self			:   Dict		):




				'''simple docstring'''
				a__     = MaskFormerModel.from_pretrained("facebook/maskformer-swin-small-coco"		).to(lowerCamelCase		)
				a__     = self.default_image_processor
				a__     = prepare_img()
				a__     = image_processor(lowerCamelCase , return_tensors="pt"		).to(lowerCamelCase		)
				a__     = inputs["pixel_values"].shape
				# check size is divisible by 32
				self.assertTrue((inputs_shape[-1] % 3_2) == 0 and (inputs_shape[-2] % 3_2) == 0		)
				# check size
				self.assertEqual(lowerCamelCase , (1, 3, 8_0_0, 1_0_8_8)		)

				with torch.no_grad():
							a__     = model(**lowerCamelCase		)

				a__     = torch.tensor(
				    [[-0.0482, 0.9228, 0.4951], [-0.2547, 0.8017, 0.8527], [-0.0069, 0.3385, -0.0089]]		).to(lowerCamelCase		)
				self.assertTrue(
				    torch.allclose(
				        outputs.encoder_last_hidden_state[0, 0, :3, :3] , lowerCamelCase , atol=lowerCamelCase		)		)

				a__     = torch.tensor(
				    [[-0.8422, -0.8434, -0.9718], [-1.0144, -0.5565, -0.4195], [-1.0038, -0.4484, -0.1961]]		).to(lowerCamelCase		)
				self.assertTrue(
				    torch.allclose(
				        outputs.pixel_decoder_last_hidden_state[0, 0, :3, :3] , lowerCamelCase , atol=lowerCamelCase		)		)

				a__     = torch.tensor(
				    [[0.2852, -0.0159, 0.9735], [0.6254, 0.1858, 0.8529], [-0.0680, -0.4116, 1.8413]]		).to(lowerCamelCase		)
				self.assertTrue(
				    torch.allclose(
				        outputs.transformer_decoder_last_hidden_state[0, :3, :3] , lowerCamelCase , atol=lowerCamelCase		)		)







	def 						__a   (				self			:   List[Any]		):




				'''simple docstring'''
				a__     = (
				    MaskFormerForInstanceSegmentation.from_pretrained("facebook/maskformer-swin-small-coco"		)
				    .to(lowerCamelCase		)
				    .eval()
				)
				a__     = self.default_image_processor
				a__     = prepare_img()
				a__     = image_processor(lowerCamelCase , return_tensors="pt"		).to(lowerCamelCase		)
				a__     = inputs["pixel_values"].shape
				# check size is divisible by 32
				self.assertTrue((inputs_shape[-1] % 3_2) == 0 and (inputs_shape[-2] % 3_2) == 0		)
				# check size
				self.assertEqual(lowerCamelCase , (1, 3, 8_0_0, 1_0_8_8)		)

				with torch.no_grad():
							a__     = model(**lowerCamelCase		)
				# masks_queries_logits
				a__     = outputs.masks_queries_logits
				self.assertEqual(
				    masks_queries_logits.shape , (1, model.config.decoder_config.num_queries, inputs_shape[-2] // 4, inputs_shape[-1] // 4) , )
				a__     = [
				    [-1.3737124, -1.7724937, -1.9364233],
				    [-1.5977281, -1.9867939, -2.1523695],
				    [-1.5795398, -1.9269832, -2.093942],
				]
				a__     = torch.tensor(lowerCamelCase		).to(lowerCamelCase		)
				self.assertTrue(torch.allclose(masks_queries_logits[0, 0, :3, :3] , lowerCamelCase , atol=lowerCamelCase		)		)
				# class_queries_logits
				a__     = outputs.class_queries_logits
				self.assertEqual(
				    class_queries_logits.shape , (1, model.config.decoder_config.num_queries, model.config.num_labels + 1)		)
				a__     = torch.tensor(
				    [
				        [1.6512e00, -5.2572e00, -3.3519e00],
				        [3.6169e-02, -5.9025e00, -2.9313e00],
				        [1.0766e-04, -7.7630e00, -5.1263e00],
				    ]		).to(lowerCamelCase		)
				self.assertTrue(torch.allclose(outputs.class_queries_logits[0, :3, :3] , lowerCamelCase , atol=lowerCamelCase		)		)







	def 						__a   (				self			:   Dict		):




				'''simple docstring'''
				a__     = (
				    MaskFormerForInstanceSegmentation.from_pretrained("facebook/maskformer-resnet101-coco-stuff"		)
				    .to(lowerCamelCase		)
				    .eval()
				)
				a__     = self.default_image_processor
				a__     = prepare_img()
				a__     = image_processor(lowerCamelCase , return_tensors="pt"		).to(lowerCamelCase		)
				a__     = inputs["pixel_values"].shape
				# check size is divisible by 32
				self.assertTrue((inputs_shape[-1] % 3_2) == 0 and (inputs_shape[-2] % 3_2) == 0		)
				# check size
				self.assertEqual(lowerCamelCase , (1, 3, 8_0_0, 1_0_8_8)		)

				with torch.no_grad():
							a__     = model(**lowerCamelCase		)
				# masks_queries_logits
				a__     = outputs.masks_queries_logits
				self.assertEqual(
				    masks_queries_logits.shape , (1, model.config.decoder_config.num_queries, inputs_shape[-2] // 4, inputs_shape[-1] // 4) , )
				a__     = [[-0.9046, -2.6366, -4.6062], [-3.4179, -5.7890, -8.8057], [-4.9179, -7.6560, -10.7711]]
				a__     = torch.tensor(lowerCamelCase		).to(lowerCamelCase		)
				self.assertTrue(torch.allclose(masks_queries_logits[0, 0, :3, :3] , lowerCamelCase , atol=lowerCamelCase		)		)
				# class_queries_logits
				a__     = outputs.class_queries_logits
				self.assertEqual(
				    class_queries_logits.shape , (1, model.config.decoder_config.num_queries, model.config.num_labels + 1)		)
				a__     = torch.tensor(
				    [[4.7188, -3.2585, -2.8857], [6.6871, -2.9181, -1.2487], [7.2449, -2.2764, -2.1874]]		).to(lowerCamelCase		)
				self.assertTrue(torch.allclose(outputs.class_queries_logits[0, :3, :3] , lowerCamelCase , atol=lowerCamelCase		)		)







	def 						__a   (				self			:   Any		):




				'''simple docstring'''
				a__     = (
				    MaskFormerForInstanceSegmentation.from_pretrained("facebook/maskformer-swin-small-coco"		)
				    .to(lowerCamelCase		)
				    .eval()
				)
				a__     = self.default_image_processor

				a__     = image_processor(
				    [np.zeros((3, 8_0_0, 1_3_3_3)		), np.zeros((3, 8_0_0, 1_3_3_3)		)] , segmentation_maps=[np.zeros((3_8_4, 3_8_4)		).astype(np.floataa		), np.zeros((3_8_4, 3_8_4)		).astype(np.floataa		)] , return_tensors="pt" , )

				a__     = inputs["pixel_values"].to(lowerCamelCase		)
				a__     = [el.to(lowerCamelCase		) for el in inputs["mask_labels"]]
				a__     = [el.to(lowerCamelCase		) for el in inputs["class_labels"]]

				with torch.no_grad():
							a__     = model(**lowerCamelCase		)

				self.assertTrue(outputs.loss is not None		)










'''simple docstring'''


import os
from shutil import copyfile
from typing import List, Optional, Tuple

from tokenizers import processors

from ...tokenization_utils import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
		from .tokenization_nllb import NllbTokenizer
else:
		lowerCAmelCase_   :   Optional[Any]       =					None


lowerCAmelCase_   :   List[str]       =					logging.get_logger(__name__)


lowerCAmelCase_   :   Union[str, Any]       =					{"vocab_file": "sentencepiece.bpe.model", "tokenizer_file": "tokenizer.json"}

lowerCAmelCase_   :   List[Any]       =					{
    "vocab_file": {
        "facebook/nllb-200-distilled-600M": (
            "https://huggingface.co/facebook/nllb-200-distilled-600M/resolve/main/sentencepiece.bpe.model"
        ),
    },
    "tokenizer_file": {
        "facebook/nllb-200-distilled-600M": (
            "https://huggingface.co/facebook/nllb-200-distilled-600M/resolve/main/tokenizer.json"
        ),
    },
}

lowerCAmelCase_   :   List[str]       =					{
    "facebook/nllb-large-en-ro": 1024,
    "facebook/nllb-200-distilled-600M": 1024,
}

# fmt: off
lowerCAmelCase_   :   int       =					["ace_Arab", "ace_Latn", "acm_Arab", "acq_Arab", "aeb_Arab", "afr_Latn", "ajp_Arab", "aka_Latn", "amh_Ethi", "apc_Arab", "arb_Arab", "ars_Arab", "ary_Arab", "arz_Arab", "asm_Beng", "ast_Latn", "awa_Deva", "ayr_Latn", "azb_Arab", "azj_Latn", "bak_Cyrl", "bam_Latn", "ban_Latn", "bel_Cyrl", "bem_Latn", "ben_Beng", "bho_Deva", "bjn_Arab", "bjn_Latn", "bod_Tibt", "bos_Latn", "bug_Latn", "bul_Cyrl", "cat_Latn", "ceb_Latn", "ces_Latn", "cjk_Latn", "ckb_Arab", "crh_Latn", "cym_Latn", "dan_Latn", "deu_Latn", "dik_Latn", "dyu_Latn", "dzo_Tibt", "ell_Grek", "eng_Latn", "epo_Latn", "est_Latn", "eus_Latn", "ewe_Latn", "fao_Latn", "pes_Arab", "fij_Latn", "fin_Latn", "fon_Latn", "fra_Latn", "fur_Latn", "fuv_Latn", "gla_Latn", "gle_Latn", "glg_Latn", "grn_Latn", "guj_Gujr", "hat_Latn", "hau_Latn", "heb_Hebr", "hin_Deva", "hne_Deva", "hrv_Latn", "hun_Latn", "hye_Armn", "ibo_Latn", "ilo_Latn", "ind_Latn", "isl_Latn", "ita_Latn", "jav_Latn", "jpn_Jpan", "kab_Latn", "kac_Latn", "kam_Latn", "kan_Knda", "kas_Arab", "kas_Deva", "kat_Geor", "knc_Arab", "knc_Latn", "kaz_Cyrl", "kbp_Latn", "kea_Latn", "khm_Khmr", "kik_Latn", "kin_Latn", "kir_Cyrl", "kmb_Latn", "kon_Latn", "kor_Hang", "kmr_Latn", "lao_Laoo", "lvs_Latn", "lij_Latn", "lim_Latn", "lin_Latn", "lit_Latn", "lmo_Latn", "ltg_Latn", "ltz_Latn", "lua_Latn", "lug_Latn", "luo_Latn", "lus_Latn", "mag_Deva", "mai_Deva", "mal_Mlym", "mar_Deva", "min_Latn", "mkd_Cyrl", "plt_Latn", "mlt_Latn", "mni_Beng", "khk_Cyrl", "mos_Latn", "mri_Latn", "zsm_Latn", "mya_Mymr", "nld_Latn", "nno_Latn", "nob_Latn", "npi_Deva", "nso_Latn", "nus_Latn", "nya_Latn", "oci_Latn", "gaz_Latn", "ory_Orya", "pag_Latn", "pan_Guru", "pap_Latn", "pol_Latn", "por_Latn", "prs_Arab", "pbt_Arab", "quy_Latn", "ron_Latn", "run_Latn", "rus_Cyrl", "sag_Latn", "san_Deva", "sat_Beng", "scn_Latn", "shn_Mymr", "sin_Sinh", "slk_Latn", "slv_Latn", "smo_Latn", "sna_Latn", "snd_Arab", "som_Latn", "sot_Latn", "spa_Latn", "als_Latn", "srd_Latn", "srp_Cyrl", "ssw_Latn", "sun_Latn", "swe_Latn", "swh_Latn", "szl_Latn", "tam_Taml", "tat_Cyrl", "tel_Telu", "tgk_Cyrl", "tgl_Latn", "tha_Thai", "tir_Ethi", "taq_Latn", "taq_Tfng", "tpi_Latn", "tsn_Latn", "tso_Latn", "tuk_Latn", "tum_Latn", "tur_Latn", "twi_Latn", "tzm_Tfng", "uig_Arab", "ukr_Cyrl", "umb_Latn", "urd_Arab", "uzn_Latn", "vec_Latn", "vie_Latn", "war_Latn", "wol_Latn", "xho_Latn", "ydd_Hebr", "yor_Latn", "yue_Hant", "zho_Hans", "zho_Hant", "zul_Latn"]






class       UpperCamelCase__  (		__lowerCAmelCase     ):
	lowerCAmelCase__					:			int								=						VOCAB_FILES_NAMES
	lowerCAmelCase__					:			str								=						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	lowerCAmelCase__					:			int								=						PRETRAINED_VOCAB_FILES_MAP
	lowerCAmelCase__					:			int								=						["input_ids", "attention_mask"]
	lowerCAmelCase__					:			List[Any]								=						NllbTokenizer

	lowerCAmelCase__					:			List[int]								=						[]
	lowerCAmelCase__					:			List[int]								=						[]







	def __init__(				self			:   List[str] , lowerCamelCase			:   Optional[Any]=None , lowerCamelCase			:   Optional[int]=None , lowerCamelCase			:   Tuple="<s>" , lowerCamelCase			:   str="</s>" , lowerCamelCase			:   Optional[Any]="</s>" , lowerCamelCase			:   Dict="<s>" , lowerCamelCase			:   Any="<unk>" , lowerCamelCase			:   int="<pad>" , lowerCamelCase			:   Optional[Any]="<mask>" , lowerCamelCase			:   Dict=None , lowerCamelCase			:   Tuple=None , lowerCamelCase			:   Union[str, Any]=None , lowerCamelCase			:   int=False , **lowerCamelCase			:   Optional[Any] , ):




				'''simple docstring'''
				# Mask token behave like a normal word, i.e. include the space before it
				a__     = AddedToken(lowerCamelCase , lstrip=lowerCamelCase , rstrip=lowerCamelCase		) if isinstance(lowerCamelCase , lowerCamelCase		) else mask_token
				a__     = legacy_behaviour
				super().__init__(
				    vocab_file=lowerCamelCase , tokenizer_file=lowerCamelCase , bos_token=lowerCamelCase , eos_token=lowerCamelCase , sep_token=lowerCamelCase , cls_token=lowerCamelCase , unk_token=lowerCamelCase , pad_token=lowerCamelCase , mask_token=lowerCamelCase , src_lang=lowerCamelCase , tgt_lang=lowerCamelCase , additional_special_tokens=lowerCamelCase , legacy_behaviour=lowerCamelCase , **lowerCamelCase , )

				a__     = vocab_file
				a__     = False if not self.vocab_file else True

				a__     = FAIRSEQ_LANGUAGE_CODES.copy()

				if additional_special_tokens is not None:
							# Only add those special tokens if they are not already there.
							_additional_special_tokens.extend(
							    [t for t in additional_special_tokens if t not in _additional_special_tokens]		)

				self.add_special_tokens({"additional_special_tokens": _additional_special_tokens}		)
				a__     = {
				    lang_code: self.convert_tokens_to_ids(lowerCamelCase		) for lang_code in FAIRSEQ_LANGUAGE_CODES
				}

				a__     = src_lang if src_lang is not None else "eng_Latn"
				a__     = self.convert_tokens_to_ids(self._src_lang		)
				a__     = tgt_lang
				self.set_src_lang_special_tokens(self._src_lang		)







	@property
	def 						__a   (				self			:   Union[str, Any]		):




				'''simple docstring'''
				return self._src_lang







	@src_lang.setter
	def 						__a   (				self			:   int , lowerCamelCase			:   str		):




				'''simple docstring'''
				a__     = new_src_lang
				self.set_src_lang_special_tokens(self._src_lang		)







	def 						__a   (				self			:   Any , lowerCamelCase			:   List[int] , lowerCamelCase			:   Optional[List[int]] = None		):




				'''simple docstring'''
				if token_ids_a is None:
							return self.prefix_tokens + token_ids_a + self.suffix_tokens
				# We don't expect to process pairs, but leave the pair logic for API consistency
				return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens







	def 						__a   (				self			:   Union[str, Any] , lowerCamelCase			:   List[int] , lowerCamelCase			:   Optional[List[int]] = None		):




				'''simple docstring'''
				a__     = [self.sep_token_id]
				a__     = [self.cls_token_id]

				if token_ids_a is None:
							return len(cls + token_ids_a + sep		) * [0]
				return len(cls + token_ids_a + sep + sep + token_ids_a + sep		) * [0]







	def 						__a   (				self			:   str , lowerCamelCase			:   str , lowerCamelCase			:   str , lowerCamelCase			:   Optional[str] , lowerCamelCase			:   Optional[str] , **lowerCamelCase			:   List[str]		):




				'''simple docstring'''
				if src_lang is None or tgt_lang is None:
							raise ValueError("Translation requires a `src_lang` and a `tgt_lang` for this model"		)
				a__     = src_lang
				a__     = self(lowerCamelCase , add_special_tokens=lowerCamelCase , return_tensors=lowerCamelCase , **lowerCamelCase		)
				a__     = self.convert_tokens_to_ids(lowerCamelCase		)
				a__     = tgt_lang_id
				return inputs







	def 						__a   (				self			:   str , lowerCamelCase			:   List[str] , lowerCamelCase			:   str = "eng_Latn" , lowerCamelCase			:   Optional[List[str]] = None , lowerCamelCase			:   str = "fra_Latn" , **lowerCamelCase			:   Any , ):




				'''simple docstring'''
				a__     = src_lang
				a__     = tgt_lang
				return super().prepare_seqaseq_batch(lowerCamelCase , lowerCamelCase , **lowerCamelCase		)







	def 						__a   (				self			:   List[str]		):




				'''simple docstring'''
				return self.set_src_lang_special_tokens(self.src_lang		)







	def 						__a   (				self			:   List[str]		):




				'''simple docstring'''
				return self.set_tgt_lang_special_tokens(self.tgt_lang		)







	def 						__a   (				self			:   List[str] , lowerCamelCase			:   Union[str, Any]		):




				'''simple docstring'''
				a__     = self.convert_tokens_to_ids(lowerCamelCase		)

				if self.legacy_behaviour:
							a__     = []
							a__     = [self.eos_token_id, self.cur_lang_code]
				else:
							a__     = [self.cur_lang_code]
							a__     = [self.eos_token_id]

				a__     = self.convert_ids_to_tokens(self.prefix_tokens		)
				a__     = self.convert_ids_to_tokens(self.suffix_tokens		)

				a__     = processors.TemplateProcessing(
				    single=prefix_tokens_str + ["$A"] + suffix_tokens_str , pair=prefix_tokens_str + ["$A", "$B"] + suffix_tokens_str , special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str , self.prefix_tokens + self.suffix_tokens		)		) , )







	def 						__a   (				self			:   Tuple , lowerCamelCase			:   str		):




				'''simple docstring'''
				a__     = self.convert_tokens_to_ids(lowerCamelCase		)
				if self.legacy_behaviour:
							a__     = []
							a__     = [self.eos_token_id, self.cur_lang_code]
				else:
							a__     = [self.cur_lang_code]
							a__     = [self.eos_token_id]

				a__     = self.convert_ids_to_tokens(self.prefix_tokens		)
				a__     = self.convert_ids_to_tokens(self.suffix_tokens		)

				a__     = processors.TemplateProcessing(
				    single=prefix_tokens_str + ["$A"] + suffix_tokens_str , pair=prefix_tokens_str + ["$A", "$B"] + suffix_tokens_str , special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str , self.prefix_tokens + self.suffix_tokens		)		) , )







	def 						__a   (				self			:   Tuple , lowerCamelCase			:   str , lowerCamelCase			:   Optional[str] = None		):




				'''simple docstring'''
				if not self.can_save_slow_tokenizer:
							raise ValueError(
							    "Your fast tokenizer does not have the necessary information to save the vocabulary for a slow "
							    "tokenizer."		)

				if not os.path.isdir(lowerCamelCase		):
							logger.error(F'''Vocabulary path ({save_directory}) should be a directory.'''		)
							return
				a__     = os.path.join(
				    lowerCamelCase , (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]		)

				if os.path.abspath(self.vocab_file		) != os.path.abspath(lowerCamelCase		):
							copyfile(self.vocab_file , lowerCamelCase		)

				return (out_vocab_file,)



