

import inspect
import unittest

import numpy as np

from tests.test_modeling_common import floats_tensor
from transformers import MaskaFormerConfig, is_torch_available, is_vision_available
from transformers.testing_utils import require_torch, require_torch_multi_gpu, require_vision, slow, torch_device
from transformers.utils import cached_property

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
					import torch

					from transformers import MaskaFormerForUniversalSegmentation, MaskaFormerModel

					if is_vision_available():
										from transformers import MaskaFormerImageProcessor

if is_vision_available():
					from PIL import Image

class 						__UpperCamelCase    :




	def __init__(		self	:						int		,	lowerCAmelCase	:						str		,	lowerCAmelCase	:						Optional[Any]=2		,	lowerCAmelCase	:						Dict=True		,	lowerCAmelCase	:						Optional[int]=False		,	lowerCAmelCase	:						Any=10		,	lowerCAmelCase	:						str=3		,	lowerCAmelCase	:						Tuple=32 * 8		,	lowerCAmelCase	:						Optional[Any]=32 * 8		,	lowerCAmelCase	:						str=4		,	lowerCAmelCase	:						Tuple=64		,	):


		'''simple docstring'''







		UpperCAmelCase_       				=			parent
		UpperCAmelCase_       				=			batch_size
		UpperCAmelCase_       				=			is_training
		UpperCAmelCase_       				=			use_auxiliary_loss
		UpperCAmelCase_       				=			num_queries
		UpperCAmelCase_       				=			num_channels
		UpperCAmelCase_       				=			min_size
		UpperCAmelCase_       				=			max_size
		UpperCAmelCase_       				=			num_labels
		UpperCAmelCase_       				=			hidden_dim
		UpperCAmelCase_       				=			hidden_dim




	def       __A					(		self	:						Tuple							):


		'''simple docstring'''







		UpperCAmelCase_       				=			floats_tensor([self.batch_size, self.num_channels, self.min_size, self.max_size]							).to(
		    lowerCAmelCase							)

		UpperCAmelCase_       				=			torch.ones([self.batch_size, self.min_size, self.max_size]		,	device=lowerCAmelCase							)

		UpperCAmelCase_       				=			(
		    torch.rand([self.batch_size, self.num_labels, self.min_size, self.max_size]		,	device=lowerCAmelCase							) > 0.5
		).float()
		UpperCAmelCase_       				=			(torch.rand((self.batch_size, self.num_labels)		,	device=lowerCAmelCase							) > 0.5).long()

		UpperCAmelCase_       				=			self.get_config()
		return config, pixel_values, pixel_mask, mask_labels, class_labels




	def       __A					(		self	:						str							):


		'''simple docstring'''







		UpperCAmelCase_       				=			MaskaFormerConfig(
		    hidden_size=self.hidden_dim		,	)
		UpperCAmelCase_       				=			self.num_queries
		UpperCAmelCase_       				=			self.num_labels

		UpperCAmelCase_       				=			[1, 1, 1, 1]
		UpperCAmelCase_       				=			self.num_channels

		UpperCAmelCase_       				=			64
		UpperCAmelCase_       				=			128
		UpperCAmelCase_       				=			self.hidden_dim
		UpperCAmelCase_       				=			self.hidden_dim
		UpperCAmelCase_       				=			self.hidden_dim
		return config




	def       __A					(		self	:						int							):


		'''simple docstring'''







		UpperCAmelCase_       				=			self.prepare_config_and_inputs()
		UpperCAmelCase_       				=			{"pixel_values": pixel_values, "pixel_mask": pixel_mask}
		return config, inputs_dict




	def       __A					(		self	:						List[str]		,	lowerCAmelCase	:						List[Any]		,	lowerCAmelCase	:						Optional[Any]							):


		'''simple docstring'''







		UpperCAmelCase_       				=			output.encoder_hidden_states
		UpperCAmelCase_       				=			output.pixel_decoder_hidden_states
		UpperCAmelCase_       				=			output.transformer_decoder_hidden_states

		self.parent.assertTrue(len(lowerCAmelCase							)		,	len(config.backbone_config.depths							)							)
		self.parent.assertTrue(len(lowerCAmelCase							)		,	len(config.backbone_config.depths							)							)
		self.parent.assertTrue(len(lowerCAmelCase							)		,	config.decoder_layers							)




	def       __A					(		self	:						Tuple		,	lowerCAmelCase	:						Optional[Any]		,	lowerCAmelCase	:						int		,	lowerCAmelCase	:						List[str]		,	lowerCAmelCase	:						Any=False							):


		'''simple docstring'''







		with torch.no_grad():
			UpperCAmelCase_       				=			MaskaFormerModel(config=lowerCAmelCase							)
			model.to(lowerCAmelCase							)
			model.eval()

			UpperCAmelCase_       				=			model(pixel_values=lowerCAmelCase		,	pixel_mask=lowerCAmelCase							)
			UpperCAmelCase_       				=			model(lowerCAmelCase		,	output_hidden_states=lowerCAmelCase							)

		self.parent.assertEqual(
		    output.transformer_decoder_last_hidden_state.shape		,	(self.batch_size, self.num_queries, self.hidden_dim)		,	)
		# let's ensure the other two hidden state exists
		self.parent.assertTrue(output.pixel_decoder_last_hidden_state is not None							)
		self.parent.assertTrue(output.encoder_last_hidden_state is not None							)

		if output_hidden_states:
			self.check_output_hidden_state(lowerCAmelCase		,	lowerCAmelCase							)




	def       __A					(		self	:						Tuple		,	lowerCAmelCase	:						Optional[Any]		,	lowerCAmelCase	:						List[Any]		,	lowerCAmelCase	:						Tuple		,	lowerCAmelCase	:						Union[str, Any]		,	lowerCAmelCase	:						Union[str, Any]							):


		'''simple docstring'''







		UpperCAmelCase_       				=			MaskaFormerForUniversalSegmentation(config=lowerCAmelCase							)
		model.to(lowerCAmelCase							)
		model.eval()

		def comm_check_on_output(lowerCAmelCase	:						Optional[Any]							):
			# let's still check that all the required stuff is there
			self.parent.assertTrue(result.transformer_decoder_last_hidden_state is not None							)
			self.parent.assertTrue(result.pixel_decoder_last_hidden_state is not None							)
			self.parent.assertTrue(result.encoder_last_hidden_state is not None							)
			# okay, now we need to check the logits shape
			# due to the encoder compression, masks have a //4 spatial size
			self.parent.assertEqual(
			    result.masks_queries_logits.shape		,	(self.batch_size, self.num_queries, self.min_size // 4, self.max_size // 4)		,	)
			# + 1 for null class
			self.parent.assertEqual(
			    result.class_queries_logits.shape		,	(self.batch_size, self.num_queries, self.num_labels + 1)							)

		with torch.no_grad():
			UpperCAmelCase_       				=			model(pixel_values=lowerCAmelCase		,	pixel_mask=lowerCAmelCase							)
			UpperCAmelCase_       				=			model(lowerCAmelCase							)

			comm_check_on_output(lowerCAmelCase							)

			UpperCAmelCase_       				=			model(
			    pixel_values=lowerCAmelCase		,	pixel_mask=lowerCAmelCase		,	mask_labels=lowerCAmelCase		,	class_labels=lowerCAmelCase							)

		comm_check_on_output(lowerCAmelCase							)

		self.parent.assertTrue(result.loss is not None							)
		self.parent.assertEqual(result.loss.shape		,	torch.Size([1]							)							)


@require_torch
class 						__UpperCamelCase    (					_UpperCamelCase     ,     _UpperCamelCase     ,     unittest.TestCase						):
	SCREAMING_SNAKE_CASE__    				=     (MaskaFormerModel, MaskaFormerForUniversalSegmentation) if is_torch_available() else ()
	SCREAMING_SNAKE_CASE__    				=     {'feature-extraction': MaskaFormerModel} if is_torch_available() else {}

	SCREAMING_SNAKE_CASE__    				=     False
	SCREAMING_SNAKE_CASE__    				=     False
	SCREAMING_SNAKE_CASE__    				=     False
	SCREAMING_SNAKE_CASE__    				=     False




	def       __A					(		self	:						Optional[int]							):


		'''simple docstring'''







		UpperCAmelCase_       				=			MaskaFormerModelTester(self							)
		UpperCAmelCase_       				=			ConfigTester(self		,	config_class=lowerCAmelCase		,	has_text_modality=lowerCAmelCase							)




	def       __A					(		self	:						int							):


		'''simple docstring'''







		self.config_tester.run_common_tests()




	def       __A					(		self	:						int							):


		'''simple docstring'''







		UpperCAmelCase_       				=			self.model_tester.prepare_config_and_inputs_for_common()
		self.model_tester.create_and_check_maskaformer_model(lowerCAmelCase		,	**lowerCAmelCase		,	output_hidden_states=lowerCAmelCase							)




	def       __A					(		self	:						int							):


		'''simple docstring'''







		UpperCAmelCase_       				=			self.model_tester.prepare_config_and_inputs()
		self.model_tester.create_and_check_maskaformer_instance_segmentation_head_model(*lowerCAmelCase							)




	@unittest.skip(reason="Mask2Former does not use inputs_embeds"							)
	def       __A					(		self	:						List[Any]							):


		'''simple docstring'''







		pass




	@unittest.skip(reason="Mask2Former does not have a get_input_embeddings method"							)
	def       __A					(		self	:						List[str]							):


		'''simple docstring'''







		pass




	@unittest.skip(reason="Mask2Former is not a generative model"							)
	def       __A					(		self	:						Optional[Any]							):


		'''simple docstring'''







		pass




	@unittest.skip(reason="Mask2Former does not use token embeddings"							)
	def       __A					(		self	:						Optional[int]							):


		'''simple docstring'''







		pass




	@require_torch_multi_gpu
	@unittest.skip(
	    reason="Mask2Former has some layers using `add_module` which doesn't work well with `nn.DataParallel`"							)
	def       __A					(		self	:						int							):


		'''simple docstring'''







		pass




	@unittest.skip("Will be fixed soon by reducing the size of the model used for common tests."							)
	def       __A					(		self	:						Dict							):


		'''simple docstring'''







		pass




	def       __A					(		self	:						int							):


		'''simple docstring'''







		UpperCAmelCase_       				=			self.model_tester.prepare_config_and_inputs_for_common()

		for model_class in self.all_model_classes:
			UpperCAmelCase_       				=			model_class(lowerCAmelCase							)
			UpperCAmelCase_       				=			inspect.signature(model.forward							)
			# signature.parameters is an OrderedDict => so arg_names order is deterministic
			UpperCAmelCase_       				=			[*signature.parameters.keys()]

			UpperCAmelCase_       				=			["pixel_values"]
			self.assertListEqual(arg_names[:1]		,	lowerCAmelCase							)




	@slow
	def       __A					(		self	:						Any							):


		'''simple docstring'''







		for model_name in ["facebook/mask2former-swin-small-coco-instance"]:
			UpperCAmelCase_       				=			MaskaFormerModel.from_pretrained(lowerCAmelCase							)
			self.assertIsNotNone(lowerCAmelCase							)




	def       __A					(		self	:						List[str]							):


		'''simple docstring'''







		UpperCAmelCase_       				=			(self.model_tester.min_size,) * 2
		UpperCAmelCase_       				=			{
		    "pixel_values": torch.randn((2, 3, *size)		,	device=lowerCAmelCase							),
		    "mask_labels": torch.randn((2, 10, *size)		,	device=lowerCAmelCase							),
		    "class_labels": torch.zeros(2		,	10		,	device=lowerCAmelCase							).long(),
		}
		UpperCAmelCase_       				=			self.model_tester.get_config()

		UpperCAmelCase_       				=			MaskaFormerForUniversalSegmentation(lowerCAmelCase							).to(lowerCAmelCase							)
		UpperCAmelCase_       				=			model(**lowerCAmelCase							)
		self.assertTrue(outputs.loss is not None							)




	def       __A					(		self	:						int							):


		'''simple docstring'''







		UpperCAmelCase_       				=			self.model_tester.prepare_config_and_inputs_for_common()
		self.model_tester.create_and_check_maskaformer_model(lowerCAmelCase		,	**lowerCAmelCase		,	output_hidden_states=lowerCAmelCase							)




	def       __A					(		self	:						Optional[Any]							):


		'''simple docstring'''







		UpperCAmelCase_       				=			self.model_tester.prepare_config_and_inputs_for_common()

		for model_class in self.all_model_classes:
			UpperCAmelCase_       				=			model_class(lowerCAmelCase							).to(lowerCAmelCase							)
			UpperCAmelCase_       				=			model(**lowerCAmelCase		,	output_attentions=lowerCAmelCase							)
			self.assertTrue(outputs.attentions is not None							)




	def       __A					(		self	:						Union[str, Any]							):


		'''simple docstring'''







		if not self.model_tester.is_training:
			return

		UpperCAmelCase_       				=			self.all_model_classes[1]
		UpperCAmelCase_       				=			self.model_tester.prepare_config_and_inputs()

		UpperCAmelCase_       				=			model_class(lowerCAmelCase							)
		model.to(lowerCAmelCase							)
		model.train()

		UpperCAmelCase_       				=			model(lowerCAmelCase		,	mask_labels=lowerCAmelCase		,	class_labels=lowerCAmelCase							).loss
		loss.backward()




	def       __A					(		self	:						Optional[Any]							):


		'''simple docstring'''







		UpperCAmelCase_       				=			self.all_model_classes[1]
		UpperCAmelCase_       				=			self.model_tester.prepare_config_and_inputs()
		UpperCAmelCase_       				=			True
		UpperCAmelCase_       				=			True

		UpperCAmelCase_       				=			model_class(lowerCAmelCase							).to(lowerCAmelCase							)
		model.train()

		UpperCAmelCase_       				=			model(lowerCAmelCase		,	mask_labels=lowerCAmelCase		,	class_labels=lowerCAmelCase							)

		UpperCAmelCase_       				=			outputs.encoder_hidden_states[0]
		encoder_hidden_states.retain_grad()

		UpperCAmelCase_       				=			outputs.pixel_decoder_hidden_states[0]
		pixel_decoder_hidden_states.retain_grad()

		UpperCAmelCase_       				=			outputs.transformer_decoder_hidden_states[0]
		transformer_decoder_hidden_states.retain_grad()

		UpperCAmelCase_       				=			outputs.attentions[0]
		attentions.retain_grad()

		outputs.loss.backward(retain_graph=lowerCAmelCase							)

		self.assertIsNotNone(encoder_hidden_states.grad							)
		self.assertIsNotNone(pixel_decoder_hidden_states.grad							)
		self.assertIsNotNone(transformer_decoder_hidden_states.grad							)
		self.assertIsNotNone(attentions.grad							)


_a:    Any 							=			1e-4



def       __lowerCAmelCase      (						):
	UpperCAmelCase_       				=			Image.open("./tests/fixtures/tests_samples/COCO/000000039769.png"    )
	return image




@require_vision
@slow
class 						__UpperCamelCase    (					unittest.TestCase						):




	@cached_property
	def       __A					(		self	:						Dict							):


		'''simple docstring'''







		return "facebook/mask2former-swin-small-coco-instance"




	@cached_property
	def       __A					(		self	:						Tuple							):


		'''simple docstring'''







		return MaskaFormerImageProcessor.from_pretrained(self.model_checkpoints							) if is_vision_available() else None




	def       __A					(		self	:						Optional[int]							):


		'''simple docstring'''







		UpperCAmelCase_       				=			MaskaFormerModel.from_pretrained(self.model_checkpoints							).to(lowerCAmelCase							)
		UpperCAmelCase_       				=			self.default_image_processor
		UpperCAmelCase_       				=			prepare_img()
		UpperCAmelCase_       				=			image_processor(lowerCAmelCase		,	return_tensors="pt"							).to(lowerCAmelCase							)
		UpperCAmelCase_       				=			inputs["pixel_values"].shape
		# check size is divisible by 32
		self.assertTrue((inputs_shape[-1] % 32) == 0 and (inputs_shape[-2] % 32) == 0							)
		# check size
		self.assertEqual(lowerCAmelCase		,	(1, 3, 384, 384)							)

		with torch.no_grad():
			UpperCAmelCase_       				=			model(**lowerCAmelCase							)

		UpperCAmelCase_       				=			torch.tensor(
		    [[-0.2_790, -1.0_717, -1.1_668], [-0.5_128, -0.3_128, -0.4_987], [-0.5_832, 0.1_971, -0.0_197]]							).to(lowerCAmelCase							)
		self.assertTrue(
		    torch.allclose(
		        outputs.encoder_last_hidden_state[0, 0, :3, :3]		,	lowerCAmelCase		,	atol=lowerCAmelCase							)							)

		UpperCAmelCase_       				=			torch.tensor(
		    [[0.8_973, 1.1_847, 1.1_776], [1.1_934, 1.5_040, 1.5_128], [1.1_153, 1.4_486, 1.4_951]]							).to(lowerCAmelCase							)
		self.assertTrue(
		    torch.allclose(
		        outputs.pixel_decoder_last_hidden_state[0, 0, :3, :3]		,	lowerCAmelCase		,	atol=lowerCAmelCase							)							)

		UpperCAmelCase_       				=			torch.tensor(
		    [[2.1_152, 1.7_000, -0.8_603], [1.5_808, 1.8_004, -0.9_353], [1.6_043, 1.7_495, -0.5_999]]							).to(lowerCAmelCase							)
		self.assertTrue(
		    torch.allclose(
		        outputs.transformer_decoder_last_hidden_state[0, :3, :3]		,	lowerCAmelCase		,	atol=lowerCAmelCase							)							)




	def       __A					(		self	:						str							):


		'''simple docstring'''







		UpperCAmelCase_       				=			MaskaFormerForUniversalSegmentation.from_pretrained(self.model_checkpoints							).to(lowerCAmelCase							).eval()
		UpperCAmelCase_       				=			self.default_image_processor
		UpperCAmelCase_       				=			prepare_img()
		UpperCAmelCase_       				=			image_processor(lowerCAmelCase		,	return_tensors="pt"							).to(lowerCAmelCase							)
		UpperCAmelCase_       				=			inputs["pixel_values"].shape
		# check size is divisible by 32
		self.assertTrue((inputs_shape[-1] % 32) == 0 and (inputs_shape[-2] % 32) == 0							)
		# check size
		self.assertEqual(lowerCAmelCase		,	(1, 3, 384, 384)							)

		with torch.no_grad():
			UpperCAmelCase_       				=			model(**lowerCAmelCase							)
		# masks_queries_logits
		UpperCAmelCase_       				=			outputs.masks_queries_logits
		self.assertEqual(
		    masks_queries_logits.shape		,	(1, model.config.num_queries, inputs_shape[-2] // 4, inputs_shape[-1] // 4)							)
		UpperCAmelCase_       				=			[
		    [-8.7_839, -9.0_056, -8.8_121],
		    [-7.4_104, -7.0_313, -6.5_401],
		    [-6.6_105, -6.3_427, -6.4_675],
		]
		UpperCAmelCase_       				=			torch.tensor(lowerCAmelCase							).to(lowerCAmelCase							)
		self.assertTrue(torch.allclose(masks_queries_logits[0, 0, :3, :3]		,	lowerCAmelCase		,	atol=lowerCAmelCase							)							)
		# class_queries_logits
		UpperCAmelCase_       				=			outputs.class_queries_logits
		self.assertEqual(class_queries_logits.shape		,	(1, model.config.num_queries, model.config.num_labels + 1)							)
		UpperCAmelCase_       				=			torch.tensor(
		    [
		        [1.8_324, -8.0_835, -4.1_922],
		        [0.8_450, -9.0_050, -3.6_053],
		        [0.3_045, -7.7_293, -3.0_275],
		    ]							).to(lowerCAmelCase							)
		self.assertTrue(torch.allclose(outputs.class_queries_logits[0, :3, :3]		,	lowerCAmelCase		,	atol=lowerCAmelCase							)							)




	def       __A					(		self	:						Union[str, Any]							):


		'''simple docstring'''







		UpperCAmelCase_       				=			MaskaFormerForUniversalSegmentation.from_pretrained(self.model_checkpoints							).to(lowerCAmelCase							).eval()
		UpperCAmelCase_       				=			self.default_image_processor

		UpperCAmelCase_       				=			image_processor(
		    [np.zeros((3, 800, 1_333)							), np.zeros((3, 800, 1_333)							)]		,	segmentation_maps=[np.zeros((384, 384)							).astype(np.floataa							), np.zeros((384, 384)							).astype(np.floataa							)]		,	return_tensors="pt"		,	)

		UpperCAmelCase_       				=			inputs["pixel_values"].to(lowerCAmelCase							)
		UpperCAmelCase_       				=			[el.to(lowerCAmelCase							) for el in inputs["mask_labels"]]
		UpperCAmelCase_       				=			[el.to(lowerCAmelCase							) for el in inputs["class_labels"]]

		with torch.no_grad():
			UpperCAmelCase_       				=			model(**lowerCAmelCase							)

		self.assertTrue(outputs.loss is not None							)


import warnings
from typing import Dict, List, Optional, Tuple

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


_a:    Dict 							=			logging.get_logger(__name__)

class 						__UpperCamelCase    (					lowercase						):
	SCREAMING_SNAKE_CASE__    				=     ['input_ids', 'attention_mask']




	def __init__(		self	:						Tuple		,	lowerCAmelCase	:						Union[str, Any]="</s>"		,	lowerCAmelCase	:						str="<unk>"		,	lowerCAmelCase	:						Optional[int]="<pad>"		,	lowerCAmelCase	:						List[str]=125		,	lowerCAmelCase	:						List[Any]=None		,	**lowerCAmelCase	:						Optional[Any]		,	):


		'''simple docstring'''







		if extra_ids > 0 and additional_special_tokens is None:
			UpperCAmelCase_       				=			[F"<extra_id_{i}>" for i in range(lowerCAmelCase							)]
		elif extra_ids > 0 and additional_special_tokens is not None:
			# Check that we have the right number of extra_id special tokens
			UpperCAmelCase_       				=			len(set(filter(lambda lowerCAmelCase							: bool("extra_id" in str(lowerCAmelCase							)							)		,	lowerCAmelCase							)							)							)
			if extra_tokens != extra_ids:
				raise ValueError(
				    F"Both extra_ids ({extra_ids}) and additional_special_tokens ({additional_special_tokens}) are"
				    " provided to ByT5Tokenizer. In this case the additional_special_tokens must include the"
				    " extra_ids tokens"							)

		UpperCAmelCase_       				=			AddedToken(lowerCAmelCase		,	lstrip=lowerCAmelCase		,	rstrip=lowerCAmelCase							) if isinstance(lowerCAmelCase		,	lowerCAmelCase							) else pad_token
		UpperCAmelCase_       				=			AddedToken(lowerCAmelCase		,	lstrip=lowerCAmelCase		,	rstrip=lowerCAmelCase							) if isinstance(lowerCAmelCase		,	lowerCAmelCase							) else eos_token
		UpperCAmelCase_       				=			AddedToken(lowerCAmelCase		,	lstrip=lowerCAmelCase		,	rstrip=lowerCAmelCase							) if isinstance(lowerCAmelCase		,	lowerCAmelCase							) else unk_token

		super().__init__(
		    eos_token=lowerCAmelCase		,	unk_token=lowerCAmelCase		,	pad_token=lowerCAmelCase		,	extra_ids=lowerCAmelCase		,	additional_special_tokens=lowerCAmelCase		,	**lowerCAmelCase		,	)

		UpperCAmelCase_       				=			extra_ids

		UpperCAmelCase_       				=			2**8  # utf is 8 bits

		# define special tokens dict
		UpperCAmelCase_       				=			{
		    self.pad_token: 0,
		    self.eos_token: 1,
		    self.unk_token: 2,
		}
		UpperCAmelCase_       				=			len(self.special_tokens_encoder							)
		UpperCAmelCase_       				=			len(lowerCAmelCase							)
		for i, token in enumerate(lowerCAmelCase							):
			UpperCAmelCase_       				=			self.vocab_size + i - n
		UpperCAmelCase_       				=			{v: k for k, v in self.special_tokens_encoder.items()}




	@property
	def       __A					(		self	:						Optional[Any]							):


		'''simple docstring'''







		return self._utf_vocab_size + self._num_special_tokens + self._extra_ids




	def       __A					(		self	:						List[Any]		,	lowerCAmelCase	:						List[int]		,	lowerCAmelCase	:						Optional[List[int]] = None		,	lowerCAmelCase	:						bool = False							):


		'''simple docstring'''







		if already_has_special_tokens:
			return super().get_special_tokens_mask(
			    token_ids_a=lowerCAmelCase		,	token_ids_a=lowerCAmelCase		,	already_has_special_tokens=lowerCAmelCase							)

		# normal case: some special tokens
		if token_ids_a is None:
			return ([0] * len(lowerCAmelCase							)) + [1]
		return ([0] * len(lowerCAmelCase							)) + [1] + ([0] * len(lowerCAmelCase							)) + [1]




	def       __A					(		self	:						Any		,	lowerCAmelCase	:						List[int]							):


		'''simple docstring'''







		if len(lowerCAmelCase							) > 0 and token_ids[-1] == self.eos_token_id:
			warnings.warn(
			    F"This sequence already has {self.eos_token}. In future versions this behavior may lead to duplicated"
			    " eos tokens being added."							)
			return token_ids
		else:
			return token_ids + [self.eos_token_id]




	def       __A					(		self	:						Optional[Any]		,	lowerCAmelCase	:						List[int]		,	lowerCAmelCase	:						Optional[List[int]] = None							):


		'''simple docstring'''







		UpperCAmelCase_       				=			[self.eos_token_id]

		if token_ids_a is None:
			return len(token_ids_a + eos							) * [0]
		return len(token_ids_a + eos + token_ids_a + eos							) * [0]




	def       __A					(		self	:						Tuple		,	lowerCAmelCase	:						List[int]		,	lowerCAmelCase	:						Optional[List[int]] = None							):


		'''simple docstring'''







		UpperCAmelCase_       				=			self._add_eos_if_not_present(lowerCAmelCase							)
		if token_ids_a is None:
			return token_ids_a
		else:
			UpperCAmelCase_       				=			self._add_eos_if_not_present(lowerCAmelCase							)
			return token_ids_a + token_ids_a




	def       __A					(		self	:						List[str]		,	lowerCAmelCase	:						str							):


		'''simple docstring'''







		UpperCAmelCase_       				=			[chr(lowerCAmelCase							) for i in text.encode("utf-8"							)]
		return tokens




	def       __A					(		self	:						Optional[Any]		,	lowerCAmelCase	:						List[Any]							):


		'''simple docstring'''







		if token in self.special_tokens_encoder:
			UpperCAmelCase_       				=			self.special_tokens_encoder[token]
		elif token in self.added_tokens_encoder:
			UpperCAmelCase_       				=			self.added_tokens_encoder[token]
		elif len(lowerCAmelCase							) != 1:
			UpperCAmelCase_       				=			self.unk_token_id
		else:
			UpperCAmelCase_       				=			ord(lowerCAmelCase							) + self._num_special_tokens
		return token_id




	def       __A					(		self	:						str		,	lowerCAmelCase	:						Any							):


		'''simple docstring'''







		if index in self.special_tokens_decoder:
			UpperCAmelCase_       				=			self.special_tokens_decoder[index]
		else:
			UpperCAmelCase_       				=			chr(index - self._num_special_tokens							)
		return token




	def       __A					(		self	:						Union[str, Any]		,	lowerCAmelCase	:						Union[str, Any]							):


		'''simple docstring'''







		UpperCAmelCase_       				=			b""
		for token in tokens:
			if token in self.special_tokens_decoder:
				UpperCAmelCase_       				=			self.special_tokens_decoder[token].encode("utf-8"							)
			elif token in self.added_tokens_decoder:
				UpperCAmelCase_       				=			self.special_tokens_decoder[token].encode("utf-8"							)
			elif token in self.special_tokens_encoder:
				UpperCAmelCase_       				=			token.encode("utf-8"							)
			elif token in self.added_tokens_encoder:
				UpperCAmelCase_       				=			token.encode("utf-8"							)
			else:
				UpperCAmelCase_       				=			bytes([ord(lowerCAmelCase							)]							)
			bstring += tok_string
		UpperCAmelCase_       				=			bstring.decode("utf-8"		,	errors="ignore"							)
		return string




	def       __A					(		self	:						List[Any]		,	lowerCAmelCase	:						str		,	lowerCAmelCase	:						Optional[str] = None							):


		'''simple docstring'''







		return ()