




import json
import os
import re
import shutil
import tempfile
import unittest
from typing import Tuple

from transformers import AddedToken, BatchEncoding, ByTaTokenizer
from transformers.utils import cached_property, is_tf_available, is_torch_available

from ...test_tokenization_common import TokenizerTesterMixin


if is_torch_available():
		__UpperCamelCase   :			List[Any]          = "pt"
elif is_tf_available():
		__UpperCamelCase   :			Union[str, Any]          = "tf"
else:
		__UpperCamelCase   :			Tuple          = "jax"

class 						__lowerCAmelCase      (      __magic_name__ ,			unittest.TestCase					):
							UpperCamelCase__   	=    ByTaTokenizer
							UpperCamelCase__   	=    False
							def  lowerCamelCase__				(		self		:Optional[int]							):







								'''simple docstring'''







								super().setUp()
								a									=				ByTaTokenizer()
								tokenizer.save_pretrained(self.tmpdirname							)
							@cached_property
							def  lowerCamelCase__				(		self		:Optional[Any]							):







								'''simple docstring'''







								return ByTaTokenizer.from_pretrained("""google/byt5-small"""							)
							def  lowerCamelCase__				(		self		:Any						, **__magic_name__		:Tuple							):







								'''simple docstring'''







								return self.tokenizer_class.from_pretrained(self.tmpdirname						, **__magic_name__							)
							def  lowerCamelCase__				(		self		:int						, __magic_name__		:Dict						, __magic_name__		:Any=False						, __magic_name__		:Union[str, Any]=20						, __magic_name__		:Optional[int]=5							):







								'''simple docstring'''







								a									=				[]
								for i in range(len(__magic_name__							)							):
									try:
										a									=				tokenizer.decode([i]						, clean_up_tokenization_spaces=__magic_name__							)
									except UnicodeDecodeError:
										pass
									toks.append((i, tok)							)

								a									=				list(filter(lambda __magic_name__							: re.match(r"""^[ a-zA-Z]+$"""						, t[1]							)						, __magic_name__							)							)
								a									=				list(filter(lambda __magic_name__							: [t[0]] == tokenizer.encode(t[1]						, add_special_tokens=__magic_name__							)						, __magic_name__							)							)
								if max_length is not None and len(__magic_name__							) > max_length:
									a									=				toks[:max_length]
								if min_length is not None and len(__magic_name__							) < min_length and len(__magic_name__							) > 0:
									while len(__magic_name__							) < min_length:
										a									=				toks + toks
        # toks_str = [t[1] for t in toks]
								a									=				[t[0] for t in toks]

								# Ensure consistency
								a									=				tokenizer.decode(__magic_name__						, clean_up_tokenization_spaces=__magic_name__							)
								if " " not in output_txt and len(__magic_name__							) > 1:
									a									=				(
									    tokenizer.decode([toks_ids[0]]						, clean_up_tokenization_spaces=__magic_name__							)
									    + """ """
									    + tokenizer.decode(toks_ids[1:]						, clean_up_tokenization_spaces=__magic_name__							)
									)
								if with_prefix_space:
									a									=				""" """ + output_txt
								a									=				tokenizer.encode(__magic_name__						, add_special_tokens=__magic_name__							)
								return output_txt, output_ids
							def  lowerCamelCase__				(		self		:Any							):







								'''simple docstring'''







								a									=				self.ta_base_tokenizer
								a									=				tokenizer(["""hi</s>""", """I went to the gym</s>""", """</s>"""]							)
								a									=				tokenizer(["""hi""", """I went to the gym""", """"""]							)
								self.assertListEqual(batch_with_eos_added["""input_ids"""]						, batch_without_eos_added["""input_ids"""]							)
							def  lowerCamelCase__				(		self		:Any							):







								'''simple docstring'''







								a									=				self.ta_base_tokenizer
								a									=				"""Unicode €."""
								a									=				tokenizer(__magic_name__							)
								a									=				[88, 113, 108, 102, 114, 103, 104, 35, 229, 133, 175, 49, 1]
								self.assertEqual(encoded["""input_ids"""]						, __magic_name__							)

								# decoding
								a									=				tokenizer.decode(__magic_name__							)
								self.assertEqual(__magic_name__						, """Unicode €.</s>"""							)

								a									=				tokenizer("""e è é ê ë"""							)
								a									=				[104, 35, 198, 171, 35, 198, 172, 35, 198, 173, 35, 198, 174, 1]
								self.assertEqual(encoded["""input_ids"""]						, __magic_name__							)
								# decoding
								a									=				tokenizer.decode(__magic_name__							)
								self.assertEqual(__magic_name__						, """e è é ê ë</s>"""							)

								# encode/decode, but with `encode` instead of `__call__`
								self.assertEqual(tokenizer.decode(tokenizer.encode("""e è é ê ë"""							)							)						, """e è é ê ë</s>"""							)
							def  lowerCamelCase__				(		self		:Tuple							):







								'''simple docstring'''







								a									=				self.ta_base_tokenizer
								a									=				["""A long paragraph for summarization.""", """Another paragraph for summarization."""]
								# fmt: off
								a									=				[68, 35, 111, 114, 113, 106, 35, 115, 100, 117, 100, 106, 117, 100, 115, 107, 35, 105, 114, 117, 35, 118, 120, 112, 112, 100, 117, 108, 125, 100, 119, 108, 114, 113, 49, 1, 0]
								# fmt: on
								a									=				tokenizer(__magic_name__						, padding=__magic_name__						, return_tensors=__magic_name__							)
								self.assertIsInstance(__magic_name__						, __magic_name__							)

								if FRAMEWORK != "jax":
									a									=				list(batch.input_ids.numpy()[0]							)
								else:
									a									=				list(batch.input_ids.tolist()[0]							)

								self.assertListEqual(__magic_name__						, __magic_name__							)

								self.assertEqual((2, 37)						, batch.input_ids.shape							)
								self.assertEqual((2, 37)						, batch.attention_mask.shape							)
							def  lowerCamelCase__				(		self		:Any							):







								'''simple docstring'''







								a									=				self.ta_base_tokenizer
								a									=				["""A long paragraph for summarization.""", """Another paragraph for summarization."""]
								a									=				tokenizer(__magic_name__						, padding=__magic_name__						, return_tensors=__magic_name__							)
								# check if input_ids are returned and no decoder_input_ids
								self.assertIn("""input_ids"""						, __magic_name__							)
								self.assertIn("""attention_mask"""						, __magic_name__							)
								self.assertNotIn("""decoder_input_ids"""						, __magic_name__							)
								self.assertNotIn("""decoder_attention_mask"""						, __magic_name__							)
							def  lowerCamelCase__				(		self		:Dict							):







								'''simple docstring'''







								a									=				self.ta_base_tokenizer
								a									=				[
								    """Summary of the text.""",
								    """Another summary.""",
								]
								a									=				tokenizer(
								    text_target=__magic_name__						, max_length=32						, padding="""max_length"""						, truncation=__magic_name__						, return_tensors=__magic_name__							)
								self.assertEqual(32						, targets["""input_ids"""].shape[1]							)
							def  lowerCamelCase__				(		self		:List[str]							):







								'''simple docstring'''







								a									=				self.ta_base_tokenizer
								a									=				["""A long paragraph for summarization. </s>"""]
								a									=				["""Summary of the text. </s>"""]
								# fmt: off
								a									=				[68, 35, 111, 114, 113, 106, 35, 115, 100, 117, 100, 106, 117, 100, 115, 107, 35, 105, 114, 117, 35, 118, 120, 112, 112, 100, 117, 108, 125, 100, 119, 108, 114, 113, 49, 35, 1]
								a									=				[86, 120, 112, 112, 100, 117, 124, 35, 114, 105, 35, 119, 107, 104, 35, 119, 104, 123, 119, 49, 35, 1]
								# fmt: on

								a									=				tokenizer(__magic_name__						, text_target=__magic_name__							)

								self.assertEqual(__magic_name__						, batch["""input_ids"""][0]							)
								self.assertEqual(__magic_name__						, batch["""labels"""][0]							)
							def  lowerCamelCase__				(		self		:List[Any]							):







								'''simple docstring'''







								a									=				self.get_tokenizers()
								for tokenizer in tokenizers:
									with self.subTest(F'{tokenizer.__class__.__name__}'							):
										self.assertNotEqual(tokenizer.model_max_length						, 42							)

        # Now let's start the test
								a									=				self.get_tokenizers()
								for tokenizer in tokenizers:
									with self.subTest(F'{tokenizer.__class__.__name__}'							):
										# Isolate this from the other tests because we save additional tokens/etc
										a									=				tempfile.mkdtemp()

										a									=				""" He is very happy, UNwant\u00E9d,running"""
										a									=				tokenizer.encode(__magic_name__						, add_special_tokens=__magic_name__							)
										tokenizer.save_pretrained(__magic_name__							)

										a									=				tokenizer.__class__.from_pretrained(__magic_name__							)
										a									=				after_tokenizer.encode(__magic_name__						, add_special_tokens=__magic_name__							)
										self.assertListEqual(__magic_name__						, __magic_name__							)

										shutil.rmtree(__magic_name__							)

								a									=				self.get_tokenizers(model_max_length=42							)
								for tokenizer in tokenizers:
									with self.subTest(F'{tokenizer.__class__.__name__}'							):
										# Isolate this from the other tests because we save additional tokens/etc
										a									=				tempfile.mkdtemp()

										a									=				""" He is very happy, UNwant\u00E9d,running"""
										tokenizer.add_tokens(["""bim""", """bambam"""]							)
										a									=				tokenizer.additional_special_tokens
										additional_special_tokens.append("""new_additional_special_token"""							)
										tokenizer.add_special_tokens({"""additional_special_tokens""": additional_special_tokens}							)
										a									=				tokenizer.encode(__magic_name__						, add_special_tokens=__magic_name__							)
										tokenizer.save_pretrained(__magic_name__							)

										a									=				tokenizer.__class__.from_pretrained(__magic_name__							)
										a									=				after_tokenizer.encode(__magic_name__						, add_special_tokens=__magic_name__							)
										self.assertListEqual(__magic_name__						, __magic_name__							)
										self.assertIn("""new_additional_special_token"""						, after_tokenizer.additional_special_tokens							)
										self.assertEqual(after_tokenizer.model_max_length						, 42							)

										a									=				tokenizer.__class__.from_pretrained(__magic_name__						, model_max_length=43							)
										self.assertEqual(tokenizer.model_max_length						, 43							)

										shutil.rmtree(__magic_name__							)
							def  lowerCamelCase__				(		self		:List[str]							):







								'''simple docstring'''







								a									=				[]
								if self.test_slow_tokenizer:
									tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())							)

								if self.test_rust_tokenizer:
									tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())							)

								for tokenizer_class, tokenizer_utils in tokenizer_list:
									with tempfile.TemporaryDirectory() as tmp_dir:
										tokenizer_utils.save_pretrained(__magic_name__							)

										with open(os.path.join(__magic_name__						, """special_tokens_map.json"""							)						, encoding="""utf-8"""							) as json_file:
											a									=				json.load(__magic_name__							)

										with open(os.path.join(__magic_name__						, """tokenizer_config.json"""							)						, encoding="""utf-8"""							) as json_file:
											a									=				json.load(__magic_name__							)

										a									=				[F'<extra_id_{i}>' for i in range(125							)]

										a									=				added_tokens_extra_ids + [
										    """an_additional_special_token"""
										]
										a									=				added_tokens_extra_ids + [
										    """an_additional_special_token"""
										]

										with open(os.path.join(__magic_name__						, """special_tokens_map.json"""							)						, """w"""						, encoding="""utf-8"""							) as outfile:
											json.dump(__magic_name__						, __magic_name__							)
										with open(os.path.join(__magic_name__						, """tokenizer_config.json"""							)						, """w"""						, encoding="""utf-8"""							) as outfile:
											json.dump(__magic_name__						, __magic_name__							)

										# the following checks allow us to verify that our test works as expected, i.e. that the tokenizer takes
										# into account the new value of additional_special_tokens given in the "tokenizer_config.json" and
										# "special_tokens_map.json" files
										a									=				tokenizer_class.from_pretrained(
										    __magic_name__						, )
										self.assertIn(
										    """an_additional_special_token"""						, tokenizer_without_change_in_init.additional_special_tokens							)
										# self.assertIn("an_additional_special_token",tokenizer_without_change_in_init.get_vocab()) # ByT5Tokenization no vocab
										self.assertEqual(
										    ["""an_additional_special_token"""]						, tokenizer_without_change_in_init.convert_ids_to_tokens(
										        tokenizer_without_change_in_init.convert_tokens_to_ids(["""an_additional_special_token"""]							)							)						, )

										# Now we test that we can change the value of additional_special_tokens in the from_pretrained
										a									=				added_tokens_extra_ids + [AddedToken("""a_new_additional_special_token"""						, lstrip=__magic_name__							)]
										a									=				tokenizer_class.from_pretrained(
										    __magic_name__						, additional_special_tokens=__magic_name__						, )

										self.assertIn("""a_new_additional_special_token"""						, tokenizer.additional_special_tokens							)
										self.assertEqual(
										    ["""a_new_additional_special_token"""]						, tokenizer.convert_ids_to_tokens(
										        tokenizer.convert_tokens_to_ids(["""a_new_additional_special_token"""]							)							)						, )
							def  lowerCamelCase__				(		self		:Any							):







								'''simple docstring'''







								a									=				[]
								if self.test_slow_tokenizer:
									tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())							)

								if self.test_rust_tokenizer:
									tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())							)

								for tokenizer_class, tokenizer_utils in tokenizer_list:
									with tempfile.TemporaryDirectory() as tmp_dir:
										tokenizer_utils.save_pretrained(__magic_name__							)

										a									=				tokenizer_class.from_pretrained(__magic_name__							)

										self.assertTrue(tokenizer.decode([255]							) == """"""							)
							def  lowerCamelCase__				(		self		:List[str]							):







								'''simple docstring'''







								pass
							def  lowerCamelCase__				(		self		:Union[str, Any]							):







								'''simple docstring'''







								pass
							def  lowerCamelCase__				(		self		:Optional[Any]							):







								'''simple docstring'''







								pass
							def  lowerCamelCase__				(		self		:Optional[int]							):







								'''simple docstring'''







								pass
							def  lowerCamelCase__				(		self		:Union[str, Any]							):







								'''simple docstring'''







								a									=				self.get_tokenizers(fast=__magic_name__						, do_lower_case=__magic_name__							)
								for tokenizer in tokenizers:
									with self.subTest(F'{tokenizer.__class__.__name__}'							):
										a									=				["""t""", """h""", """i""", """s""", """ """, """i""", """s""", """ """, """a""", """ """, """t""", """e""", """x""", """t""", """</s>"""]
										a									=				tokenizer.convert_tokens_to_string(__magic_name__							)

										self.assertIsInstance(__magic_name__						, __magic_name__							)





							def  lowerCamelCase__				(		self		:Optional[int]							):







								'''simple docstring'''







								a									=				self.get_tokenizers()
								for tokenizer in tokenizers:
									with self.subTest(F'{tokenizer.__class__.__name__}'							):
										a									=				[
										    """bos_token""",
										    """eos_token""",
										    """unk_token""",
										    """sep_token""",
										    """pad_token""",
										    """cls_token""",
										    """mask_token""",
										]

										a									=				0
										a									=				tokenizer.convert_ids_to_tokens(
										    __magic_name__						, skip_special_tokens=__magic_name__							)

										for attr in attributes_list:
											setattr(__magic_name__						, attr + """_id"""						, __magic_name__							)
											self.assertEqual(getattr(__magic_name__						, __magic_name__							)						, __magic_name__							)
											self.assertEqual(getattr(__magic_name__						, attr + """_id"""							)						, __magic_name__							)

											setattr(__magic_name__						, attr + """_id"""						, __magic_name__							)
											self.assertEqual(getattr(__magic_name__						, __magic_name__							)						, __magic_name__							)
											self.assertEqual(getattr(__magic_name__						, attr + """_id"""							)						, __magic_name__							)

										setattr(__magic_name__						, """additional_special_tokens_ids"""						, []							)
										self.assertListEqual(getattr(__magic_name__						, """additional_special_tokens"""							)						, []							)
										self.assertListEqual(getattr(__magic_name__						, """additional_special_tokens_ids"""							)						, []							)

										setattr(__magic_name__						, """additional_special_tokens_ids"""						, [token_id_to_test_setters]							)
										self.assertListEqual(getattr(__magic_name__						, """additional_special_tokens"""							)						, [token_to_test_setters]							)
										self.assertListEqual(getattr(__magic_name__						, """additional_special_tokens_ids"""							)						, [token_id_to_test_setters]							)







import copy

from ...configuration_utils import PretrainedConfig
from ...utils import logging
from ..auto import CONFIG_MAPPING


__UpperCamelCase   :			Any          = logging.get_logger(__name__)

__UpperCamelCase   :			Tuple          = {
    "SenseTime/deformable-detr": "https://huggingface.co/sensetime/deformable-detr/resolve/main/config.json",
    # See all Deformable DETR models at https://huggingface.co/models?filter=deformable-detr
}

class 						__lowerCAmelCase      (      __magic_name__					):
							UpperCamelCase__   	=    '''deformable_detr'''
							UpperCamelCase__   	=    {
							    '''hidden_size''': '''d_model''',
							    '''num_attention_heads''': '''encoder_attention_heads''',
							}
							def __init__(		self		:List[str]						, __magic_name__		:Tuple=True						, __magic_name__		:str=None						, __magic_name__		:Tuple=3						, __magic_name__		:List[str]=300						, __magic_name__		:Dict=1024						, __magic_name__		:Tuple=6						, __magic_name__		:Tuple=1024						, __magic_name__		:int=8						, __magic_name__		:List[Any]=6						, __magic_name__		:Tuple=1024						, __magic_name__		:List[Any]=8						, __magic_name__		:Union[str, Any]=0.0						, __magic_name__		:Union[str, Any]=True						, __magic_name__		:str="relu"						, __magic_name__		:Optional[Any]=256						, __magic_name__		:int=0.1						, __magic_name__		:List[str]=0.0						, __magic_name__		:str=0.0						, __magic_name__		:str=0.02						, __magic_name__		:int=1.0						, __magic_name__		:List[str]=True						, __magic_name__		:Union[str, Any]=False						, __magic_name__		:List[Any]="sine"						, __magic_name__		:Union[str, Any]="resnet50"						, __magic_name__		:int=True						, __magic_name__		:Any=False						, __magic_name__		:List[str]=4						, __magic_name__		:Tuple=4						, __magic_name__		:Union[str, Any]=4						, __magic_name__		:Optional[int]=False						, __magic_name__		:Tuple=300						, __magic_name__		:Tuple=False						, __magic_name__		:List[Any]=1						, __magic_name__		:List[str]=5						, __magic_name__		:List[Any]=2						, __magic_name__		:Optional[int]=1						, __magic_name__		:Union[str, Any]=1						, __magic_name__		:List[Any]=5						, __magic_name__		:Optional[Any]=2						, __magic_name__		:Optional[int]=0.1						, __magic_name__		:List[str]=0.25						, __magic_name__		:str=False						, **__magic_name__		:List[str]						, ):







								'''simple docstring'''







								if backbone_config is not None and use_timm_backbone:
									raise ValueError("""You can't specify both `backbone_config` and `use_timm_backbone`."""							)

								if not use_timm_backbone:
									if backbone_config is None:
										logger.info("""`backbone_config` is `None`. Initializing the config with the default `ResNet` backbone."""							)
										a									=				CONFIG_MAPPING["""resnet"""](out_features=["""stage4"""]							)
									elif isinstance(__magic_name__						, __magic_name__							):
										a									=				backbone_config.get("""model_type"""							)
										a									=				CONFIG_MAPPING[backbone_model_type]
										a									=				config_class.from_dict(__magic_name__							)
								a									=				use_timm_backbone
								a									=				backbone_config
								a									=				num_channels
								a									=				num_queries
								a									=				max_position_embeddings
								a									=				d_model
								a									=				encoder_ffn_dim
								a									=				encoder_layers
								a									=				encoder_attention_heads
								a									=				decoder_ffn_dim
								a									=				decoder_layers
								a									=				decoder_attention_heads
								a									=				dropout
								a									=				attention_dropout
								a									=				activation_dropout
								a									=				activation_function
								a									=				init_std
								a									=				init_xavier_std
								a									=				encoder_layerdrop
								a									=				auxiliary_loss
								a									=				position_embedding_type
								a									=				backbone
								a									=				use_pretrained_backbone
								a									=				dilation
								# deformable attributes
								a									=				num_feature_levels
								a									=				encoder_n_points
								a									=				decoder_n_points
								a									=				two_stage
								a									=				two_stage_num_proposals
								a									=				with_box_refine
								if two_stage is True and with_box_refine is False:
									raise ValueError("""If two_stage is True, with_box_refine must be True."""							)
								# Hungarian matcher
								a									=				class_cost
								a									=				bbox_cost
								a									=				giou_cost
								# Loss coefficients
								a									=				mask_loss_coefficient
								a									=				dice_loss_coefficient
								a									=				bbox_loss_coefficient
								a									=				giou_loss_coefficient
								a									=				eos_coefficient
								a									=				focal_alpha
								a									=				disable_custom_kernels
								super().__init__(is_encoder_decoder=__magic_name__						, **__magic_name__							)
							@property
							def  lowerCamelCase__				(		self		:List[Any]							):







								'''simple docstring'''







								return self.encoder_attention_heads
							@property
							def  lowerCamelCase__				(		self		:int							):







								'''simple docstring'''







								return self.d_model





							def  lowerCamelCase__				(		self		:Optional[int]							):







								'''simple docstring'''







								a									=				copy.deepcopy(self.__dict__							)
								if self.backbone_config is not None:
									a									=				self.backbone_config.to_dict()
								a									=				self.__class__.model_type
								return output

