




import json
import os
import unittest

from transformers import CLIPTokenizer, CLIPTokenizerFast
from transformers.models.clip.tokenization_clip import VOCAB_FILES_NAMES
from transformers.testing_utils import require_ftfy, require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin

@require_tokenizers
class 						__lowerCAmelCase      (      __magic_name__ ,			unittest.TestCase					):
							UpperCamelCase__   	=    CLIPTokenizer
							UpperCamelCase__   	=    CLIPTokenizerFast
							UpperCamelCase__   	=    True
							UpperCamelCase__   	=    {}
							UpperCamelCase__   	=    False
							def  lowerCamelCase__				(		self		:Union[str, Any]							):







								'''simple docstring'''







								super().setUp()

								# fmt: off
								a									=				["""l""", """o""", """w""", """e""", """r""", """s""", """t""", """i""", """d""", """n""", """lo""", """l</w>""", """w</w>""", """r</w>""", """t</w>""", """low</w>""", """er</w>""", """lowest</w>""", """newer</w>""", """wider""", """<unk>""", """<|startoftext|>""", """<|endoftext|>"""]
								# fmt: on
								a									=				dict(zip(__magic_name__						, range(len(__magic_name__							)							)							)							)
								a									=				["""#version: 0.2""", """l o""", """lo w</w>""", """e r</w>"""]
								a									=				{"""unk_token""": """<unk>"""}

								a									=				os.path.join(self.tmpdirname						, VOCAB_FILES_NAMES["""vocab_file"""]							)
								a									=				os.path.join(self.tmpdirname						, VOCAB_FILES_NAMES["""merges_file"""]							)
								with open(self.vocab_file						, """w"""						, encoding="""utf-8"""							) as fp:
									fp.write(json.dumps(__magic_name__							) + """\n"""							)
								with open(self.merges_file						, """w"""						, encoding="""utf-8"""							) as fp:
									fp.write("""\n""".join(__magic_name__							)							)
							def  lowerCamelCase__				(		self		:Dict						, **__magic_name__		:Optional[int]							):







								'''simple docstring'''







								kwargs.update(self.special_tokens_map							)
								return CLIPTokenizer.from_pretrained(self.tmpdirname						, **__magic_name__							)
							def  lowerCamelCase__				(		self		:str						, **__magic_name__		:Tuple							):







								'''simple docstring'''







								kwargs.update(self.special_tokens_map							)
								return CLIPTokenizerFast.from_pretrained(self.tmpdirname						, **__magic_name__							)
							def  lowerCamelCase__				(		self		:Optional[Any]						, __magic_name__		:Any							):







								'''simple docstring'''







								a									=				"""lower newer"""
								a									=				"""lower newer"""
								return input_text, output_text
							def  lowerCamelCase__				(		self		:Tuple							):







								'''simple docstring'''







								a									=				CLIPTokenizer(self.vocab_file						, self.merges_file						, **self.special_tokens_map							)
								a									=				"""lower newer"""
								a									=				["""lo""", """w""", """er</w>""", """n""", """e""", """w""", """er</w>"""]
								a									=				tokenizer.tokenize(__magic_name__							)
								self.assertListEqual(__magic_name__						, __magic_name__							)

								a									=				tokens + [tokenizer.unk_token]
								a									=				[10, 2, 16, 9, 3, 2, 16, 20]
								self.assertListEqual(tokenizer.convert_tokens_to_ids(__magic_name__							)						, __magic_name__							)
							@require_ftfy
							def  lowerCamelCase__				(		self		:List[str]							):







								'''simple docstring'''







								for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
									with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'							):
										a									=				self.tokenizer_class.from_pretrained(__magic_name__						, **__magic_name__							)
										a									=				self.rust_tokenizer_class.from_pretrained(__magic_name__						, **__magic_name__							)

										a									=				"""A\n'll 11p223RF☆ho!!to?'d'd''d of a cat to-$''d."""
										a									=				tokenizer_s.tokenize(__magic_name__							)
										a									=				tokenizer_r.tokenize(__magic_name__							)

										self.assertListEqual(__magic_name__						, __magic_name__							)

										# Test that the tokenization is identical on an example containing a character (Latin Small Letter A
										# with Tilde) encoded in 2 different ways
										a									=				"""xa\u0303y""" + """ """ + """x\xe3y"""
										a									=				tokenizer_s.tokenize(__magic_name__							)
										a									=				tokenizer_r.tokenize(__magic_name__							)

										self.assertListEqual(__magic_name__						, __magic_name__							)

										# Test that the tokenization is identical on unicode of space type
										a									=				[
										    """\u0009""",  # (horizontal tab, '\t')
										    """\u000B""",  # (vertical tab)
										    """\u000C""",  # (form feed)
										    """\u0020""",  # (space, ' ')
										    """\u200E""",  # (left-to-right mark):w
										    """\u200F""",  # (right-to-left mark)
										]
										for unicode_seq in spaces_unicodes:
											a									=				tokenizer_s.tokenize(__magic_name__							)
											a									=				tokenizer_r.tokenize(__magic_name__							)

											self.assertListEqual(__magic_name__						, __magic_name__							)

										# Test that the tokenization is identical on unicode of line break type
										a									=				[
										    """\u000A""",  # (line feed, '\n')
										    """\r\n""",  # (carriage return and line feed, '\r\n')
										    """\u000D""",  # (carriage return, '\r')
										    """\r""",  # (carriage return, '\r')
										    """\u000D""",  # (carriage return, '\r')
										    """\u2028""",  # (line separator)
										    """\u2029""",  # (paragraph separator)
										    # "\u0085", # (next line)
										]

										# The tokenization is not identical for the character "\u0085" (next line). The slow version using ftfy transforms
										# it into the Horizontal Ellipsis character "…" ("\u2026") while the fast version transforms it into a
										# space (and thus into an empty list).

										for unicode_seq in line_break_unicodes:
											a									=				tokenizer_s.tokenize(__magic_name__							)
											a									=				tokenizer_r.tokenize(__magic_name__							)

											self.assertListEqual(__magic_name__						, __magic_name__							)
							def  lowerCamelCase__				(		self		:List[str]							):







								'''simple docstring'''







								for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
									with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'							):
										a									=				"""hello"""  # `hello` is a token in the vocabulary of `pretrained_name`
										a									=				F'{text_of_1_token} {text_of_1_token}'

										a									=				self.rust_tokenizer_class.from_pretrained(
										    __magic_name__						, use_fast=__magic_name__						, )
										a									=				tokenizer_r(__magic_name__						, return_offsets_mapping=__magic_name__						, add_special_tokens=__magic_name__							)
										self.assertEqual(encoding.offset_mapping[0]						, (0, len(__magic_name__							))							)
										self.assertEqual(
										    encoding.offset_mapping[1]						, (len(__magic_name__							) + 1, len(__magic_name__							) + 1 + len(__magic_name__							))						, )

										a									=				F' {text}'

										a									=				self.rust_tokenizer_class.from_pretrained(
										    __magic_name__						, use_fast=__magic_name__						, )
										a									=				tokenizer_r(__magic_name__						, return_offsets_mapping=__magic_name__						, add_special_tokens=__magic_name__							)
										self.assertEqual(encoding.offset_mapping[0]						, (1, 1 + len(__magic_name__							))							)
										self.assertEqual(
										    encoding.offset_mapping[1]						, (1 + len(__magic_name__							) + 1, 1 + len(__magic_name__							) + 1 + len(__magic_name__							))						, )
							def  lowerCamelCase__				(		self		:Union[str, Any]							):







								'''simple docstring'''







								with self.assertRaises(__magic_name__							) as context:
									self.rust_tokenizer_class.from_pretrained("""robot-test/old-clip-tokenizer"""							)

								self.assertTrue(
								    context.exception.args[0].startswith(
								        """The `backend_tokenizer` provided does not match the expected format."""							)							)
							@require_ftfy
							def  lowerCamelCase__				(		self		:str							):







								'''simple docstring'''







								super().test_tokenization_python_rust_equals()





							def  lowerCamelCase__				(		self		:List[Any]							):







								'''simple docstring'''







								pass







import copy

from ...configuration_utils import PretrainedConfig
from ...utils import logging
from ..auto import CONFIG_MAPPING


__UpperCamelCase   :			Any          = logging.get_logger(__name__)

__UpperCamelCase   :			Tuple          = {
    "SenseTime/deformable-detr": "https://huggingface.co/sensetime/deformable-detr/resolve/main/config.json",
    # See all Deformable DETR models at https://huggingface.co/models?filter=deformable-detr
}

class 						__lowerCAmelCase      (      __magic_name__					):
							UpperCamelCase__   	=    '''deformable_detr'''
							UpperCamelCase__   	=    {
							    '''hidden_size''': '''d_model''',
							    '''num_attention_heads''': '''encoder_attention_heads''',
							}
							def __init__(		self		:List[str]						, __magic_name__		:Tuple=True						, __magic_name__		:str=None						, __magic_name__		:Tuple=3						, __magic_name__		:List[str]=300						, __magic_name__		:Dict=1024						, __magic_name__		:Tuple=6						, __magic_name__		:Tuple=1024						, __magic_name__		:int=8						, __magic_name__		:List[Any]=6						, __magic_name__		:Tuple=1024						, __magic_name__		:List[Any]=8						, __magic_name__		:Union[str, Any]=0.0						, __magic_name__		:Union[str, Any]=True						, __magic_name__		:str="relu"						, __magic_name__		:Optional[Any]=256						, __magic_name__		:int=0.1						, __magic_name__		:List[str]=0.0						, __magic_name__		:str=0.0						, __magic_name__		:str=0.02						, __magic_name__		:int=1.0						, __magic_name__		:List[str]=True						, __magic_name__		:Union[str, Any]=False						, __magic_name__		:List[Any]="sine"						, __magic_name__		:Union[str, Any]="resnet50"						, __magic_name__		:int=True						, __magic_name__		:Any=False						, __magic_name__		:List[str]=4						, __magic_name__		:Tuple=4						, __magic_name__		:Union[str, Any]=4						, __magic_name__		:Optional[int]=False						, __magic_name__		:Tuple=300						, __magic_name__		:Tuple=False						, __magic_name__		:List[Any]=1						, __magic_name__		:List[str]=5						, __magic_name__		:List[Any]=2						, __magic_name__		:Optional[int]=1						, __magic_name__		:Union[str, Any]=1						, __magic_name__		:List[Any]=5						, __magic_name__		:Optional[Any]=2						, __magic_name__		:Optional[int]=0.1						, __magic_name__		:List[str]=0.25						, __magic_name__		:str=False						, **__magic_name__		:List[str]						, ):







								'''simple docstring'''







								if backbone_config is not None and use_timm_backbone:
									raise ValueError("""You can't specify both `backbone_config` and `use_timm_backbone`."""							)

								if not use_timm_backbone:
									if backbone_config is None:
										logger.info("""`backbone_config` is `None`. Initializing the config with the default `ResNet` backbone."""							)
										a									=				CONFIG_MAPPING["""resnet"""](out_features=["""stage4"""]							)
									elif isinstance(__magic_name__						, __magic_name__							):
										a									=				backbone_config.get("""model_type"""							)
										a									=				CONFIG_MAPPING[backbone_model_type]
										a									=				config_class.from_dict(__magic_name__							)
								a									=				use_timm_backbone
								a									=				backbone_config
								a									=				num_channels
								a									=				num_queries
								a									=				max_position_embeddings
								a									=				d_model
								a									=				encoder_ffn_dim
								a									=				encoder_layers
								a									=				encoder_attention_heads
								a									=				decoder_ffn_dim
								a									=				decoder_layers
								a									=				decoder_attention_heads
								a									=				dropout
								a									=				attention_dropout
								a									=				activation_dropout
								a									=				activation_function
								a									=				init_std
								a									=				init_xavier_std
								a									=				encoder_layerdrop
								a									=				auxiliary_loss
								a									=				position_embedding_type
								a									=				backbone
								a									=				use_pretrained_backbone
								a									=				dilation
								# deformable attributes
								a									=				num_feature_levels
								a									=				encoder_n_points
								a									=				decoder_n_points
								a									=				two_stage
								a									=				two_stage_num_proposals
								a									=				with_box_refine
								if two_stage is True and with_box_refine is False:
									raise ValueError("""If two_stage is True, with_box_refine must be True."""							)
								# Hungarian matcher
								a									=				class_cost
								a									=				bbox_cost
								a									=				giou_cost
								# Loss coefficients
								a									=				mask_loss_coefficient
								a									=				dice_loss_coefficient
								a									=				bbox_loss_coefficient
								a									=				giou_loss_coefficient
								a									=				eos_coefficient
								a									=				focal_alpha
								a									=				disable_custom_kernels
								super().__init__(is_encoder_decoder=__magic_name__						, **__magic_name__							)
							@property
							def  lowerCamelCase__				(		self		:List[Any]							):







								'''simple docstring'''







								return self.encoder_attention_heads
							@property
							def  lowerCamelCase__				(		self		:int							):







								'''simple docstring'''







								return self.d_model





							def  lowerCamelCase__				(		self		:Optional[int]							):







								'''simple docstring'''







								a									=				copy.deepcopy(self.__dict__							)
								if self.backbone_config is not None:
									a									=				self.backbone_config.to_dict()
								a									=				self.__class__.model_type
								return output

