






'''simple docstring'''


import os
import re
import warnings
from shutil import copyfile
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer


if TYPE_CHECKING:
					from ...tokenization_utils_base import TextInput
from ...utils import logging


lowerCamelCase_				: int    	=							logging.get_logger(__name__)

lowerCamelCase_				: Union[str, Any]    	=							{'''vocab_file''': '''spiece.model'''}

lowerCamelCase_				: Tuple    	=							{
    '''vocab_file''': {
        '''t5-small''': '''https://huggingface.co/t5-small/resolve/main/spiece.model''',
        '''t5-base''': '''https://huggingface.co/t5-base/resolve/main/spiece.model''',
        '''t5-large''': '''https://huggingface.co/t5-large/resolve/main/spiece.model''',
        '''t5-3b''': '''https://huggingface.co/t5-3b/resolve/main/spiece.model''',
        '''t5-11b''': '''https://huggingface.co/t5-11b/resolve/main/spiece.model''',
    }
}


# TODO(PVP) - this should be removed in Transformers v5
lowerCamelCase_				: str    	=							{
    '''t5-small''': 512,
    '''t5-base''': 512,
    '''t5-large''': 512,
    '''t5-3b''': 512,
    '''t5-11b''': 512,
}

lowerCamelCase_				: Union[str, Any]    	=							'''▁'''
class    _SCREAMING_SNAKE_CASE							(		SCREAMING_SNAKE_CASE	):


			'''simple docstring'''





			__a							:							Optional[Any]					     =					VOCAB_FILES_NAMES
			__a							:							Tuple					     =					PRETRAINED_VOCAB_FILES_MAP
			__a							:							str					     =					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			__a							:							Optional[int]					     =					["input_ids", "attention_mask"]






			def __init__(   self							:						Optional[int] ,      lowercase							:						Tuple ,      lowercase							:						List[Any]="</s>" ,      lowercase							:						str="<unk>" ,      lowercase							:						Any="<pad>" ,      lowercase							:						str=1_0_0 ,      lowercase							:						Optional[Any]=None ,      lowercase							:						Optional[Dict[str, Any]] = None ,      lowercase							:						Tuple=True ,      **lowercase							:						Any ,      )							->    None:

									'''simple docstring'''







									if extra_ids > 0 and additional_special_tokens is None:
															UpperCamelCase__         = [f"<extra_id_{i}>" for i in range(lowercase			)]
									elif extra_ids > 0 and additional_special_tokens is not None:
															# Check that we have the right number of extra_id special tokens
															UpperCamelCase__         = len(set(filter(lambda lowercase			: bool("""extra_id""" in str(lowercase			)			) ,      lowercase			)			)			)
															if extra_tokens != extra_ids:
																					raise ValueError(
																					    f"Both extra_ids ({extra_ids}) and additional_special_tokens ({additional_special_tokens}) are"
																					    """ provided to T5Tokenizer. In this case the additional_special_tokens must include the extra_ids"""
																					    """ tokens"""			)
									if legacy:
															logger.warning_once(
															    f"You are using the legacy behaviour of the {self.__class__}. This means that tokens that come after special tokens will not be properly handled. We recommend you to"
															    """ read the related pull request available at https://github.com/huggingface/transformers/pull/24565"""			)

									UpperCamelCase__         = legacy
									UpperCamelCase__         = {} if sp_model_kwargs is None else sp_model_kwargs

									super().__init__(
									    eos_token=lowercase ,      unk_token=lowercase ,      pad_token=lowercase ,      extra_ids=lowercase ,      additional_special_tokens=lowercase ,      sp_model_kwargs=self.sp_model_kwargs ,      legacy=lowercase ,      **lowercase ,      )

									UpperCamelCase__         = vocab_file
									UpperCamelCase__         = extra_ids

									UpperCamelCase__         = spm.SentencePieceProcessor(**self.sp_model_kwargs			)
									self.sp_model.Load(lowercase			)






			@staticmethod
			def 				A      (   lowercase							:						List[Any] ,      lowercase							:						Optional[Any] ,      lowercase							:						int			)							->    Optional[Any]:

									'''simple docstring'''







									if pretrained_model_name_or_path in TaTokenizer.max_model_input_sizes:
															UpperCamelCase__         = TaTokenizer.max_model_input_sizes[pretrained_model_name_or_path]
															if init_max_model_length is not None and init_max_model_length != max_model_length:
																					return init_max_model_length
															elif init_max_model_length is None:
																					warnings.warn(
																					    """This tokenizer was incorrectly instantiated with a model max length of"""
																					    f" {deprecated_max_model_length} which will be corrected in Transformers v5.\nFor now, this"
																					    """ behavior is kept to avoid breaking backwards compatibility when padding/encoding with"""
																					    """ `truncation is True`.\n- Be aware that you SHOULD NOT rely on"""
																					    f" {pretrained_model_name_or_path} automatically truncating your input to"
																					    f" {deprecated_max_model_length} when padding/encoding.\n- If you want to encode/pad to sequences"
																					    f" longer than {deprecated_max_model_length} you can either instantiate this tokenizer with"
																					    """ `model_max_length` or pass `max_length` when encoding/padding.\n- To avoid this warning, please"""
																					    """ instantiate this tokenizer with `model_max_length` set to your preferred value.""" ,      lowercase ,      )

									return max_model_length






			@property
			def 				A      (   self							:						str			)							->    Dict:

									'''simple docstring'''







									return self.sp_model.get_piece_size() + self._extra_ids






			def 				A      (   self							:						Tuple			)							->    int:

									'''simple docstring'''







									UpperCamelCase__         = {self.convert_ids_to_tokens(lowercase			): i for i in range(self.vocab_size			)}
									vocab.update(self.added_tokens_encoder			)
									return vocab






			def 				A      (   self							:						Any ,      lowercase							:						List[int] ,      lowercase							:						Optional[List[int]] = None ,      lowercase							:						bool = False			)							->    List[int]:

									'''simple docstring'''







									if already_has_special_tokens:
															return super().get_special_tokens_mask(
															    token_ids_a=lowercase ,      token_ids_a=lowercase ,      already_has_special_tokens=lowercase			)

									# normal case: some special tokens
									if token_ids_a is None:
															return ([0] * len(lowercase			)) + [1]
									return ([0] * len(lowercase			)) + [1] + ([0] * len(lowercase			)) + [1]






			def 				A      (   self							:						Tuple			)							->    int:

									'''simple docstring'''







									return list(
									    set(filter(lambda lowercase			: bool(re.search(r"""<extra_id_\d+>""" ,      lowercase			)			) is not None ,      self.additional_special_tokens			)			)			)






			def 				A      (   self							:						Optional[int]			)							->    Tuple:

									'''simple docstring'''







									return [self._convert_token_to_id(lowercase			) for token in self.get_sentinel_tokens()]






			def 				A      (   self							:						Tuple ,      lowercase							:						List[int]			)							->    List[int]:

									'''simple docstring'''







									if len(lowercase			) > 0 and token_ids[-1] == self.eos_token_id:
															warnings.warn(
															    f"This sequence already has {self.eos_token}. In future versions this behavior may lead to duplicated"
															    """ eos tokens being added."""			)
															return token_ids
									else:
															return token_ids + [self.eos_token_id]






			def 				A      (   self							:						List[Any] ,      lowercase							:						List[int] ,      lowercase							:						Optional[List[int]] = None			)							->    List[int]:

									'''simple docstring'''







									UpperCamelCase__         = [self.eos_token_id]

									if token_ids_a is None:
															return len(token_ids_a + eos			) * [0]
									return len(token_ids_a + eos + token_ids_a + eos			) * [0]






			def 				A      (   self							:						int ,      lowercase							:						List[int] ,      lowercase							:						Optional[List[int]] = None			)							->    List[int]:

									'''simple docstring'''







									UpperCamelCase__         = self._add_eos_if_not_present(lowercase			)
									if token_ids_a is None:
															return token_ids_a
									else:
															UpperCamelCase__         = self._add_eos_if_not_present(lowercase			)
															return token_ids_a + token_ids_a






			def __getstate__(   self							:						Optional[Any]			)							->    Dict:

									'''simple docstring'''







									UpperCamelCase__         = self.__dict__.copy()
									UpperCamelCase__         = None
									return state






			def __setstate__(   self							:						Tuple ,      lowercase							:						str			)							->    str:

									'''simple docstring'''







									UpperCamelCase__         = d

									# for backward compatibility
									if not hasattr(self ,      """sp_model_kwargs"""			):
															UpperCamelCase__         = {}

									UpperCamelCase__         = spm.SentencePieceProcessor(**self.sp_model_kwargs			)
									self.sp_model.Load(self.vocab_file			)






			def 				A      (   self							:						Optional[int] ,      lowercase							:						"TextInput" ,      **lowercase							:						Tuple			)							->    List[str]:

									'''simple docstring'''







									if not self.legacy:
															UpperCamelCase__         = SPIECE_UNDERLINE + text.replace(lowercase ,      """ """			)
									return super().tokenize(lowercase ,      **lowercase			)






			def 				A      (   self							:						List[Any] ,      lowercase							:						Tuple ,      **lowercase							:						Tuple			)							->    List[str]:

									'''simple docstring'''







									if not self.legacy:
															UpperCamelCase__         = text.startswith(lowercase			)
															if is_first:
																					UpperCamelCase__         = text[1:]

									UpperCamelCase__         = self.sp_model.encode(lowercase ,      out_type=lowercase			)

									if not self.legacy and not is_first and not text.startswith(""" """			) and tokens[0].startswith(lowercase			):
															UpperCamelCase__         = ([tokens[0][1:]] if len(tokens[0]			) > 1 else []) + tokens[1:]
									return tokens






			def 				A      (   self							:						Optional[int] ,      lowercase							:						int			)							->    int:

									'''simple docstring'''







									if token.startswith("""<extra_id_"""			):
															UpperCamelCase__         = re.match(r"""<extra_id_(\d+)>""" ,      lowercase			)
															UpperCamelCase__         = int(match.group(1			)			)
															return self.vocab_size - num - 1
									return self.sp_model.piece_to_id(lowercase			)






			def 				A      (   self							:						List[Any] ,      lowercase							:						Tuple			)							->    Any:

									'''simple docstring'''







									if index < self.sp_model.get_piece_size():
															UpperCamelCase__         = self.sp_model.IdToPiece(lowercase			)
									else:
															UpperCamelCase__         = f"<extra_id_{self.vocab_size - 1 - index}>"
									return token






			def 				A      (   self							:						Union[str, Any] ,      lowercase							:						Optional[int]			)							->    Dict:

									'''simple docstring'''







									UpperCamelCase__         = []
									UpperCamelCase__         = """"""
									UpperCamelCase__         = False
									for token in tokens:
															# make sure that special tokens are not decoded using sentencepiece model
															if token in self.all_special_tokens:
																					if not prev_is_special:
																											out_string += " "
																					out_string += self.sp_model.decode(lowercase			) + token
																					UpperCamelCase__         = True
																					UpperCamelCase__         = []
															else:
																					current_sub_tokens.append(lowercase			)
																					UpperCamelCase__         = False
									out_string += self.sp_model.decode(lowercase			)
									return out_string.strip()







			def 				A      (   self							:						str ,      lowercase							:						str ,      lowercase							:						Optional[str] = None			)							->    Tuple[str]:

									'''simple docstring'''







									if not os.path.isdir(lowercase			):
															logger.error(f"Vocabulary path ({save_directory}) should be a directory"			)
															return
									UpperCamelCase__         = os.path.join(
									    lowercase ,      (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]			)

									if os.path.abspath(self.vocab_file			) != os.path.abspath(lowercase			) and os.path.isfile(self.vocab_file			):
															copyfile(self.vocab_file ,      lowercase			)
									elif not os.path.isfile(self.vocab_file			):
															with open(lowercase ,      """wb"""			) as fi:
																					UpperCamelCase__         = self.sp_model.serialized_model_proto()
																					fi.write(lowercase			)

									return (out_vocab_file,)








'''simple docstring'''


import copy

from ...configuration_utils import PretrainedConfig
from ...utils import logging
from ..auto import CONFIG_MAPPING


lowerCamelCase_				: str    	=							logging.get_logger(__name__)

lowerCamelCase_				: Optional[int]    	=							{
    '''SenseTime/deformable-detr''': '''https://huggingface.co/sensetime/deformable-detr/resolve/main/config.json''',
    # See all Deformable DETR models at https://huggingface.co/models?filter=deformable-detr
}
class    _SCREAMING_SNAKE_CASE							(		SCREAMING_SNAKE_CASE	):


			'''simple docstring'''





			__a							:							Tuple					     =					"deformable_detr"
			__a							:							str					     =					{
			    "hidden_size": "d_model",
			    "num_attention_heads": "encoder_attention_heads",
			}






			def __init__(   self							:						List[Any] ,      lowercase							:						List[Any]=True ,      lowercase							:						Tuple=None ,      lowercase							:						Tuple=3 ,      lowercase							:						List[str]=3_0_0 ,      lowercase							:						List[Any]=1_0_2_4 ,      lowercase							:						List[Any]=6 ,      lowercase							:						Tuple=1_0_2_4 ,      lowercase							:						Union[str, Any]=8 ,      lowercase							:						Optional[Any]=6 ,      lowercase							:						Tuple=1_0_2_4 ,      lowercase							:						Any=8 ,      lowercase							:						List[str]=0.0 ,      lowercase							:						Any=True ,      lowercase							:						Union[str, Any]="relu" ,      lowercase							:						Dict=2_5_6 ,      lowercase							:						Optional[int]=0.1 ,      lowercase							:						Optional[int]=0.0 ,      lowercase							:						List[str]=0.0 ,      lowercase							:						Any=0.0_2 ,      lowercase							:						Optional[int]=1.0 ,      lowercase							:						Tuple=True ,      lowercase							:						Optional[int]=False ,      lowercase							:						Any="sine" ,      lowercase							:						List[str]="resnet50" ,      lowercase							:						List[str]=True ,      lowercase							:						Optional[int]=False ,      lowercase							:						int=4 ,      lowercase							:						str=4 ,      lowercase							:						Dict=4 ,      lowercase							:						int=False ,      lowercase							:						List[Any]=3_0_0 ,      lowercase							:						List[Any]=False ,      lowercase							:						Dict=1 ,      lowercase							:						int=5 ,      lowercase							:						List[Any]=2 ,      lowercase							:						List[str]=1 ,      lowercase							:						Tuple=1 ,      lowercase							:						Dict=5 ,      lowercase							:						Dict=2 ,      lowercase							:						Dict=0.1 ,      lowercase							:						List[str]=0.2_5 ,      lowercase							:						Tuple=False ,      **lowercase							:						str ,      )							->    str:

									'''simple docstring'''







									if backbone_config is not None and use_timm_backbone:
															raise ValueError("""You can't specify both `backbone_config` and `use_timm_backbone`."""			)

									if not use_timm_backbone:
															if backbone_config is None:
																					logger.info("""`backbone_config` is `None`. Initializing the config with the default `ResNet` backbone."""			)
																					UpperCamelCase__         = CONFIG_MAPPING["""resnet"""](out_features=["""stage4"""]			)
															elif isinstance(lowercase ,      lowercase			):
																					UpperCamelCase__         = backbone_config.get("""model_type"""			)
																					UpperCamelCase__         = CONFIG_MAPPING[backbone_model_type]
																					UpperCamelCase__         = config_class.from_dict(lowercase			)
									UpperCamelCase__         = use_timm_backbone
									UpperCamelCase__         = backbone_config
									UpperCamelCase__         = num_channels
									UpperCamelCase__         = num_queries
									UpperCamelCase__         = max_position_embeddings
									UpperCamelCase__         = d_model
									UpperCamelCase__         = encoder_ffn_dim
									UpperCamelCase__         = encoder_layers
									UpperCamelCase__         = encoder_attention_heads
									UpperCamelCase__         = decoder_ffn_dim
									UpperCamelCase__         = decoder_layers
									UpperCamelCase__         = decoder_attention_heads
									UpperCamelCase__         = dropout
									UpperCamelCase__         = attention_dropout
									UpperCamelCase__         = activation_dropout
									UpperCamelCase__         = activation_function
									UpperCamelCase__         = init_std
									UpperCamelCase__         = init_xavier_std
									UpperCamelCase__         = encoder_layerdrop
									UpperCamelCase__         = auxiliary_loss
									UpperCamelCase__         = position_embedding_type
									UpperCamelCase__         = backbone
									UpperCamelCase__         = use_pretrained_backbone
									UpperCamelCase__         = dilation
									# deformable attributes
									UpperCamelCase__         = num_feature_levels
									UpperCamelCase__         = encoder_n_points
									UpperCamelCase__         = decoder_n_points
									UpperCamelCase__         = two_stage
									UpperCamelCase__         = two_stage_num_proposals
									UpperCamelCase__         = with_box_refine
									if two_stage is True and with_box_refine is False:
															raise ValueError("""If two_stage is True, with_box_refine must be True."""			)
									# Hungarian matcher
									UpperCamelCase__         = class_cost
									UpperCamelCase__         = bbox_cost
									UpperCamelCase__         = giou_cost
									# Loss coefficients
									UpperCamelCase__         = mask_loss_coefficient
									UpperCamelCase__         = dice_loss_coefficient
									UpperCamelCase__         = bbox_loss_coefficient
									UpperCamelCase__         = giou_loss_coefficient
									UpperCamelCase__         = eos_coefficient
									UpperCamelCase__         = focal_alpha
									UpperCamelCase__         = disable_custom_kernels
									super().__init__(is_encoder_decoder=lowercase ,      **lowercase			)






			@property
			def 				A      (   self							:						str			)							->    int:

									'''simple docstring'''







									return self.encoder_attention_heads






			@property
			def 				A      (   self							:						Tuple			)							->    int:

									'''simple docstring'''







									return self.d_model







			def 				A      (   self							:						Dict			)							->    Optional[Any]:

									'''simple docstring'''







									UpperCamelCase__         = copy.deepcopy(self.__dict__			)
									if self.backbone_config is not None:
															UpperCamelCase__         = self.backbone_config.to_dict()
									UpperCamelCase__         = self.__class__.model_type
									return output
