





import json
import os
from pathlib import Path
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple, Union

import sentencepiece

from ...tokenization_utils import BatchEncoding, PreTrainedTokenizer
from ...utils import logging


SCREAMING_SNAKE_CASE_		     = logging.get_logger(__name__)

SCREAMING_SNAKE_CASE_		     = """▁"""

SCREAMING_SNAKE_CASE_		     = {
    """vocab_file""": """vocab.json""",
    """spm_file""": """sentencepiece.bpe.model""",
    """tokenizer_config_file""": """tokenizer_config.json""",
}

SCREAMING_SNAKE_CASE_		     = {
    """vocab_file""": {
        """facebook/m2m100_418M""": """https://huggingface.co/facebook/m2m100_418M/resolve/main/vocab.json""",
        """facebook/m2m100_1.2B""": """https://huggingface.co/facebook/m2m100_1.2B/resolve/main/vocab.json""",
    },
    """spm_file""": {
        """facebook/m2m100_418M""": """https://huggingface.co/facebook/m2m100_418M/resolve/main/sentencepiece.bpe.model""",
        """facebook/m2m100_1.2B""": """https://huggingface.co/facebook/m2m100_1.2B/resolve/main/sentencepiece.bpe.model""",
    },
    """tokenizer_config_file""": {
        """facebook/m2m100_418M""": """https://huggingface.co/facebook/m2m100_418M/resolve/main/tokenizer_config.json""",
        """facebook/m2m100_1.2B""": """https://huggingface.co/facebook/m2m100_1.2B/resolve/main/tokenizer_config.json""",
    },
}

SCREAMING_SNAKE_CASE_		     = {
    """facebook/m2m100_418M""": 1_0_2_4,
}

# fmt: off
SCREAMING_SNAKE_CASE_		     = {
    """m2m100""": ["""af""", """am""", """ar""", """ast""", """az""", """ba""", """be""", """bg""", """bn""", """br""", """bs""", """ca""", """ceb""", """cs""", """cy""", """da""", """de""", """el""", """en""", """es""", """et""", """fa""", """ff""", """fi""", """fr""", """fy""", """ga""", """gd""", """gl""", """gu""", """ha""", """he""", """hi""", """hr""", """ht""", """hu""", """hy""", """id""", """ig""", """ilo""", """is""", """it""", """ja""", """jv""", """ka""", """kk""", """km""", """kn""", """ko""", """lb""", """lg""", """ln""", """lo""", """lt""", """lv""", """mg""", """mk""", """ml""", """mn""", """mr""", """ms""", """my""", """ne""", """nl""", """no""", """ns""", """oc""", """or""", """pa""", """pl""", """ps""", """pt""", """ro""", """ru""", """sd""", """si""", """sk""", """sl""", """so""", """sq""", """sr""", """ss""", """su""", """sv""", """sw""", """ta""", """th""", """tl""", """tn""", """tr""", """uk""", """ur""", """uz""", """vi""", """wo""", """xh""", """yi""", """yo""", """zh""", """zu"""],
    """wmt21""": ["""en""", """ha""", """is""", """ja""", """cs""", """ru""", """zh""", """de"""]
}

class 	UpperCamelCase__				(   lowerCAmelCase_     ):



     '''simple docstring'''

     __snake_case					:			str				=    VOCAB_FILES_NAMES
     __snake_case					:			Tuple				=    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     __snake_case					:			Optional[int]				=    PRETRAINED_VOCAB_FILES_MAP
     __snake_case					:			Optional[Any]				=    ["input_ids", "attention_mask"]

     __snake_case					:			List[int]				=    []
     __snake_case					:			List[int]				=    []






     def __init__(							self		:					str		,lowerCamelCase__		:					Dict		,lowerCamelCase__		:					Optional[Any]		,lowerCamelCase__		:					Optional[int]=None		,lowerCamelCase__		:					Dict=None		,lowerCamelCase__		:					Union[str, Any]="<s>"		,lowerCamelCase__		:					Union[str, Any]="</s>"		,lowerCamelCase__		:					Union[str, Any]="</s>"		,lowerCamelCase__		:					List[Any]="<pad>"		,lowerCamelCase__		:					Any="<unk>"		,lowerCamelCase__		:					Union[str, Any]="m2m100"		,lowerCamelCase__		:					Optional[Dict[str, Any]] = None		,lowerCamelCase__		:					Union[str, Any]=8		,**lowerCamelCase__		:					List[str]		,)  ->	None:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE	=		{} if sp_model_kwargs is None else sp_model_kwargs

           SCREAMING_SNAKE_CASE	=		language_codes
           SCREAMING_SNAKE_CASE	=		FAIRSEQ_LANGUAGE_CODES[language_codes]
           SCREAMING_SNAKE_CASE	=		{lang_code: F"""__{lang_code}__""" for lang_code in fairseq_language_code}

           SCREAMING_SNAKE_CASE	=		kwargs.get("""additional_special_tokens"""		,[] )
           kwargs["additional_special_tokens"] += [
               self.get_lang_token(lowerCamelCase__ )
               for lang_code in fairseq_language_code
               if self.get_lang_token(lowerCamelCase__ ) not in kwargs["additional_special_tokens"]
           ]

           super().__init__(
               src_lang=lowerCamelCase__		,tgt_lang=lowerCamelCase__		,bos_token=lowerCamelCase__		,eos_token=lowerCamelCase__		,sep_token=lowerCamelCase__		,unk_token=lowerCamelCase__		,pad_token=lowerCamelCase__		,language_codes=lowerCamelCase__		,sp_model_kwargs=self.sp_model_kwargs		,num_madeup_words=lowerCamelCase__		,**lowerCamelCase__		,)

           SCREAMING_SNAKE_CASE	=		vocab_file
           SCREAMING_SNAKE_CASE	=		load_json(lowerCamelCase__ )
           SCREAMING_SNAKE_CASE	=		{v: k for k, v in self.encoder.items()}
           SCREAMING_SNAKE_CASE	=		spm_file
           SCREAMING_SNAKE_CASE	=		load_spm(lowerCamelCase__		,self.sp_model_kwargs )

           SCREAMING_SNAKE_CASE	=		len(self.encoder )

           SCREAMING_SNAKE_CASE	=		{
               self.get_lang_token(lowerCamelCase__ ): self.encoder_size + i for i, lang_code in enumerate(lowerCamelCase__ )
           }
           SCREAMING_SNAKE_CASE	=		{lang_code: self.encoder_size + i for i, lang_code in enumerate(lowerCamelCase__ )}
           SCREAMING_SNAKE_CASE	=		{v: k for k, v in self.lang_token_to_id.items()}

           SCREAMING_SNAKE_CASE	=		src_lang if src_lang is not None else """en"""
           SCREAMING_SNAKE_CASE	=		tgt_lang
           SCREAMING_SNAKE_CASE	=		self.get_lang_id(self._src_lang )
           self.set_src_lang_special_tokens(self._src_lang )

           SCREAMING_SNAKE_CASE	=		num_madeup_words






     @property
     def        SCREAMING_SNAKE_CASE__						(							self		:					Any )  ->	int:




           '''simple docstring'''


           return len(self.encoder ) + len(self.lang_token_to_id )






     @property
     def        SCREAMING_SNAKE_CASE__						(							self		:					Any )  ->	str:




           '''simple docstring'''


           return self._src_lang






     @src_lang.setter
     def        SCREAMING_SNAKE_CASE__						(							self		:					int		,lowerCamelCase__		:					str )  ->	None:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE	=		new_src_lang
           self.set_src_lang_special_tokens(self._src_lang )






     def        SCREAMING_SNAKE_CASE__						(							self		:					str		,lowerCamelCase__		:					str )  ->	List[str]:




           '''simple docstring'''


           return self.sp_model.encode(lowerCamelCase__		,out_type=lowerCamelCase__ )






     def        SCREAMING_SNAKE_CASE__						(							self		:					Optional[int]		,lowerCamelCase__		:					Any )  ->	Dict:




           '''simple docstring'''


           if token in self.lang_token_to_id:
                 return self.lang_token_to_id[token]
           return self.encoder.get(lowerCamelCase__		,self.encoder[self.unk_token] )






     def        SCREAMING_SNAKE_CASE__						(							self		:					Union[str, Any]		,lowerCamelCase__		:					int )  ->	str:




           '''simple docstring'''


           if index in self.id_to_lang_token:
                 return self.id_to_lang_token[index]
           return self.decoder.get(lowerCamelCase__		,self.unk_token )






     def        SCREAMING_SNAKE_CASE__						(							self		:					Tuple		,lowerCamelCase__		:					Tuple )  ->	Any:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE	=		[]
           SCREAMING_SNAKE_CASE	=		""""""
           for token in tokens:
                 # make sure that special tokens are not decoded using sentencepiece model
                 if token in self.all_special_tokens:
                       out_string += self.sp_model.decode(lowerCamelCase__ ) + token
                       SCREAMING_SNAKE_CASE	=		[]
                 else:
                       current_sub_tokens.append(lowerCamelCase__ )
           out_string += self.sp_model.decode(lowerCamelCase__ )
           return out_string.strip()






     def        SCREAMING_SNAKE_CASE__						(							self		:					Any		,lowerCamelCase__		:					List[int]		,lowerCamelCase__		:					Optional[List[int]] = None		,lowerCamelCase__		:					bool = False )  ->	List[int]:




           '''simple docstring'''


           if already_has_special_tokens:
                 return super().get_special_tokens_mask(
                     token_ids_a=lowerCamelCase__		,token_ids_a=lowerCamelCase__		,already_has_special_tokens=lowerCamelCase__ )

           SCREAMING_SNAKE_CASE	=		[1] * len(self.prefix_tokens )
           SCREAMING_SNAKE_CASE	=		[1] * len(self.suffix_tokens )
           if token_ids_a is None:
                 return prefix_ones + ([0] * len(lowerCamelCase__ )) + suffix_ones
           return prefix_ones + ([0] * len(lowerCamelCase__ )) + ([0] * len(lowerCamelCase__ )) + suffix_ones






     def        SCREAMING_SNAKE_CASE__						(							self		:					str		,lowerCamelCase__		:					List[int]		,lowerCamelCase__		:					Optional[List[int]] = None )  ->	List[int]:




           '''simple docstring'''


           if token_ids_a is None:
                 return self.prefix_tokens + token_ids_a + self.suffix_tokens
           # We don't expect to process pairs, but leave the pair logic for API consistency
           return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens






     def        SCREAMING_SNAKE_CASE__						(							self		:					Optional[Any] )  ->	Dict:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE	=		{self.convert_ids_to_tokens(lowerCamelCase__ ): i for i in range(self.vocab_size )}
           vocab.update(self.added_tokens_encoder )
           return vocab






     def __getstate__(							self		:					Tuple )  ->	Dict:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE	=		self.__dict__.copy()
           SCREAMING_SNAKE_CASE	=		None
           return state






     def __setstate__(							self		:					Optional[Any]		,lowerCamelCase__		:					Dict )  ->	None:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE	=		d

           # for backward compatibility
           if not hasattr(self		,"""sp_model_kwargs""" ):
                 SCREAMING_SNAKE_CASE	=		{}

           SCREAMING_SNAKE_CASE	=		load_spm(self.spm_file		,self.sp_model_kwargs )






     def        SCREAMING_SNAKE_CASE__						(							self		:					Optional[int]		,lowerCamelCase__		:					str		,lowerCamelCase__		:					Optional[str] = None )  ->	Tuple[str]:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE	=		Path(lowerCamelCase__ )
           if not save_dir.is_dir():
                 raise OSError(F"""{save_directory} should be a directory""" )
           SCREAMING_SNAKE_CASE	=		save_dir / (
               (filename_prefix + """-""" if filename_prefix else """""") + self.vocab_files_names["""vocab_file"""]
           )
           SCREAMING_SNAKE_CASE	=		save_dir / (
               (filename_prefix + """-""" if filename_prefix else """""") + self.vocab_files_names["""spm_file"""]
           )

           save_json(self.encoder		,lowerCamelCase__ )

           if os.path.abspath(self.spm_file ) != os.path.abspath(lowerCamelCase__ ) and os.path.isfile(self.spm_file ):
                 copyfile(self.spm_file		,lowerCamelCase__ )
           elif not os.path.isfile(self.spm_file ):
                 with open(lowerCamelCase__		,"""wb""" ) as fi:
                       SCREAMING_SNAKE_CASE	=		self.sp_model.serialized_model_proto()
                       fi.write(lowerCamelCase__ )

           return (str(lowerCamelCase__ ), str(lowerCamelCase__ ))






     def        SCREAMING_SNAKE_CASE__						(							self		:					Union[str, Any]		,lowerCamelCase__		:					List[str]		,lowerCamelCase__		:					str = "en"		,lowerCamelCase__		:					Optional[List[str]] = None		,lowerCamelCase__		:					str = "ro"		,**lowerCamelCase__		:					int		,)  ->	BatchEncoding:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE	=		src_lang
           SCREAMING_SNAKE_CASE	=		tgt_lang
           self.set_src_lang_special_tokens(self.src_lang )
           return super().prepare_seqaseq_batch(lowerCamelCase__		,lowerCamelCase__		,**lowerCamelCase__ )






     def        SCREAMING_SNAKE_CASE__						(							self		:					int		,lowerCamelCase__		:					Any		,lowerCamelCase__		:					Optional[str]		,lowerCamelCase__		:					Optional[str]		,**lowerCamelCase__		:					Union[str, Any] )  ->	List[Any]:




           '''simple docstring'''


           if src_lang is None or tgt_lang is None:
                 raise ValueError("""Translation requires a `src_lang` and a `tgt_lang` for this model""" )
           SCREAMING_SNAKE_CASE	=		src_lang
           SCREAMING_SNAKE_CASE	=		self(lowerCamelCase__		,add_special_tokens=lowerCamelCase__		,**lowerCamelCase__ )
           SCREAMING_SNAKE_CASE	=		self.get_lang_id(lowerCamelCase__ )
           SCREAMING_SNAKE_CASE	=		tgt_lang_id
           return inputs






     def        SCREAMING_SNAKE_CASE__						(							self		:					str )  ->	int:




           '''simple docstring'''


           self.set_src_lang_special_tokens(self.src_lang )






     def        SCREAMING_SNAKE_CASE__						(							self		:					Union[str, Any] )  ->	List[Any]:




           '''simple docstring'''


           self.set_tgt_lang_special_tokens(self.tgt_lang )






     def        SCREAMING_SNAKE_CASE__						(							self		:					int		,lowerCamelCase__		:					str )  ->	None:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE	=		self.get_lang_token(lowerCamelCase__ )
           SCREAMING_SNAKE_CASE	=		self.lang_token_to_id[lang_token]
           SCREAMING_SNAKE_CASE	=		[self.cur_lang_id]
           SCREAMING_SNAKE_CASE	=		[self.eos_token_id]






     def        SCREAMING_SNAKE_CASE__						(							self		:					Tuple		,lowerCamelCase__		:					str )  ->	None:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE	=		self.get_lang_token(lowerCamelCase__ )
           SCREAMING_SNAKE_CASE	=		self.lang_token_to_id[lang_token]
           SCREAMING_SNAKE_CASE	=		[self.cur_lang_id]
           SCREAMING_SNAKE_CASE	=		[self.eos_token_id]






     def        SCREAMING_SNAKE_CASE__						(							self		:					Dict		,lowerCamelCase__		:					str )  ->	str:




           '''simple docstring'''


           return self.lang_code_to_token[lang]






     def        SCREAMING_SNAKE_CASE__						(							self		:					Tuple		,lowerCamelCase__		:					str )  ->	int:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE	=		self.get_lang_token(lowerCamelCase__ )
           return self.lang_token_to_id[lang_token]



def      __lowercase     (   _SCREAMING_SNAKE_CASE				,		_SCREAMING_SNAKE_CASE			)				->     sentencepiece.SentencePieceProcessor:


      '''simple docstring'''


      SCREAMING_SNAKE_CASE	=		sentencepiece.SentencePieceProcessor(**_SCREAMING_SNAKE_CASE			)
      spm.Load(str(_SCREAMING_SNAKE_CASE			)			)
      return spm



def      __lowercase     (   _SCREAMING_SNAKE_CASE			)				->     Union[Dict, List]:


      '''simple docstring'''


      with open(_SCREAMING_SNAKE_CASE				,		"""r"""			) as f:
            return json.load(_SCREAMING_SNAKE_CASE			)



def      __lowercase     (   _SCREAMING_SNAKE_CASE				,		_SCREAMING_SNAKE_CASE			)				->     None:


      '''simple docstring'''


      with open(_SCREAMING_SNAKE_CASE				,		"""w"""			) as f:
            json.dump(_SCREAMING_SNAKE_CASE				,		_SCREAMING_SNAKE_CASE				,		indent=2			)









import copy

from ...configuration_utils import PretrainedConfig
from ...utils import logging
from ..auto import CONFIG_MAPPING


SCREAMING_SNAKE_CASE_		     = logging.get_logger(__name__)

SCREAMING_SNAKE_CASE_		     = {
    """ut/deta""": """https://huggingface.co/ut/deta/resolve/main/config.json""",
}

class 	UpperCamelCase__				(   lowerCAmelCase_     ):



     '''simple docstring'''

     __snake_case					:			str				=    "deta"
     __snake_case					:			int				=    {
         "hidden_size": "d_model",
         "num_attention_heads": "encoder_attention_heads",
     }






     def __init__(							self		:					List[Any]		,lowerCamelCase__		:					Any=None		,lowerCamelCase__		:					Optional[int]=900		,lowerCamelCase__		:					Optional[Any]=2048		,lowerCamelCase__		:					Optional[Any]=6		,lowerCamelCase__		:					Dict=2048		,lowerCamelCase__		:					Optional[int]=8		,lowerCamelCase__		:					List[str]=6		,lowerCamelCase__		:					Union[str, Any]=1024		,lowerCamelCase__		:					str=8		,lowerCamelCase__		:					Union[str, Any]=0.0		,lowerCamelCase__		:					Dict=True		,lowerCamelCase__		:					Dict="relu"		,lowerCamelCase__		:					int=256		,lowerCamelCase__		:					Optional[Any]=0.1		,lowerCamelCase__		:					Dict=0.0		,lowerCamelCase__		:					Dict=0.0		,lowerCamelCase__		:					Dict=0.02		,lowerCamelCase__		:					List[Any]=1.0		,lowerCamelCase__		:					List[str]=True		,lowerCamelCase__		:					int=False		,lowerCamelCase__		:					int="sine"		,lowerCamelCase__		:					Any=5		,lowerCamelCase__		:					Optional[int]=4		,lowerCamelCase__		:					Optional[int]=4		,lowerCamelCase__		:					List[Any]=True		,lowerCamelCase__		:					Optional[int]=300		,lowerCamelCase__		:					Optional[Any]=True		,lowerCamelCase__		:					Optional[Any]=True		,lowerCamelCase__		:					Optional[int]=1		,lowerCamelCase__		:					Tuple=5		,lowerCamelCase__		:					Tuple=2		,lowerCamelCase__		:					Dict=1		,lowerCamelCase__		:					str=1		,lowerCamelCase__		:					List[Any]=5		,lowerCamelCase__		:					Tuple=2		,lowerCamelCase__		:					Union[str, Any]=0.1		,lowerCamelCase__		:					List[str]=0.25		,**lowerCamelCase__		:					int		,)  ->	List[Any]:




           '''simple docstring'''


           if backbone_config is None:
                 logger.info("""`backbone_config` is `None`. Initializing the config with the default `ResNet` backbone.""" )
                 SCREAMING_SNAKE_CASE	=		CONFIG_MAPPING["""resnet"""](out_features=["""stage2""", """stage3""", """stage4"""] )
           else:
                 if isinstance(lowerCamelCase__		,lowerCamelCase__ ):
                       SCREAMING_SNAKE_CASE	=		backbone_config.pop("""model_type""" )
                       SCREAMING_SNAKE_CASE	=		CONFIG_MAPPING[backbone_model_type]
                       SCREAMING_SNAKE_CASE	=		config_class.from_dict(lowerCamelCase__ )

           SCREAMING_SNAKE_CASE	=		backbone_config
           SCREAMING_SNAKE_CASE	=		num_queries
           SCREAMING_SNAKE_CASE	=		max_position_embeddings
           SCREAMING_SNAKE_CASE	=		d_model
           SCREAMING_SNAKE_CASE	=		encoder_ffn_dim
           SCREAMING_SNAKE_CASE	=		encoder_layers
           SCREAMING_SNAKE_CASE	=		encoder_attention_heads
           SCREAMING_SNAKE_CASE	=		decoder_ffn_dim
           SCREAMING_SNAKE_CASE	=		decoder_layers
           SCREAMING_SNAKE_CASE	=		decoder_attention_heads
           SCREAMING_SNAKE_CASE	=		dropout
           SCREAMING_SNAKE_CASE	=		attention_dropout
           SCREAMING_SNAKE_CASE	=		activation_dropout
           SCREAMING_SNAKE_CASE	=		activation_function
           SCREAMING_SNAKE_CASE	=		init_std
           SCREAMING_SNAKE_CASE	=		init_xavier_std
           SCREAMING_SNAKE_CASE	=		encoder_layerdrop
           SCREAMING_SNAKE_CASE	=		auxiliary_loss
           SCREAMING_SNAKE_CASE	=		position_embedding_type
           # deformable attributes
           SCREAMING_SNAKE_CASE	=		num_feature_levels
           SCREAMING_SNAKE_CASE	=		encoder_n_points
           SCREAMING_SNAKE_CASE	=		decoder_n_points
           SCREAMING_SNAKE_CASE	=		two_stage
           SCREAMING_SNAKE_CASE	=		two_stage_num_proposals
           SCREAMING_SNAKE_CASE	=		with_box_refine
           SCREAMING_SNAKE_CASE	=		assign_first_stage
           if two_stage is True and with_box_refine is False:
                 raise ValueError("""If two_stage is True, with_box_refine must be True.""" )
           # Hungarian matcher
           SCREAMING_SNAKE_CASE	=		class_cost
           SCREAMING_SNAKE_CASE	=		bbox_cost
           SCREAMING_SNAKE_CASE	=		giou_cost
           # Loss coefficients
           SCREAMING_SNAKE_CASE	=		mask_loss_coefficient
           SCREAMING_SNAKE_CASE	=		dice_loss_coefficient
           SCREAMING_SNAKE_CASE	=		bbox_loss_coefficient
           SCREAMING_SNAKE_CASE	=		giou_loss_coefficient
           SCREAMING_SNAKE_CASE	=		eos_coefficient
           SCREAMING_SNAKE_CASE	=		focal_alpha
           super().__init__(is_encoder_decoder=lowerCamelCase__		,**lowerCamelCase__ )






     @property
     def        SCREAMING_SNAKE_CASE__						(							self		:					str )  ->	int:




           '''simple docstring'''


           return self.encoder_attention_heads






     @property
     def        SCREAMING_SNAKE_CASE__						(							self		:					List[str] )  ->	int:




           '''simple docstring'''


           return self.d_model






     def        SCREAMING_SNAKE_CASE__						(							self		:					Optional[Any] )  ->	Optional[int]:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE	=		copy.deepcopy(self.__dict__ )
           SCREAMING_SNAKE_CASE	=		self.backbone_config.to_dict()
           SCREAMING_SNAKE_CASE	=		self.__class__.model_type
           return output


