




from typing import Callable, Optional, Union

from ...configuration_utils import PretrainedConfig
from ...utils import logging


UpperCamelCase__					    =    logging.get_logger(__name__)

UpperCamelCase__					    =    {
    """microsoft/xprophetnet-large-wiki100-cased""": (
        """https://huggingface.co/microsoft/xprophetnet-large-wiki100-cased/resolve/main/config.json"""
    ),
}
class     a__					(      snake_case__   ):
						_a							:					Any      =					"""xlm-prophetnet"""
						_a							:					Any      =					["""past_key_values"""]
						_a							:					List[Any]      =					{
						    """num_attention_heads""": """num_encoder_attention_heads""",
						}






						def __init__(     self	,    _A = 0.1	,    _A = "gelu"	,    _A = 3_0_5_2_2	,    _A = 1_0_2_4	,    _A = 4_0_9_6	,    _A = 1_2	,    _A = 1_6	,    _A = 4_0_9_6	,    _A = 1_2	,    _A = 1_6	,    _A = 0.1	,    _A = 0.1	,    _A = 5_1_2	,    _A = 0.02	,    _A = True	,    _A = True	,    _A = 0	,    _A = 2	,    _A = 3_2	,    _A = 1_2_8	,    _A = False	,    _A = 0.0	,    _A = True	,    _A = 0	,    _A = 1	,    _A = 2	,    **_A	,    ):




											"""simple docstring"""



											__lowerCAmelCase            = vocab_size
											__lowerCAmelCase            = hidden_size
											__lowerCAmelCase            = encoder_ffn_dim
											__lowerCAmelCase            = num_encoder_layers
											__lowerCAmelCase            = num_encoder_attention_heads
											__lowerCAmelCase            = decoder_ffn_dim
											__lowerCAmelCase            = num_decoder_layers
											__lowerCAmelCase            = num_decoder_attention_heads
											__lowerCAmelCase            = max_position_embeddings
											__lowerCAmelCase            = init_std  # Normal(0, this parameter)
											__lowerCAmelCase            = activation_function

											# parameters for xlmprophetnet
											__lowerCAmelCase            = ngram
											__lowerCAmelCase            = num_buckets
											__lowerCAmelCase            = relative_max_distance
											__lowerCAmelCase            = disable_ngram_loss
											__lowerCAmelCase            = eps

											# 3 Types of Dropout
											__lowerCAmelCase            = attention_dropout
											__lowerCAmelCase            = activation_dropout
											__lowerCAmelCase            = dropout

											__lowerCAmelCase            = use_cache

											super().__init__(
											    pad_token_id=_A	,    bos_token_id=_A	,    eos_token_id=_A	,    is_encoder_decoder=_A	,    add_cross_attention=_A	,    decoder_start_token_id=_A	,    **_A	,    )






						@property
						def        __SCREAMING_SNAKE_CASE(     self    ):




											"""simple docstring"""



											return self.num_encoder_layers + self.num_decoder_layers






						@num_hidden_layers.setter
						def        __SCREAMING_SNAKE_CASE(     self	,    _A    ):




											"""simple docstring"""



											raise NotImplementedError(
											    "This model does not support the setting of `num_hidden_layers`. Please set `num_encoder_layers` and"
											    " `num_decoder_layers`."    )










import torch
from transformers import PreTrainedModel, XLMRobertaConfig, XLMRobertaModel
class     a__					(      snake_case__   ):
						_a							:					Optional[Any]      =					"""M-CLIP"""






						def __init__(     self	,    _A=1_0_2_4	,    _A=7_6_8	,    **_A    ):




											"""simple docstring"""



											__lowerCAmelCase            = transformerDimSize
											__lowerCAmelCase            = imageDimSize
											super().__init__(**_A    )






class     a__					(      snake_case__   ):
						_a							:					Tuple      =					MCLIPConfig






						def __init__(     self	,    _A	,    *_A	,    **_A    ):




											"""simple docstring"""



											super().__init__(_A	,    *_A	,    **_A    )
											__lowerCAmelCase            = XLMRobertaModel(_A    )
											__lowerCAmelCase            = torch.nn.Linear(
											    in_features=config.transformerDimensions	,    out_features=config.numDims    )






						def        __SCREAMING_SNAKE_CASE(     self	,    _A	,    _A    ):




											"""simple docstring"""



											__lowerCAmelCase            = self.transformer(input_ids=_A	,    attention_mask=_A    )[0]
											__lowerCAmelCase            = (embs * attention_mask.unsqueeze(2    )).sum(dim=1    ) / attention_mask.sum(dim=1    )[:, None]
											return self.LinearTransformation(_A    ), embs




