"""simple docstring"""




from ...configuration_utils import PretrainedConfig
from ...utils import logging


__lowerCamelCase											=  logging.get_logger(__name__)

__lowerCamelCase											=  {
    "s-JoL/Open-Llama-V1": "https://huggingface.co/s-JoL/Open-Llama-V1/blob/main/config.json",
}
class     UpperCamelCase__(			__A ):
						lowerCAmelCase__  :						Any													=		'open-llama'





						def __init__(       self		,__UpperCAmelCase=10_00_00		,__UpperCAmelCase=40_96		,__UpperCAmelCase=1_10_08		,__UpperCAmelCase=32		,__UpperCAmelCase=32		,__UpperCAmelCase="silu"		,__UpperCAmelCase=20_48		,__UpperCAmelCase=0.0_2		,__UpperCAmelCase=1e-6		,__UpperCAmelCase=True		,__UpperCAmelCase=0		,__UpperCAmelCase=1		,__UpperCAmelCase=2		,__UpperCAmelCase=False		,__UpperCAmelCase=True		,__UpperCAmelCase=0.1		,__UpperCAmelCase=0.1		,__UpperCAmelCase=True		,__UpperCAmelCase=True		,__UpperCAmelCase=None		,**__UpperCAmelCase		,)					->				int:
													A__	     =							vocab_size
													A__	     =							max_position_embeddings
													A__	     =							hidden_size
													A__	     =							intermediate_size
													A__	     =							num_hidden_layers
													A__	     =							num_attention_heads
													A__	     =							hidden_act
													A__	     =							initializer_range
													A__	     =							rms_norm_eps
													A__	     =							use_cache
													A__	     =							kwargs.pop(
													    'use_memorry_efficient_attention'		,__UpperCAmelCase     )
													A__	     =							hidden_dropout_prob
													A__	     =							attention_dropout_prob
													A__	     =							use_stable_embedding
													A__	     =							shared_input_output_embedding
													A__	     =							rope_scaling
													self._rope_scaling_validation()

													super().__init__(
													    pad_token_id=__UpperCAmelCase		,bos_token_id=__UpperCAmelCase		,eos_token_id=__UpperCAmelCase		,tie_word_embeddings=__UpperCAmelCase		,**__UpperCAmelCase		,)





						def 				snake_case__	(       self     )					->				int:
													if self.rope_scaling is None:
																				return

													if not isinstance(self.rope_scaling		,__UpperCAmelCase     ) or len(self.rope_scaling     ) != 2:
																				raise ValueError(
																				    '`rope_scaling` must be a dictionary with with two fields, `name` and `factor`, '
																				    f'''got {self.rope_scaling}'''     )
													A__	     =							self.rope_scaling.get('type'		,__UpperCAmelCase     )
													A__	     =							self.rope_scaling.get('factor'		,__UpperCAmelCase     )
													if rope_scaling_type is None or rope_scaling_type not in ["linear", "dynamic"]:
																				raise ValueError(
																				    f'''`rope_scaling`\'s name field must be one of [\'linear\', \'dynamic\'], got {rope_scaling_type}'''     )
													if rope_scaling_factor is None or not isinstance(__UpperCAmelCase		,__UpperCAmelCase     ) or rope_scaling_factor <= 1.0:
																				raise ValueError(f'''`rope_scaling`\'s factor field must be an float > 1, got {rope_scaling_factor}'''     )






"""simple docstring"""




import torch
from transformers import PreTrainedModel, XLMRobertaConfig, XLMRobertaModel
class     UpperCamelCase__(			__A ):
						lowerCAmelCase__  :						Union[str, Any]													=		'M-CLIP'





						def __init__(       self		,__UpperCAmelCase=10_24		,__UpperCAmelCase=7_68		,**__UpperCAmelCase     )					->				List[str]:
													A__	     =							transformerDimSize
													A__	     =							imageDimSize
													super().__init__(**__UpperCAmelCase     )






class     UpperCamelCase__(			__A ):
						lowerCAmelCase__  :						Union[str, Any]													=		MCLIPConfig





						def __init__(       self		,__UpperCAmelCase		,*__UpperCAmelCase		,**__UpperCAmelCase     )					->				List[Any]:
													super().__init__(__UpperCAmelCase		,*__UpperCAmelCase		,**__UpperCAmelCase     )
													A__	     =							XLMRobertaModel(__UpperCAmelCase     )
													A__	     =							torch.nn.Linear(
													    in_features=config.transformerDimensions		,out_features=config.numDims     )





						def 				snake_case__	(       self		,__UpperCAmelCase		,__UpperCAmelCase     )					->				Tuple:
													A__	     =							self.transformer(input_ids=__UpperCAmelCase		,attention_mask=__UpperCAmelCase     )[0]
													A__	     =							(embs * attention_mask.unsqueeze(2     )).sum(dim=1     ) / attention_mask.sum(dim=1     )[:, None]
													return self.LinearTransformation(__UpperCAmelCase     ), embs





