import torch
from transformers import PreTrainedModel, XLMRobertaConfig, XLMRobertaModel





class 						_lowercase		(   _SCREAMING_SNAKE_CASE			):
 lowercase								=  'M-CLIP'

 def __init__(					self      :				Tuple  , snake_case      :				List[str]=1_0_2_4  , snake_case      :				str=7_6_8  , **snake_case      :				Union[str, Any]     )  ->		int:

  """simple docstring"""





  UpperCamelCase_   :		str					=   transformerDimSize
  UpperCamelCase_   :		List[str]					=   imageDimSize
  super().__init__(**_lowerCAmelCase     )





class 						_lowercase		(   _SCREAMING_SNAKE_CASE			):
 lowercase								=  MCLIPConfig

 def __init__(					self      :				Dict  , snake_case      :				Union[str, Any]  , *snake_case      :				str  , **snake_case      :				str     )  ->		Optional[int]:

  """simple docstring"""





  super().__init__(_lowerCAmelCase  , *_lowerCAmelCase  , **_lowerCAmelCase     )
  UpperCamelCase_   :		int					=   XLMRobertaModel(_lowerCAmelCase     )
  UpperCamelCase_   :		Union[str, Any]					=   torch.nn.Linear(
      in_features=config.transformerDimensions  , out_features=config.numDims     )


 def 		SCREAMING_SNAKE_CASE__  (					self      :				List[Any]  , snake_case      :				Any  , snake_case      :				Union[str, Any]     )  ->		Dict:

  """simple docstring"""





  UpperCamelCase_   :		Union[str, Any]					=   self.transformer(input_ids=_lowerCAmelCase  , attention_mask=_lowerCAmelCase     )[0]
  UpperCamelCase_   :		str					=   (embs * attention_mask.unsqueeze(2     )).sum(dim=1     ) / attention_mask.sum(dim=1     )[:, None]
  return self.LinearTransformation(_lowerCAmelCase     ), embs









from typing import TYPE_CHECKING

from ....utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


lowerCamelCase__					:  Optional[int]      							=				{'configuration_mmbt': ['MMBTConfig']}

try:
   if not is_torch_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   lowerCamelCase__					:  Any      							=				['MMBTForClassification', 'MMBTModel', 'ModalEmbeddings']


if TYPE_CHECKING:
   from .configuration_mmbt import MMBTConfig

   try:
      if not is_torch_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .modeling_mmbt import MMBTForClassification, MMBTModel, ModalEmbeddings

else:
   import sys

   lowerCamelCase__					:  str      							=				_LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)