


from ...configuration_utils import PretrainedConfig
from ...utils import logging


UpperCAmelCase    =					logging.get_logger(__name__)

UpperCAmelCase    =					{
    """microsoft/cvt-13""": """https://huggingface.co/microsoft/cvt-13/resolve/main/config.json""",
    # See all Cvt models at https://huggingface.co/models?filter=cvt
}







class lowerCAmelCase_		(				lowerCamelCase__ ):

 '''simple docstring'''

 __snake_case				=						"cvt"



 def __init__(			self	,   _UpperCAmelCase=3	,   _UpperCAmelCase=[7, 3, 3]	,   _UpperCAmelCase=[4, 2, 2]	,   _UpperCAmelCase=[2, 1, 1]	,   _UpperCAmelCase=[64, 1_92, 3_84]	,   _UpperCAmelCase=[1, 3, 6]	,   _UpperCAmelCase=[1, 2, 10]	,   _UpperCAmelCase=[4.0, 4.0, 4.0]	,   _UpperCAmelCase=[0.0, 0.0, 0.0]	,   _UpperCAmelCase=[0.0, 0.0, 0.0]	,   _UpperCAmelCase=[0.0, 0.0, 0.1]	,   _UpperCAmelCase=[True, True, True]	,   _UpperCAmelCase=[False, False, True]	,   _UpperCAmelCase=["dw_bn", "dw_bn", "dw_bn"]	,   _UpperCAmelCase=[3, 3, 3]	,   _UpperCAmelCase=[1, 1, 1]	,   _UpperCAmelCase=[2, 2, 2]	,   _UpperCAmelCase=[1, 1, 1]	,   _UpperCAmelCase=[1, 1, 1]	,   _UpperCAmelCase=0.02	,   _UpperCAmelCase=1E-12	,   **_UpperCAmelCase	,   ):
       super().__init__(**_UpperCAmelCase      )
       snake_case_ =   num_channels
       snake_case_ =   patch_sizes
       snake_case_ =   patch_stride
       snake_case_ =   patch_padding
       snake_case_ =   embed_dim
       snake_case_ =   num_heads
       snake_case_ =   depth
       snake_case_ =   mlp_ratio
       snake_case_ =   attention_drop_rate
       snake_case_ =   drop_rate
       snake_case_ =   drop_path_rate
       snake_case_ =   qkv_bias
       snake_case_ =   cls_token
       snake_case_ =   qkv_projection_method
       snake_case_ =   kernel_qkv
       snake_case_ =   padding_kv
       snake_case_ =   stride_kv
       snake_case_ =   padding_q
       snake_case_ =   stride_q
       snake_case_ =   initializer_range
       snake_case_ =   layer_norm_eps



from dataclasses import dataclass
from typing import Optional, Tuple

import torch
from torch import nn
from transformers import RobertaPreTrainedModel, XLMRobertaConfig, XLMRobertaModel
from transformers.utils import ModelOutput







@dataclass
class lowerCAmelCase_		(				lowerCamelCase__ ):

 '''simple docstring'''

 __snake_case				=						None
 __snake_case				=						None
 __snake_case				=						None
 __snake_case				=						None







class lowerCAmelCase_		(				lowerCamelCase__ ):

 '''simple docstring'''

 def __init__(			self	,   _UpperCAmelCase=1	,   _UpperCAmelCase=0	,   _UpperCAmelCase=2	,   _UpperCAmelCase=5_12	,   _UpperCAmelCase="cls"	,   _UpperCAmelCase=False	,   _UpperCAmelCase=True	,   **_UpperCAmelCase	,   ):
       super().__init__(pad_token_id=_UpperCAmelCase	,   bos_token_id=_UpperCAmelCase	,   eos_token_id=_UpperCAmelCase	,   **_UpperCAmelCase      )
       snake_case_ =   project_dim
       snake_case_ =   pooler_fn
       snake_case_ =   learn_encoder
       snake_case_ =   use_attention_mask







class lowerCAmelCase_		(				lowerCamelCase__ ):

 '''simple docstring'''

 __snake_case				=						[R"pooler", R"logit_scale"]
 __snake_case				=						[R"position_ids", R"predictions.decoder.bias"]
 __snake_case				=						"roberta"
 __snake_case				=						RobertaSeriesConfig



 def __init__(			self	,   _UpperCAmelCase      ):
       super().__init__(_UpperCAmelCase      )
       snake_case_ =   XLMRobertaModel(_UpperCAmelCase      )
       snake_case_ =   nn.Linear(config.hidden_size	,   config.project_dim      )
       snake_case_ =   getattr(_UpperCAmelCase	,   '''has_pre_transformation'''	,   _UpperCAmelCase      )
       if self.has_pre_transformation:
             snake_case_ =   nn.Linear(config.hidden_size	,   config.project_dim      )
             snake_case_ =   nn.LayerNorm(config.hidden_size	,   eps=config.layer_norm_eps      )
       self.post_init()





 def 		UpperCamelCase__  (			self	,   _UpperCAmelCase = None	,   _UpperCAmelCase = None	,   _UpperCAmelCase = None	,   _UpperCAmelCase = None	,   _UpperCAmelCase = None	,   _UpperCAmelCase = None	,   _UpperCAmelCase = None	,   _UpperCAmelCase = None	,   _UpperCAmelCase = None	,   _UpperCAmelCase = None	,   _UpperCAmelCase = None	,   ):
       snake_case_ =   return_dict if return_dict is not None else self.config.use_return_dict

       snake_case_ =   self.base_model(
           input_ids=_UpperCAmelCase	,   attention_mask=_UpperCAmelCase	,   token_type_ids=_UpperCAmelCase	,   position_ids=_UpperCAmelCase	,   head_mask=_UpperCAmelCase	,   inputs_embeds=_UpperCAmelCase	,   encoder_hidden_states=_UpperCAmelCase	,   encoder_attention_mask=_UpperCAmelCase	,   output_attentions=_UpperCAmelCase	,   output_hidden_states=True if self.has_pre_transformation else output_hidden_states	,   return_dict=_UpperCAmelCase	,   )

       if self.has_pre_transformation:
             snake_case_ =   outputs['''hidden_states'''][-2]
             snake_case_ =   self.pre_LN(_UpperCAmelCase      )
             snake_case_ =   self.transformation_pre(_UpperCAmelCase      )

             return TransformationModelOutput(
                 projection_state=_UpperCAmelCase	,   last_hidden_state=outputs.last_hidden_state	,   hidden_states=outputs.hidden_states	,   attentions=outputs.attentions	,   )
       else:
             snake_case_ =   self.transformation(outputs.last_hidden_state      )
             return TransformationModelOutput(
                 projection_state=_UpperCAmelCase	,   last_hidden_state=outputs.last_hidden_state	,   hidden_states=outputs.hidden_states	,   attentions=outputs.attentions	,   )