
"""simple docstring"""


import torch
import torch.nn as nn
from transformers.modeling_utils import ModuleUtilsMixin
from transformers.models.ta.modeling_ta import (
    TaBlock,
    TaConfig,
    TaLayerNorm,
)

from ...configuration_utils import ConfigMixin, register_to_config
from ...models import ModelMixin







class 		lowerCamelCase__					(					lowerCAmelCase_ ,	lowerCAmelCase_ ,	lowerCAmelCase_				):





			'''simple docstring'''



			@register_to_config
			def __init__(							self     ,     __UpperCAmelCase     ,     __UpperCAmelCase     ,     __UpperCAmelCase     ,     __UpperCAmelCase     ,     __UpperCAmelCase     ,     __UpperCAmelCase     ,     __UpperCAmelCase     ,     __UpperCAmelCase     ,     __UpperCAmelCase     ,     __UpperCAmelCase = False     ,     ) ->     int:
				super().__init__()

				_lowerCAmelCase								=nn.Linear(_SCREAMING_SNAKE_CASE     ,     _SCREAMING_SNAKE_CASE     ,     bias=_SCREAMING_SNAKE_CASE  )

				_lowerCAmelCase								=nn.Embedding(_SCREAMING_SNAKE_CASE     ,     _SCREAMING_SNAKE_CASE  )
				_lowerCAmelCase								=False

				_lowerCAmelCase								=nn.Dropout(p=_SCREAMING_SNAKE_CASE  )

				_lowerCAmelCase								=TaConfig(
				    d_model=_SCREAMING_SNAKE_CASE     ,     num_heads=_SCREAMING_SNAKE_CASE     ,     d_kv=_SCREAMING_SNAKE_CASE     ,     d_ff=_SCREAMING_SNAKE_CASE     ,     feed_forward_proj=_SCREAMING_SNAKE_CASE     ,     dropout_rate=_SCREAMING_SNAKE_CASE     ,     is_decoder=_SCREAMING_SNAKE_CASE     ,     is_encoder_decoder=_SCREAMING_SNAKE_CASE     ,     )
				_lowerCAmelCase								=nn.ModuleList()
				for lyr_num in range(_SCREAMING_SNAKE_CASE  ):
					_lowerCAmelCase								=TaBlock(_SCREAMING_SNAKE_CASE  )
					self.encoders.append(_SCREAMING_SNAKE_CASE  )

				_lowerCAmelCase								=TaLayerNorm(_SCREAMING_SNAKE_CASE  )
				_lowerCAmelCase								=nn.Dropout(p=_SCREAMING_SNAKE_CASE  )



			def 					_lowerCAmelCase		(							self     ,     __UpperCAmelCase     ,     __UpperCAmelCase  ) ->     List[Any]:
				_lowerCAmelCase								=self.input_proj(_SCREAMING_SNAKE_CASE  )

				# terminal relative positional encodings
				_lowerCAmelCase								=encoder_inputs.shape[1]
				_lowerCAmelCase								=torch.arange(_SCREAMING_SNAKE_CASE     ,     device=encoder_inputs.device  )

				_lowerCAmelCase								=encoder_inputs_mask.sum(-1  )
				_lowerCAmelCase								=torch.roll(input_positions.unsqueeze(0  )     ,     tuple(seq_lens.tolist()  )     ,     dims=0  )
				x += self.position_encoding(_SCREAMING_SNAKE_CASE  )

				_lowerCAmelCase								=self.dropout_pre(_SCREAMING_SNAKE_CASE  )

				# inverted the attention mask
				_lowerCAmelCase								=encoder_inputs.size()
				_lowerCAmelCase								=self.get_extended_attention_mask(_SCREAMING_SNAKE_CASE     ,     _SCREAMING_SNAKE_CASE  )

				for lyr in self.encoders:
					_lowerCAmelCase								=lyr(_SCREAMING_SNAKE_CASE     ,     _SCREAMING_SNAKE_CASE  )[0]
				_lowerCAmelCase								=self.layer_norm(_SCREAMING_SNAKE_CASE  )

				return self.dropout_post(_SCREAMING_SNAKE_CASE  ), encoder_inputs_mask










from ...configuration_utils import PretrainedConfig
from ...utils import logging


_UpperCAmelCase				       = logging.get_logger(__name__)

_UpperCAmelCase				       = {
    'MCG-NJU/videomae-base': 'https://huggingface.co/MCG-NJU/videomae-base/resolve/main/config.json',
}



class 							_UpperCamelCase    (		lowerCAmelCase_   ):
       _UpperCamelCase :    List[Any]   			=      '''videomae'''

       def __init__(						self:       Optional[Any]    ,      _SCREAMING_SNAKE_CASE:       List[str]=224    ,      _SCREAMING_SNAKE_CASE:       Dict=16    ,      _SCREAMING_SNAKE_CASE:       List[Any]=3    ,      _SCREAMING_SNAKE_CASE:       Tuple=16    ,      _SCREAMING_SNAKE_CASE:       Tuple=2    ,      _SCREAMING_SNAKE_CASE:       Optional[Any]=768    ,      _SCREAMING_SNAKE_CASE:       str=12    ,      _SCREAMING_SNAKE_CASE:       List[Any]=12    ,      _SCREAMING_SNAKE_CASE:       Dict=3072    ,      _SCREAMING_SNAKE_CASE:       Tuple="gelu"    ,      _SCREAMING_SNAKE_CASE:       List[Any]=0.0    ,      _SCREAMING_SNAKE_CASE:       List[Any]=0.0    ,      _SCREAMING_SNAKE_CASE:       Tuple=0.02    ,      _SCREAMING_SNAKE_CASE:       Optional[int]=1e-12    ,      _SCREAMING_SNAKE_CASE:       Dict=True    ,      _SCREAMING_SNAKE_CASE:       Dict=True    ,      _SCREAMING_SNAKE_CASE:       List[str]=6    ,      _SCREAMING_SNAKE_CASE:       Dict=384    ,      _SCREAMING_SNAKE_CASE:       Any=4    ,      _SCREAMING_SNAKE_CASE:       Dict=1536    ,      _SCREAMING_SNAKE_CASE:       Optional[int]=True    ,      **_SCREAMING_SNAKE_CASE:       str    ,      )				->       Dict:




              """simple docstring"""


              super().__init__(**_SCREAMING_SNAKE_CASE						)

              UpperCamelCase_      				=     image_size
              UpperCamelCase_      				=     patch_size
              UpperCamelCase_      				=     num_channels
              UpperCamelCase_      				=     num_frames
              UpperCamelCase_      				=     tubelet_size

              UpperCamelCase_      				=     hidden_size
              UpperCamelCase_      				=     num_hidden_layers
              UpperCamelCase_      				=     num_attention_heads
              UpperCamelCase_      				=     intermediate_size
              UpperCamelCase_      				=     hidden_act
              UpperCamelCase_      				=     hidden_dropout_prob
              UpperCamelCase_      				=     attention_probs_dropout_prob
              UpperCamelCase_      				=     initializer_range
              UpperCamelCase_      				=     layer_norm_eps
              UpperCamelCase_      				=     qkv_bias
              UpperCamelCase_      				=     use_mean_pooling

              UpperCamelCase_      				=     decoder_num_attention_heads
              UpperCamelCase_      				=     decoder_hidden_size
              UpperCamelCase_      				=     decoder_num_hidden_layers
              UpperCamelCase_      				=     decoder_intermediate_size
              UpperCamelCase_      				=     norm_pix_loss

