
"""simple docstring"""




import logging

from transformers import PretrainedConfig


a		:       Union[str, Any]     						=       logging.getLogger(__name__)


a		:       Union[str, Any]     						=       {
    'bertabs-finetuned-cnndm': 'https://huggingface.co/remi/bertabs-finetuned-cnndm-extractive-abstractive-summarization/resolve/main/config.json',
}







class     lowercase(_lowercase       ):
     __snake_case:	Optional[int]						=     'bertabs'



     def __init__(							self     ,				__SCREAMING_SNAKE_CASE=3_0_5_2_2     ,				__SCREAMING_SNAKE_CASE=5_1_2     ,				__SCREAMING_SNAKE_CASE=6     ,				__SCREAMING_SNAKE_CASE=5_1_2     ,				__SCREAMING_SNAKE_CASE=8     ,				__SCREAMING_SNAKE_CASE=5_1_2     ,				__SCREAMING_SNAKE_CASE=0.2     ,				__SCREAMING_SNAKE_CASE=6     ,				__SCREAMING_SNAKE_CASE=7_6_8     ,				__SCREAMING_SNAKE_CASE=8     ,				__SCREAMING_SNAKE_CASE=2_0_4_8     ,				__SCREAMING_SNAKE_CASE=0.2     ,				**__SCREAMING_SNAKE_CASE     ,				)		->     int:







         """simple docstring"""



         super().__init__(**__SCREAMING_SNAKE_CASE   )

         a__		=			vocab_size
         a__		=			max_pos

         a__		=			enc_layers
         a__		=			enc_hidden_size
         a__		=			enc_heads
         a__		=			enc_ff_size
         a__		=			enc_dropout

         a__		=			dec_layers
         a__		=			dec_hidden_size
         a__		=			dec_heads
         a__		=			dec_ff_size
         a__		=			dec_dropout




"""simple docstring"""




import torch
import torch.nn as nn
from transformers.modeling_utils import ModuleUtilsMixin
from transformers.models.ta.modeling_ta import TaBlock, TaConfig, TaLayerNorm

from ...configuration_utils import ConfigMixin, register_to_config
from ...models import ModelMixin







class     lowercase(_lowercase    ,						_lowercase    ,						_lowercase       ):



     @register_to_config
     def __init__(							self     ,				__SCREAMING_SNAKE_CASE     ,				__SCREAMING_SNAKE_CASE     ,				__SCREAMING_SNAKE_CASE     ,				__SCREAMING_SNAKE_CASE     ,				__SCREAMING_SNAKE_CASE     ,				__SCREAMING_SNAKE_CASE     ,				__SCREAMING_SNAKE_CASE     ,				__SCREAMING_SNAKE_CASE     ,				__SCREAMING_SNAKE_CASE     ,				__SCREAMING_SNAKE_CASE = False     ,				)		->     Union[str, Any]:







         """simple docstring"""



         super().__init__()

         a__		=			nn.Embedding(__SCREAMING_SNAKE_CASE     ,				__SCREAMING_SNAKE_CASE   )

         a__		=			nn.Embedding(__SCREAMING_SNAKE_CASE     ,				__SCREAMING_SNAKE_CASE   )
         a__		=			False

         a__		=			nn.Dropout(p=__SCREAMING_SNAKE_CASE   )

         a__		=			TaConfig(
             vocab_size=__SCREAMING_SNAKE_CASE     ,				d_model=__SCREAMING_SNAKE_CASE     ,				num_heads=__SCREAMING_SNAKE_CASE     ,				d_kv=__SCREAMING_SNAKE_CASE     ,				d_ff=__SCREAMING_SNAKE_CASE     ,				dropout_rate=__SCREAMING_SNAKE_CASE     ,				feed_forward_proj=__SCREAMING_SNAKE_CASE     ,				is_decoder=__SCREAMING_SNAKE_CASE     ,				is_encoder_decoder=__SCREAMING_SNAKE_CASE     ,				)

         a__		=			nn.ModuleList()
         for lyr_num in range(__SCREAMING_SNAKE_CASE   ):
             a__		=			TaBlock(__SCREAMING_SNAKE_CASE   )
             self.encoders.append(__SCREAMING_SNAKE_CASE   )

         a__		=			TaLayerNorm(__SCREAMING_SNAKE_CASE   )
         a__		=			nn.Dropout(p=__SCREAMING_SNAKE_CASE   )




     def 			lowercase__  (							self     ,				__SCREAMING_SNAKE_CASE     ,				__SCREAMING_SNAKE_CASE   )		->     List[Any]:







         """simple docstring"""



         a__		=			self.token_embedder(__SCREAMING_SNAKE_CASE   )

         a__		=			encoder_input_tokens.shape[1]
         a__		=			torch.arange(__SCREAMING_SNAKE_CASE     ,				device=encoder_input_tokens.device   )
         x += self.position_encoding(__SCREAMING_SNAKE_CASE   )

         a__		=			self.dropout_pre(__SCREAMING_SNAKE_CASE   )

         # inverted the attention mask
         a__		=			encoder_input_tokens.size()
         a__		=			self.get_extended_attention_mask(__SCREAMING_SNAKE_CASE     ,				__SCREAMING_SNAKE_CASE   )

         for lyr in self.encoders:
             a__		=			lyr(__SCREAMING_SNAKE_CASE     ,				__SCREAMING_SNAKE_CASE   )[0]
         a__		=			self.layer_norm(__SCREAMING_SNAKE_CASE   )

         return self.dropout_post(__SCREAMING_SNAKE_CASE   ), encoder_inputs_mask


