from ... import PretrainedConfig


SCREAMING_SNAKE_CASE :	Any							     =		{
    """sijunhe/nezha-cn-base""": """https://huggingface.co/sijunhe/nezha-cn-base/resolve/main/config.json""",
}
class    A_ (       a_ ):
		_SCREAMING_SNAKE_CASE    				=			NEZHA_PRETRAINED_CONFIG_ARCHIVE_MAP
		_SCREAMING_SNAKE_CASE    				=			"""nezha"""

		def __init__(		self :      Any     ,							__SCREAMING_SNAKE_CASE :      List[str]=2_11_28     ,							__SCREAMING_SNAKE_CASE :      Dict=7_68     ,							__SCREAMING_SNAKE_CASE :      List[Any]=12     ,							__SCREAMING_SNAKE_CASE :      str=12     ,							__SCREAMING_SNAKE_CASE :      Optional[Any]=30_72     ,							__SCREAMING_SNAKE_CASE :      Any="gelu"     ,							__SCREAMING_SNAKE_CASE :      int=0.1     ,							__SCREAMING_SNAKE_CASE :      List[str]=0.1     ,							__SCREAMING_SNAKE_CASE :      int=5_12     ,							__SCREAMING_SNAKE_CASE :      Union[str, Any]=64     ,							__SCREAMING_SNAKE_CASE :      List[Any]=2     ,							__SCREAMING_SNAKE_CASE :      int=0.02     ,							__SCREAMING_SNAKE_CASE :      Optional[Any]=1E-12     ,							__SCREAMING_SNAKE_CASE :      int=0.1     ,							__SCREAMING_SNAKE_CASE :      Any=0     ,							__SCREAMING_SNAKE_CASE :      Any=2     ,							__SCREAMING_SNAKE_CASE :      Optional[Any]=3     ,							__SCREAMING_SNAKE_CASE :      List[Any]=True     ,							**__SCREAMING_SNAKE_CASE :      List[str]     ,							):
			super().__init__(pad_token_id=__SCREAMING_SNAKE_CASE     ,							bos_token_id=__SCREAMING_SNAKE_CASE     ,							eos_token_id=__SCREAMING_SNAKE_CASE     ,							**__SCREAMING_SNAKE_CASE       )

			__a       =  vocab_size
			__a       =  hidden_size
			__a       =  num_hidden_layers
			__a       =  num_attention_heads
			__a       =  hidden_act
			__a       =  intermediate_size
			__a       =  hidden_dropout_prob
			__a       =  attention_probs_dropout_prob
			__a       =  max_position_embeddings
			__a       =  max_relative_position
			__a       =  type_vocab_size
			__a       =  initializer_range
			__a       =  layer_norm_eps
			__a       =  classifier_dropout
			__a       =  use_cache



import torch
import torch.nn as nn
from transformers.modeling_utils import ModuleUtilsMixin
from transformers.models.ta.modeling_ta import TaBlock, TaConfig, TaLayerNorm

from ...configuration_utils import ConfigMixin, register_to_config
from ...models import ModelMixin
class    A_ (       a_			,       a_			,       a_ ):

  @register_to_config
  def __init__(		self :      List[Any]     ,							__SCREAMING_SNAKE_CASE :      int     ,							__SCREAMING_SNAKE_CASE :      int     ,							__SCREAMING_SNAKE_CASE :      int     ,							__SCREAMING_SNAKE_CASE :      float     ,							__SCREAMING_SNAKE_CASE :      int     ,							__SCREAMING_SNAKE_CASE :      int     ,							__SCREAMING_SNAKE_CASE :      int     ,							__SCREAMING_SNAKE_CASE :      int     ,							__SCREAMING_SNAKE_CASE :      str     ,							__SCREAMING_SNAKE_CASE :      bool = False     ,							):
   super().__init__()

   __a       =  nn.Embedding(__SCREAMING_SNAKE_CASE     ,							__SCREAMING_SNAKE_CASE       )

   __a       =  nn.Embedding(__SCREAMING_SNAKE_CASE     ,							__SCREAMING_SNAKE_CASE       )
   __a       =  False

   __a       =  nn.Dropout(p=__SCREAMING_SNAKE_CASE       )

   __a       =  TaConfig(
       vocab_size=__SCREAMING_SNAKE_CASE     ,							d_model=__SCREAMING_SNAKE_CASE     ,							num_heads=__SCREAMING_SNAKE_CASE     ,							d_kv=__SCREAMING_SNAKE_CASE     ,							d_ff=__SCREAMING_SNAKE_CASE     ,							dropout_rate=__SCREAMING_SNAKE_CASE     ,							feed_forward_proj=__SCREAMING_SNAKE_CASE     ,							is_decoder=__SCREAMING_SNAKE_CASE     ,							is_encoder_decoder=__SCREAMING_SNAKE_CASE     ,							)

   __a       =  nn.ModuleList()
   for lyr_num in range(__SCREAMING_SNAKE_CASE       ):
    __a       =  TaBlock(__SCREAMING_SNAKE_CASE       )
    self.encoders.append(__SCREAMING_SNAKE_CASE       )

   __a       =  TaLayerNorm(__SCREAMING_SNAKE_CASE       )
   __a       =  nn.Dropout(p=__SCREAMING_SNAKE_CASE       )

  def 		_UpperCAmelCase   (		self :      int     ,							__SCREAMING_SNAKE_CASE :      List[str]     ,							__SCREAMING_SNAKE_CASE :      Any       ):
   __a       =  self.token_embedder(__SCREAMING_SNAKE_CASE       )

   __a       =  encoder_input_tokens.shape[1]
   __a       =  torch.arange(__SCREAMING_SNAKE_CASE     ,							device=encoder_input_tokens.device       )
   x += self.position_encoding(__SCREAMING_SNAKE_CASE       )

   __a       =  self.dropout_pre(__SCREAMING_SNAKE_CASE       )

   # inverted the attention mask
   __a       =  encoder_input_tokens.size()
   __a       =  self.get_extended_attention_mask(__SCREAMING_SNAKE_CASE     ,							__SCREAMING_SNAKE_CASE       )

   for lyr in self.encoders:
    __a       =  lyr(__SCREAMING_SNAKE_CASE     ,							__SCREAMING_SNAKE_CASE       )[0]
   __a       =  self.layer_norm(__SCREAMING_SNAKE_CASE       )

   return self.dropout_post(__SCREAMING_SNAKE_CASE       ), encoder_inputs_mask


