




from ... import PretrainedConfig


__a							     =					{
    'sijunhe/nezha-cn-base': 'https://huggingface.co/sijunhe/nezha-cn-base/resolve/main/config.json',
}

class     __a(	_a  ):






    """simple docstring"""





    lowerCAmelCase         = NEZHA_PRETRAINED_CONFIG_ARCHIVE_MAP
    lowerCAmelCase         = '''nezha'''





    def __init__( self  ,_SCREAMING_SNAKE_CASE=21_128  ,_SCREAMING_SNAKE_CASE=768  ,_SCREAMING_SNAKE_CASE=12  ,_SCREAMING_SNAKE_CASE=12  ,_SCREAMING_SNAKE_CASE=3_072  ,_SCREAMING_SNAKE_CASE="gelu"  ,_SCREAMING_SNAKE_CASE=0.1  ,_SCREAMING_SNAKE_CASE=0.1  ,_SCREAMING_SNAKE_CASE=512  ,_SCREAMING_SNAKE_CASE=64  ,_SCREAMING_SNAKE_CASE=2  ,_SCREAMING_SNAKE_CASE=0.02  ,_SCREAMING_SNAKE_CASE=1e-12  ,_SCREAMING_SNAKE_CASE=0.1  ,_SCREAMING_SNAKE_CASE=0  ,_SCREAMING_SNAKE_CASE=2  ,_SCREAMING_SNAKE_CASE=3  ,_SCREAMING_SNAKE_CASE=True  ,**_SCREAMING_SNAKE_CASE  ,)					->						List[Any]:
         super().__init__(pad_token_id=_SCREAMING_SNAKE_CASE  ,bos_token_id=_SCREAMING_SNAKE_CASE  ,eos_token_id=_SCREAMING_SNAKE_CASE  ,**_SCREAMING_SNAKE_CASE    )

         UpperCAmelCase_     :				str  =     vocab_size
         UpperCAmelCase_     :				Tuple  =     hidden_size
         UpperCAmelCase_     :				Dict  =     num_hidden_layers
         UpperCAmelCase_     :				Dict  =     num_attention_heads
         UpperCAmelCase_     :				Tuple  =     hidden_act
         UpperCAmelCase_     :				List[str]  =     intermediate_size
         UpperCAmelCase_     :				str  =     hidden_dropout_prob
         UpperCAmelCase_     :				Tuple  =     attention_probs_dropout_prob
         UpperCAmelCase_     :				Any  =     max_position_embeddings
         UpperCAmelCase_     :				str  =     max_relative_position
         UpperCAmelCase_     :				str  =     type_vocab_size
         UpperCAmelCase_     :				Tuple  =     initializer_range
         UpperCAmelCase_     :				Any  =     layer_norm_eps
         UpperCAmelCase_     :				Any  =     classifier_dropout
         UpperCAmelCase_     :				Dict  =     use_cache






import torch
import torch.nn as nn
from transformers.modeling_utils import ModuleUtilsMixin
from transformers.models.ta.modeling_ta import TaBlock, TaConfig, TaLayerNorm

from ...configuration_utils import ConfigMixin, register_to_config
from ...models import ModelMixin
class     UpperCamelCase__					(     lowerCamelCase__				, lowerCamelCase__				, lowerCamelCase__				):






		'''simple docstring'''






		@register_to_config
		def __init__(							self,   snake_case__,   snake_case__,   snake_case__,   snake_case__,   snake_case__,   snake_case__,   snake_case__,   snake_case__,   snake_case__,   snake_case__ = False,   )       -> Any:






									"""simple docstring"""
									super().__init__()

									lowercase_	:   str  				=     nn.Embedding(snake_case__,   snake_case__  )

									lowercase_	:   Dict  				=     nn.Embedding(snake_case__,   snake_case__  )
									lowercase_	:   List[str]  				=     False

									lowercase_	:   Optional[int]  				=     nn.Dropout(p=snake_case__  )

									lowercase_	:   Any  				=     TaConfig(
									    vocab_size=snake_case__,   d_model=snake_case__,   num_heads=snake_case__,   d_kv=snake_case__,   d_ff=snake_case__,   dropout_rate=snake_case__,   feed_forward_proj=snake_case__,   is_decoder=snake_case__,   is_encoder_decoder=snake_case__,   )

									lowercase_	:   List[Any]  				=     nn.ModuleList()
									for lyr_num in range(snake_case__  ):
																lowercase_	:   List[str]  				=     TaBlock(snake_case__  )
																self.encoders.append(snake_case__  )

									lowercase_	:   Tuple  				=     TaLayerNorm(snake_case__  )
									lowercase_	:   Dict  				=     nn.Dropout(p=snake_case__  )






		def 						snake_case__				(							self,   snake_case__,   snake_case__  )       -> Union[str, Any]:






									"""simple docstring"""
									lowercase_	:   Union[str, Any]  				=     self.token_embedder(snake_case__  )

									lowercase_	:   Tuple  				=     encoder_input_tokens.shape[1]
									lowercase_	:   List[str]  				=     torch.arange(snake_case__,   device=encoder_input_tokens.device  )
									x += self.position_encoding(snake_case__  )

									lowercase_	:   Union[str, Any]  				=     self.dropout_pre(snake_case__  )

									# inverted the attention mask
									lowercase_	:   List[str]  				=     encoder_input_tokens.size()
									lowercase_	:   Union[str, Any]  				=     self.get_extended_attention_mask(snake_case__,   snake_case__  )

									for lyr in self.encoders:
																lowercase_	:   Optional[Any]  				=     lyr(snake_case__,   snake_case__  )[0]
									lowercase_	:   Dict  				=     self.layer_norm(snake_case__  )

									return self.dropout_post(snake_case__  ), encoder_inputs_mask