





'''simple docstring'''




from dataclasses import dataclass
from typing import Optional, Tuple

import torch
from torch import nn
from transformers import RobertaPreTrainedModel, XLMRobertaConfig, XLMRobertaModel
from transformers.utils import ModelOutput


@dataclass
class 					SCREAMING_SNAKE_CASE       (a__				):
     lowerCAmelCase						     =			None
     lowerCAmelCase						     =			None
     lowerCAmelCase						     =			None
     lowerCAmelCase						     =			None




class 					SCREAMING_SNAKE_CASE       (a__				):


     def __init__(		self		,      _UpperCAmelCase=1		,      _UpperCAmelCase=0		,      _UpperCAmelCase=2		,      _UpperCAmelCase=512		,      _UpperCAmelCase="cls"		,      _UpperCAmelCase=False		,      _UpperCAmelCase=True		,      **_UpperCAmelCase		,      ):

          '''simple docstring'''






          super().__init__(pad_token_id=_UpperCAmelCase		,      bos_token_id=_UpperCAmelCase		,      eos_token_id=_UpperCAmelCase		,      **_UpperCAmelCase)
          __A		:					Any						=			project_dim
          __A		:					List[str]						=			pooler_fn
          __A		:					Tuple						=			learn_encoder
          __A		:					Optional[int]						=			use_attention_mask




class 					SCREAMING_SNAKE_CASE       (a__				):
     lowerCAmelCase						     =			[r'''pooler''', r'''logit_scale''']
     lowerCAmelCase						     =			[r'''position_ids''', r'''predictions.decoder.bias''']
     lowerCAmelCase						     =			'''roberta'''
     lowerCAmelCase						     =			RobertaSeriesConfig


     def __init__(		self		,      _UpperCAmelCase):

          '''simple docstring'''






          super().__init__(_UpperCAmelCase)
          __A		:					List[str]						=			XLMRobertaModel(_UpperCAmelCase)
          __A		:					List[str]						=			nn.Linear(config.hidden_size		,      config.project_dim)
          __A		:					List[Any]						=			getattr(_UpperCAmelCase		,      'has_pre_transformation'		,      _UpperCAmelCase)
          if self.has_pre_transformation:
               __A		:					Tuple						=			nn.Linear(config.hidden_size		,      config.project_dim)
               __A		:					Optional[int]						=			nn.LayerNorm(config.hidden_size		,      eps=config.layer_norm_eps)
          self.post_init()


     def 				SCREAMING_SNAKE_CASE    (		self		,      _UpperCAmelCase = None		,      _UpperCAmelCase = None		,      _UpperCAmelCase = None		,      _UpperCAmelCase = None		,      _UpperCAmelCase = None		,      _UpperCAmelCase = None		,      _UpperCAmelCase = None		,      _UpperCAmelCase = None		,      _UpperCAmelCase = None		,      _UpperCAmelCase = None		,      _UpperCAmelCase = None		,      ):

          '''simple docstring'''






          __A		:					Optional[Any]						=			return_dict if return_dict is not None else self.config.use_return_dict

          __A		:					Any						=			self.base_model(
              input_ids=_UpperCAmelCase		,      attention_mask=_UpperCAmelCase		,      token_type_ids=_UpperCAmelCase		,      position_ids=_UpperCAmelCase		,      head_mask=_UpperCAmelCase		,      inputs_embeds=_UpperCAmelCase		,      encoder_hidden_states=_UpperCAmelCase		,      encoder_attention_mask=_UpperCAmelCase		,      output_attentions=_UpperCAmelCase		,      output_hidden_states=True if self.has_pre_transformation else output_hidden_states		,      return_dict=_UpperCAmelCase		,      )

          if self.has_pre_transformation:
               __A		:					Optional[int]						=			outputs['hidden_states'][-2]
               __A		:					Dict						=			self.pre_LN(_UpperCAmelCase)
               __A		:					Optional[Any]						=			self.transformation_pre(_UpperCAmelCase)

               return TransformationModelOutput(
                   projection_state=_UpperCAmelCase		,      last_hidden_state=outputs.last_hidden_state		,      hidden_states=outputs.hidden_states		,      attentions=outputs.attentions		,      )
          else:
               __A		:					Optional[Any]						=			self.transformation(outputs.last_hidden_state)
               return TransformationModelOutput(
                   projection_state=_UpperCAmelCase		,      last_hidden_state=outputs.last_hidden_state		,      hidden_states=outputs.hidden_states		,      attentions=outputs.attentions		,      )



from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowerCAmelCase       		=    logging.get_logger(__name__)

lowerCAmelCase       		=    {
    """abeja/gpt-neox-japanese-2.7b""": """https://huggingface.co/abeja/gpt-neox-japanese-2.7b/resolve/main/config.json""",
}




class 							lowerCamelCase   (  _UpperCamelCase ):
	_lowerCAmelCase     :		Tuple										=						'''gpt_neox_japanese'''


	def __init__(    self				,     lowercase__=3_2_0_0_0				,     lowercase__=2_5_6_0				,     lowercase__=3_2				,     lowercase__=3_2				,     lowercase__=4				,     lowercase__="gelu"				,     lowercase__=1.0_0				,     lowercase__=1_0_0_0_0				,     lowercase__=2_0_4_8				,     lowercase__=0.0_2				,     lowercase__=1e-5				,     lowercase__=True				,     lowercase__=3_1_9_9_6				,     lowercase__=3_1_9_9_9				,     lowercase__=0.1				,     lowercase__=0.0				,     **lowercase__				,     ):
							super().__init__(bos_token_id=lowercase__				,     eos_token_id=lowercase__				,     **lowercase__)
							__UpperCAmelCase			:							Optional[Any]	    =	vocab_size
							__UpperCAmelCase			:							Dict	    =	max_position_embeddings
							__UpperCAmelCase			:							int	    =	hidden_size
							__UpperCAmelCase			:							List[Any]	    =	num_hidden_layers
							__UpperCAmelCase			:							Union[str, Any]	    =	num_attention_heads
							__UpperCAmelCase			:							Optional[int]	    =	intermediate_multiple_size
							__UpperCAmelCase			:							Any	    =	hidden_act
							__UpperCAmelCase			:							str	    =	rotary_pct
							__UpperCAmelCase			:							Any	    =	rotary_emb_base
							__UpperCAmelCase			:							Dict	    =	initializer_range
							__UpperCAmelCase			:							Tuple	    =	layer_norm_eps
							__UpperCAmelCase			:							Optional[int]	    =	use_cache
							__UpperCAmelCase			:							str	    =	attention_dropout
							__UpperCAmelCase			:							Any	    =	hidden_dropout

