

from dataclasses import asdict, dataclass
from typing import Optional

from ...configuration_utils import PretrainedConfig
from ...utils import logging


__snake_case				:      Optional[int]   		=			logging.get_logger(__name__)

# TODO Update this
__snake_case				:      Any   		=			{
    """facebook/esm-1b""": """https://huggingface.co/facebook/esm-1b/resolve/main/config.json""",
    # See all ESM models at https://huggingface.co/models?filter=esm
}



class        A__(_a							):
						"""simple docstring"""






						_A    :				Optional[int]       					=			'''esm'''





						def __init__(    self  ,      _lowercase=None  ,      _lowercase=None  ,      _lowercase=None  ,      _lowercase=768  ,      _lowercase=12  ,      _lowercase=12  ,      _lowercase=3_072  ,      _lowercase=0.1  ,      _lowercase=0.1  ,      _lowercase=1_026  ,      _lowercase=0.0_2  ,      _lowercase=1e-12  ,      _lowercase="absolute"  ,      _lowercase=True  ,      _lowercase=None  ,      _lowercase=False  ,      _lowercase=False  ,      _lowercase=None  ,      _lowercase=None  ,      **_lowercase  ,      )     -> Dict:
									super().__init__(pad_token_id=_lowercase  ,      mask_token_id=_lowercase  ,      **_lowercase    )

									a_						:						Dict 							= vocab_size
									a_						:						int 							= hidden_size
									a_						:						List[str] 							= num_hidden_layers
									a_						:						Union[str, Any] 							= num_attention_heads
									a_						:						int 							= intermediate_size
									a_						:						Optional[Any] 							= hidden_dropout_prob
									a_						:						Any 							= attention_probs_dropout_prob
									a_						:						Any 							= max_position_embeddings
									a_						:						Optional[Any] 							= initializer_range
									a_						:						Optional[int] 							= layer_norm_eps
									a_						:						List[Any] 							= position_embedding_type
									a_						:						str 							= use_cache
									a_						:						Optional[int] 							= emb_layer_norm_before
									a_						:						Union[str, Any] 							= token_dropout
									a_						:						str 							= is_folding_model
									if is_folding_model:
												if esmfold_config is None:
															logger.info("""No esmfold_config supplied for folding model, using default values."""    )
															a_						:						Union[str, Any] 							= EsmFoldConfig()
												elif isinstance(_lowercase  ,      _lowercase    ):
															a_						:						Dict 							= EsmFoldConfig(**_lowercase    )
												a_						:						int 							= esmfold_config
												if vocab_list is None:
															logger.warning("""No vocab_list supplied for folding model, assuming the ESM-2 vocabulary!"""    )
															a_						:						Optional[int] 							= get_default_vocab_list()
												else:
															a_						:						Any 							= vocab_list
									else:
												a_						:						Optional[int] 							= None
												a_						:						List[str] 							= None
									if self.esmfold_config is not None and getattr(self.esmfold_config  ,      """use_esm_attn_map"""  ,      _lowercase    ):
												raise ValueError("""The HuggingFace port of ESMFold does not support use_esm_attn_map at this time!"""    )





						def 			UpperCamelCase__					(    self    )     -> str:
									a_						:						List[str] 							= super().to_dict()
									if isinstance(self.esmfold_config  ,      _lowercase    ):
												a_						:						int 							= self.esmfold_config.to_dict()
									return output







@dataclass
class        A__:
						"""simple docstring"""






						_A    :				str       					=			None
						_A    :				bool       					=			True
						_A    :				bool       					=			False
						_A    :				bool       					=			False
						_A    :				bool       					=			False
						_A    :				float       					=			0

						_A    :				bool       					=			True
						_A    :				bool       					=			False

						_A    :				int       					=			128
						_A    :				"TrunkConfig"       					=			None





						def 			UpperCamelCase__					(    self    )     -> Any:

									if self.trunk is None:
												a_						:						List[Any] 							= TrunkConfig()
									elif isinstance(self.trunk  ,      _lowercase    ):
												a_						:						Optional[int] 							= TrunkConfig(**self.trunk    )





						def 			UpperCamelCase__					(    self    )     -> int:
									a_						:						Any 							= asdict(self    )
									a_						:						Any 							= self.trunk.to_dict()
									return output







@dataclass
class        A__:
						"""simple docstring"""






						_A    :				int       					=			48
						_A    :				int       					=			1_024
						_A    :				int       					=			128
						_A    :				int       					=			32
						_A    :				int       					=			32
						_A    :				int       					=			32
						_A    :				float       					=			0
						_A    :				float       					=			0
						_A    :				bool       					=			False
						_A    :				int       					=			4
						_A    :				Optional[int]       					=			128
						_A    :				"StructureModuleConfig"       					=			None





						def 			UpperCamelCase__					(    self    )     -> List[Any]:

									if self.structure_module is None:
												a_						:						Optional[int] 							= StructureModuleConfig()
									elif isinstance(self.structure_module  ,      _lowercase    ):
												a_						:						Tuple 							= StructureModuleConfig(**self.structure_module    )

									if self.max_recycles <= 0:
												raise ValueError(F'''`max_recycles` should be positive, got {self.max_recycles}.'''    )
									if self.sequence_state_dim % self.sequence_state_dim != 0:
												raise ValueError(
												    """`sequence_state_dim` should be a round multiple of `sequence_state_dim`, got"""
												    F''' {self.sequence_state_dim} and {self.sequence_state_dim}.'''    )
									if self.pairwise_state_dim % self.pairwise_state_dim != 0:
												raise ValueError(
												    """`pairwise_state_dim` should be a round multiple of `pairwise_state_dim`, got"""
												    F''' {self.pairwise_state_dim} and {self.pairwise_state_dim}.'''    )

									a_						:						Tuple 							= self.sequence_state_dim // self.sequence_head_width
									a_						:						int 							= self.pairwise_state_dim // self.pairwise_head_width

									if self.sequence_state_dim != sequence_num_heads * self.sequence_head_width:
												raise ValueError(
												    """`sequence_state_dim` should be equal to `sequence_num_heads * sequence_head_width, got"""
												    F''' {self.sequence_state_dim} != {sequence_num_heads} * {self.sequence_head_width}.'''    )
									if self.pairwise_state_dim != pairwise_num_heads * self.pairwise_head_width:
												raise ValueError(
												    """`pairwise_state_dim` should be equal to `pairwise_num_heads * pairwise_head_width, got"""
												    F''' {self.pairwise_state_dim} != {pairwise_num_heads} * {self.pairwise_head_width}.'''    )
									if self.pairwise_state_dim % 2 != 0:
												raise ValueError(F'''`pairwise_state_dim` should be even, got {self.pairwise_state_dim}.'''    )

									if self.dropout >= 0.4:
												raise ValueError(F'''`dropout` should not be greater than 0.4, got {self.dropout}.'''    )





						def 			UpperCamelCase__					(    self    )     -> str:
									a_						:						Tuple 							= asdict(self    )
									a_						:						Dict 							= self.structure_module.to_dict()
									return output







@dataclass
class        A__:
						"""simple docstring"""






						_A    :				int       					=			384
						_A    :				int       					=			128
						_A    :				int       					=			16
						_A    :				int       					=			128
						_A    :				int       					=			12
						_A    :				int       					=			4
						_A    :				int       					=			8
						_A    :				float       					=			0.1
						_A    :				int       					=			8
						_A    :				int       					=			1
						_A    :				int       					=			2
						_A    :				int       					=			7
						_A    :				int       					=			10
						_A    :				float       					=			1E-8
						_A    :				float       					=			1E5





						def 			UpperCamelCase__					(    self    )     -> Optional[Any]:
									return asdict(self    )







def 					_UpperCAmelCase				(						):

			'''simple docstring'''



			return (
			    "<cls>",
			    "<pad>",
			    "<eos>",
			    "<unk>",
			    "L",
			    "A",
			    "G",
			    "V",
			    "S",
			    "E",
			    "R",
			    "T",
			    "I",
			    "D",
			    "P",
			    "K",
			    "Q",
			    "N",
			    "F",
			    "Y",
			    "M",
			    "H",
			    "W",
			    "C",
			    "X",
			    "B",
			    "U",
			    "Z",
			    "O",
			    ".",
			    "-",
			    "<null_1>",
			    "<mask>",
			)










import unittest

from transformers import MraConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask


if is_torch_available():
    import torch

    from transformers import (
        MraForMaskedLM,
        MraForMultipleChoice,
        MraForQuestionAnswering,
        MraForSequenceClassification,
        MraForTokenClassification,
        MraModel,
    )
    from transformers.models.mra.modeling_mra import MRA_PRETRAINED_MODEL_ARCHIVE_LIST






class 					SCREAMING_SNAKE_CASE_  :
       """simple docstring"""


       def __init__(				self	:str,      snake_case	:Any,      snake_case	:str=2,      snake_case	:int=8,      snake_case	:List[Any]=True,      snake_case	:str=True,      snake_case	:Optional[int]=True,      snake_case	:str=True,      snake_case	:Optional[Any]=99,      snake_case	:Tuple=16,      snake_case	:Optional[Any]=5,      snake_case	:Optional[Any]=2,      snake_case	:Optional[Any]=36,      snake_case	:List[str]="gelu",      snake_case	:str=0.0,      snake_case	:List[Any]=0.0,      snake_case	:List[str]=512,      snake_case	:Dict=16,      snake_case	:int=2,      snake_case	:Tuple=0.0_2,      snake_case	:List[str]=3,      snake_case	:Any=4,      snake_case	:Union[str, Any]=None,      ):



          """simple docstring"""

          _lowercase              =parent
          _lowercase              =batch_size
          _lowercase              =seq_length
          _lowercase              =is_training
          _lowercase              =use_input_mask
          _lowercase              =use_token_type_ids
          _lowercase              =use_labels
          _lowercase              =vocab_size
          _lowercase              =hidden_size
          _lowercase              =num_hidden_layers
          _lowercase              =num_attention_heads
          _lowercase              =intermediate_size
          _lowercase              =hidden_act
          _lowercase              =hidden_dropout_prob
          _lowercase              =attention_probs_dropout_prob
          _lowercase              =max_position_embeddings
          _lowercase              =type_vocab_size
          _lowercase              =type_sequence_label_size
          _lowercase              =initializer_range
          _lowercase              =num_labels
          _lowercase              =num_choices
          _lowercase              =scope





       def 				UpperCamelCase__						(				self	:int):



          """simple docstring"""

          _lowercase              =ids_tensor([self.batch_size, self.seq_length],      self.vocab_size)

          _lowercase              =None
          if self.use_input_mask:
             _lowercase              =random_attention_mask([self.batch_size, self.seq_length])

          _lowercase              =None
          if self.use_token_type_ids:
             _lowercase              =ids_tensor([self.batch_size, self.seq_length],      self.type_vocab_size)

          _lowercase              =None
          _lowercase              =None
          _lowercase              =None
          if self.use_labels:
             _lowercase              =ids_tensor([self.batch_size],      self.type_sequence_label_size)
             _lowercase              =ids_tensor([self.batch_size, self.seq_length],      self.num_labels)
             _lowercase              =ids_tensor([self.batch_size],      self.num_choices)

          _lowercase              =self.get_config()

          return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels





       def 				UpperCamelCase__						(				self	:Union[str, Any]):



          """simple docstring"""

          return MraConfig(
              vocab_size=self.vocab_size,      hidden_size=self.hidden_size,      num_hidden_layers=self.num_hidden_layers,      num_attention_heads=self.num_attention_heads,      intermediate_size=self.intermediate_size,      hidden_act=self.hidden_act,      hidden_dropout_prob=self.hidden_dropout_prob,      attention_probs_dropout_prob=self.attention_probs_dropout_prob,      max_position_embeddings=self.max_position_embeddings,      type_vocab_size=self.type_vocab_size,      is_decoder=snake_case,      initializer_range=self.initializer_range,      )





       def 				UpperCamelCase__						(				self	:Optional[Any]):



          """simple docstring"""

          _lowercase              =self.get_config()
          _lowercase              =300
          return config





       def 				UpperCamelCase__						(				self	:Any):



          """simple docstring"""

          (
              (
              _lowercase
          )       ,       (
              _lowercase
          )       ,       (
              _lowercase
          )       ,       (
              _lowercase
          )       ,       (
              _lowercase
          )       ,       (
              _lowercase
          )       ,       (
              _lowercase
          )       ,       
          )              =self.prepare_config_and_inputs()

          _lowercase              =True
          _lowercase              =floats_tensor([self.batch_size, self.seq_length, self.hidden_size])
          _lowercase              =ids_tensor([self.batch_size, self.seq_length],      vocab_size=2)

          return (
              config,
              input_ids,
              token_type_ids,
              input_mask,
              sequence_labels,
              token_labels,
              choice_labels,
              encoder_hidden_states,
              encoder_attention_mask,
          )





       def 				UpperCamelCase__						(				self	:Optional[int],      snake_case	:Optional[int],      snake_case	:List[str],      snake_case	:int,      snake_case	:Tuple,      snake_case	:Optional[int],      snake_case	:Dict,      snake_case	:str):



          """simple docstring"""

          _lowercase              =MraModel(config=snake_case)
          model.to(snake_case)
          model.eval()
          _lowercase              =model(snake_case,      attention_mask=snake_case,      token_type_ids=snake_case)
          _lowercase              =model(snake_case,      token_type_ids=snake_case)
          _lowercase              =model(snake_case)
          self.parent.assertEqual(result.last_hidden_state.shape,      (self.batch_size, self.seq_length, self.hidden_size))





       def 				UpperCamelCase__						(				self	:Any,      snake_case	:Any,      snake_case	:str,      snake_case	:Optional[int],      snake_case	:int,      snake_case	:int,      snake_case	:int,      snake_case	:Optional[Any],      snake_case	:Tuple,      snake_case	:Tuple,      ):



          """simple docstring"""

          _lowercase              =True
          _lowercase              =MraModel(snake_case)
          model.to(snake_case)
          model.eval()
          _lowercase              =model(
              snake_case,      attention_mask=snake_case,      token_type_ids=snake_case,      encoder_hidden_states=snake_case,      encoder_attention_mask=snake_case,      )
          _lowercase              =model(
              snake_case,      attention_mask=snake_case,      token_type_ids=snake_case,      encoder_hidden_states=snake_case,      )
          _lowercase              =model(snake_case,      attention_mask=snake_case,      token_type_ids=snake_case)
          self.parent.assertEqual(result.last_hidden_state.shape,      (self.batch_size, self.seq_length, self.hidden_size))





       def 				UpperCamelCase__						(				self	:str,      snake_case	:Optional[int],      snake_case	:Optional[int],      snake_case	:int,      snake_case	:Optional[int],      snake_case	:List[str],      snake_case	:Optional[int],      snake_case	:Optional[Any]):



          """simple docstring"""

          _lowercase              =MraForMaskedLM(config=snake_case)
          model.to(snake_case)
          model.eval()
          _lowercase              =model(snake_case,      attention_mask=snake_case,      token_type_ids=snake_case,      labels=snake_case)
          self.parent.assertEqual(result.logits.shape,      (self.batch_size, self.seq_length, self.vocab_size))





       def 				UpperCamelCase__						(				self	:Tuple,      snake_case	:Union[str, Any],      snake_case	:Dict,      snake_case	:Any,      snake_case	:Union[str, Any],      snake_case	:List[Any],      snake_case	:Tuple,      snake_case	:Any):



          """simple docstring"""

          _lowercase              =MraForQuestionAnswering(config=snake_case)
          model.to(snake_case)
          model.eval()
          _lowercase              =model(
              snake_case,      attention_mask=snake_case,      token_type_ids=snake_case,      start_positions=snake_case,      end_positions=snake_case,      )
          self.parent.assertEqual(result.start_logits.shape,      (self.batch_size, self.seq_length))
          self.parent.assertEqual(result.end_logits.shape,      (self.batch_size, self.seq_length))





       def 				UpperCamelCase__						(				self	:int,      snake_case	:List[str],      snake_case	:int,      snake_case	:Optional[Any],      snake_case	:Union[str, Any],      snake_case	:str,      snake_case	:List[Any],      snake_case	:Optional[Any]):



          """simple docstring"""

          _lowercase              =self.num_labels
          _lowercase              =MraForSequenceClassification(snake_case)
          model.to(snake_case)
          model.eval()
          _lowercase              =model(snake_case,      attention_mask=snake_case,      token_type_ids=snake_case,      labels=snake_case)
          self.parent.assertEqual(result.logits.shape,      (self.batch_size, self.num_labels))





       def 				UpperCamelCase__						(				self	:Any,      snake_case	:Optional[int],      snake_case	:List[Any],      snake_case	:Optional[Any],      snake_case	:int,      snake_case	:int,      snake_case	:Dict,      snake_case	:List[Any]):



          """simple docstring"""

          _lowercase              =self.num_labels
          _lowercase              =MraForTokenClassification(config=snake_case)
          model.to(snake_case)
          model.eval()
          _lowercase              =model(snake_case,      attention_mask=snake_case,      token_type_ids=snake_case,      labels=snake_case)
          self.parent.assertEqual(result.logits.shape,      (self.batch_size, self.seq_length, self.num_labels))





       def 				UpperCamelCase__						(				self	:Optional[int],      snake_case	:List[str],      snake_case	:Optional[int],      snake_case	:Dict,      snake_case	:str,      snake_case	:Optional[Any],      snake_case	:Tuple,      snake_case	:Dict):



          """simple docstring"""

          _lowercase              =self.num_choices
          _lowercase              =MraForMultipleChoice(config=snake_case)
          model.to(snake_case)
          model.eval()
          _lowercase              =input_ids.unsqueeze(1).expand(-1,      self.num_choices,      -1).contiguous()
          _lowercase              =token_type_ids.unsqueeze(1).expand(-1,      self.num_choices,      -1).contiguous()
          _lowercase              =input_mask.unsqueeze(1).expand(-1,      self.num_choices,      -1).contiguous()
          _lowercase              =model(
              snake_case,      attention_mask=snake_case,      token_type_ids=snake_case,      labels=snake_case,      )
          self.parent.assertEqual(result.logits.shape,      (self.batch_size, self.num_choices))





       def 				UpperCamelCase__						(				self	:Optional[Any]):



          """simple docstring"""

          _lowercase              =self.prepare_config_and_inputs()
          (
              (
              _lowercase
          )       ,       (
              _lowercase
          )       ,       (
              _lowercase
          )       ,       (
              _lowercase
          )       ,       (
              _lowercase
          )       ,       (
              _lowercase
          )       ,       (
              _lowercase
          )       ,       
          )              =config_and_inputs
          _lowercase              ={'input_ids': input_ids, 'token_type_ids': token_type_ids, 'attention_mask': input_mask}
          return config, inputs_dict






@require_torch
class 					SCREAMING_SNAKE_CASE_  (       _a  ,						unittest.TestCase ):
       """simple docstring"""


       __lowerCAmelCase    :		Optional[int]					=(
           (
               MraModel,
               MraForMaskedLM,
               MraForMultipleChoice,
               MraForQuestionAnswering,
               MraForSequenceClassification,
               MraForTokenClassification,
           )
           if is_torch_available()
           else ()
       )
       __lowerCAmelCase    :		int					=False
       __lowerCAmelCase    :		Union[str, Any]					=False
       __lowerCAmelCase    :		str					=False
       __lowerCAmelCase    :		str					=False

       __lowerCAmelCase    :		Dict					=()





       def 				UpperCamelCase__						(				self	:Any):



          """simple docstring"""

          _lowercase              =MraModelTester(self)
          _lowercase              =ConfigTester(self,      config_class=snake_case,      hidden_size=37)





       def 				UpperCamelCase__						(				self	:str):



          """simple docstring"""

          self.config_tester.run_common_tests()





       def 				UpperCamelCase__						(				self	:Optional[int]):



          """simple docstring"""

          _lowercase              =self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_model(*snake_case)





       def 				UpperCamelCase__						(				self	:str):



          """simple docstring"""

          _lowercase              =self.model_tester.prepare_config_and_inputs()
          for type in ["absolute", "relative_key", "relative_key_query"]:
             _lowercase              =type
             self.model_tester.create_and_check_model(*snake_case)





       def 				UpperCamelCase__						(				self	:Dict):



          """simple docstring"""

          _lowercase              =self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_for_masked_lm(*snake_case)





       def 				UpperCamelCase__						(				self	:Optional[Any]):



          """simple docstring"""

          _lowercase              =self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_for_multiple_choice(*snake_case)





       def 				UpperCamelCase__						(				self	:int):



          """simple docstring"""

          _lowercase              =self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_for_question_answering(*snake_case)





       def 				UpperCamelCase__						(				self	:Union[str, Any]):



          """simple docstring"""

          _lowercase              =self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_for_sequence_classification(*snake_case)





       def 				UpperCamelCase__						(				self	:Optional[int]):



          """simple docstring"""

          _lowercase              =self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_for_token_classification(*snake_case)





       @slow
       def 				UpperCamelCase__						(				self	:int):



          """simple docstring"""

          for model_name in MRA_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
             _lowercase              =MraModel.from_pretrained(snake_case)
             self.assertIsNotNone(snake_case)





       @unittest.skip(reason='MRA does not output attentions')
       def 				UpperCamelCase__						(				self	:Optional[Any]):



          """simple docstring"""

          return






@require_torch
class 					SCREAMING_SNAKE_CASE_  (       unittest.TestCase ):
       """simple docstring"""


       @slow
       def 				UpperCamelCase__						(				self	:Dict):



          """simple docstring"""

          _lowercase              =MraModel.from_pretrained('uw-madison/mra-base-512-4')
          _lowercase              =torch.arange(256).unsqueeze(0)

          with torch.no_grad():
             _lowercase              =model(snake_case)[0]

          _lowercase              =torch.Size((1, 256, 768))
          self.assertEqual(output.shape,      snake_case)

          _lowercase              =torch.tensor(
              [[[-0.0_1_4_0, 0.0_8_3_0, -0.0_3_8_1], [0.1_5_4_6, 0.1_4_0_2, 0.0_2_2_0], [0.1_1_6_2, 0.0_8_5_1, 0.0_1_6_5]]])

          self.assertTrue(torch.allclose(output[:, :3, :3],      snake_case,      atol=1e-4))





       @slow
       def 				UpperCamelCase__						(				self	:List[Any]):



          """simple docstring"""

          _lowercase              =MraForMaskedLM.from_pretrained('uw-madison/mra-base-512-4')
          _lowercase              =torch.arange(256).unsqueeze(0)

          with torch.no_grad():
             _lowercase              =model(snake_case)[0]

          _lowercase              =5_0265

          _lowercase              =torch.Size((1, 256, vocab_size))
          self.assertEqual(output.shape,      snake_case)

          _lowercase              =torch.tensor(
              [[[9.2_5_9_5, -3.6_0_3_8, 1_1.8_8_1_9], [9.3_8_6_9, -3.2_6_9_3, 1_1.0_9_5_6], [1_1.8_5_2_4, -3.4_9_3_8, 1_3.1_2_1_0]]])

          self.assertTrue(torch.allclose(output[:, :3, :3],      snake_case,      atol=1e-4))





       @slow
       def 				UpperCamelCase__						(				self	:Dict):



          """simple docstring"""

          _lowercase              =MraForMaskedLM.from_pretrained('uw-madison/mra-base-4096-8-d3')
          _lowercase              =torch.arange(4096).unsqueeze(0)

          with torch.no_grad():
             _lowercase              =model(snake_case)[0]

          _lowercase              =5_0265

          _lowercase              =torch.Size((1, 4096, vocab_size))
          self.assertEqual(output.shape,      snake_case)

          _lowercase              =torch.tensor(
              [[[5.4_7_8_9, -2.3_5_6_4, 7.5_0_6_4], [7.9_0_6_7, -1.3_3_6_9, 9.9_6_6_8], [9.0_7_1_2, -1.8_1_0_6, 7.0_3_8_0]]])

          self.assertTrue(torch.allclose(output[:, :3, :3],      snake_case,      atol=1e-4))


