
"""simple docstring"""


import unittest

from transformers import AutoTokenizer, FalconConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
		import torch

		from transformers import (
		    FalconForCausalLM,
		    FalconForQuestionAnswering,
		    FalconForSequenceClassification,
		    FalconForTokenClassification,
		    FalconModel,
		)







class 					lowerCAmelCase_					:



						'''simple docstring'''


						def __init__(			self  :   int	,A_  :   int	,A_  :   int=3	,A_  :   Union[str, Any]=7	,A_  :   int=True	,A_  :   Tuple=True	,A_  :   Dict=False	,A_  :   List[str]=True	,A_  :   int=99	,A_  :   List[str]=32	,A_  :   int=5	,A_  :   Tuple=4	,A_  :   Dict=37	,A_  :   List[str]="gelu"	,A_  :   str=0.1	,A_  :   Dict=0.1	,A_  :   List[Any]=512	,A_  :   Tuple=16	,A_  :   Union[str, Any]=2	,A_  :   List[str]=0.02	,A_  :   str=3	,A_  :   Optional[int]=4	,A_  :   Tuple=None	,)  -> Union[str, Any]:
							A     = parent
							A     = batch_size
							A     = seq_length
							A     = is_training
							A     = use_input_mask
							A     = use_token_type_ids
							A     = use_labels
							A     = vocab_size
							A     = hidden_size
							A     = num_hidden_layers
							A     = num_attention_heads
							A     = intermediate_size
							A     = hidden_act
							A     = hidden_dropout_prob
							A     = attention_probs_dropout_prob
							A     = max_position_embeddings
							A     = type_vocab_size
							A     = type_sequence_label_size
							A     = initializer_range
							A     = num_labels
							A     = num_choices
							A     = scope


						def     _SCREAMING_SNAKE_CASE   (			self  :   List[str]   )  -> Optional[int]:
							A     = ids_tensor([self.batch_size, self.seq_length]	,self.vocab_size   )

							A     = None
							if self.use_input_mask:
								A     = random_attention_mask([self.batch_size, self.seq_length]   )

							A     = None

							A     = None
							A     = None
							A     = None
							if self.use_labels:
								A     = ids_tensor([self.batch_size]	,self.type_sequence_label_size   )
								A     = ids_tensor([self.batch_size, self.seq_length]	,self.num_labels   )
								A     = ids_tensor([self.batch_size]	,self.num_choices   )

							A     = self.get_config()

							return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels


						def     _SCREAMING_SNAKE_CASE   (			self  :   Optional[Any]   )  -> int:
							return FalconConfig(
							    vocab_size=self.vocab_size	,hidden_size=self.hidden_size	,num_hidden_layers=self.num_hidden_layers	,num_attention_heads=self.num_attention_heads	,intermediate_size=self.intermediate_size	,hidden_act=self.hidden_act	,hidden_dropout_prob=self.hidden_dropout_prob	,attention_probs_dropout_prob=self.attention_probs_dropout_prob	,max_position_embeddings=self.max_position_embeddings	,type_vocab_size=self.type_vocab_size	,is_decoder=A_	,initializer_range=self.initializer_range	,pad_token_id=1	,new_decoder_architecture=A_	,)


						def     _SCREAMING_SNAKE_CASE   (			self  :   List[str]	,A_  :   str	,A_  :   Union[str, Any]	,A_  :   int	,A_  :   Any	,A_  :   List[str]	,A_  :   List[Any]	,A_  :   Any   )  -> List[Any]:
							A     = FalconModel(config=A_   )
							model.to(A_   )
							model.eval()
							A     = model(A_	,attention_mask=A_   )
							A     = model(A_   )
							self.parent.assertEqual(result.last_hidden_state.shape	,(self.batch_size, self.seq_length, self.hidden_size)   )


						def     _SCREAMING_SNAKE_CASE   (			self  :   Tuple	,A_  :   Union[str, Any]	,A_  :   Union[str, Any]	,A_  :   Optional[int]	,A_  :   List[Any]	,A_  :   List[Any]	,A_  :   Dict	,A_  :   Union[str, Any]	,A_  :   List[str]	,A_  :   List[str]	,)  -> Optional[Any]:
							A     = True
							A     = FalconModel(A_   )
							model.to(A_   )
							model.eval()
							A     = model(
							    A_	,attention_mask=A_	,encoder_hidden_states=A_	,encoder_attention_mask=A_	,)
							A     = model(
							    A_	,attention_mask=A_	,encoder_hidden_states=A_	,)
							A     = model(A_	,attention_mask=A_   )
							self.parent.assertEqual(result.last_hidden_state.shape	,(self.batch_size, self.seq_length, self.hidden_size)   )


						def     _SCREAMING_SNAKE_CASE   (			self  :   Tuple	,A_  :   Dict	,A_  :   int	,A_  :   Tuple	,A_  :   Union[str, Any]	,A_  :   int	,A_  :   List[str]	,A_  :   List[Any]	,A_  :   Optional[Any]	,A_  :   List[Any]	,)  -> str:
							A     = FalconForCausalLM(config=A_   )
							model.to(A_   )
							model.eval()
							A     = model(A_	,attention_mask=A_	,labels=A_   )
							self.parent.assertEqual(result.logits.shape	,(self.batch_size, self.seq_length, self.vocab_size)   )


						def     _SCREAMING_SNAKE_CASE   (			self  :   str	,A_  :   Union[str, Any]	,A_  :   str	,A_  :   Dict	,A_  :   Dict	,A_  :   Optional[Any]	,A_  :   Tuple	,A_  :   List[Any]	,A_  :   Union[str, Any]	,A_  :   Tuple	,)  -> List[str]:
							A     = True
							A     = True
							A     = FalconForCausalLM(config=A_   )
							model.to(A_   )
							model.eval()

							# first forward pass
							A     = model(
							    A_	,attention_mask=A_	,encoder_hidden_states=A_	,encoder_attention_mask=A_	,use_cache=A_	,)
							A     = outputs.past_key_values

							# create hypothetical multiple next token and extent to next_input_ids
							A     = ids_tensor((self.batch_size, 3)	,config.vocab_size   )
							A     = ids_tensor((self.batch_size, 3)	,vocab_size=2   )

							# append to next input_ids and
							A     = torch.cat([input_ids, next_tokens]	,dim=-1   )
							A     = torch.cat([input_mask, next_mask]	,dim=-1   )

							A     = model(
							    A_	,attention_mask=A_	,encoder_hidden_states=A_	,encoder_attention_mask=A_	,output_hidden_states=A_	,)['hidden_states'][0]
							A     = model(
							    A_	,attention_mask=A_	,encoder_hidden_states=A_	,encoder_attention_mask=A_	,past_key_values=A_	,output_hidden_states=A_	,)['hidden_states'][0]

							# select random slice
							A     = ids_tensor((1,)	,output_from_past.shape[-1]   ).item()
							A     = output_from_no_past[:, -3:, random_slice_idx].detach()
							A     = output_from_past[:, :, random_slice_idx].detach()

							self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1]   )

							# test that outputs are equal for slice
							self.parent.assertTrue(torch.allclose(A_	,A_	,atol=1e-3   )   )




						def     _SCREAMING_SNAKE_CASE   (			self  :   Optional[Any]   )  -> List[Any]:
							A     = self.prepare_config_and_inputs()
							(
							    (
							    A
							) ,   (
							    A
							) ,   (
							    A
							) ,   (
							    A
							) ,   (
							    A
							) ,   (
							    A
							) ,   (
							    A
							) ,   
							)     = config_and_inputs
							A     = {'input_ids': input_ids, 'attention_mask': input_mask}
							return config, inputs_dict







@require_torch
class 					lowerCAmelCase_					(   _lowercase		,       _lowercase		,       _lowercase		,       unittest.TestCase     ):



						'''simple docstring'''


						_lowerCamelCase:     List[str]   =     (
						    (
						        FalconModel,
						        FalconForCausalLM,
						        FalconForSequenceClassification,
						        FalconForTokenClassification,
						        FalconForQuestionAnswering,
						    )
						    if is_torch_available()
						    else ()
						)
						_lowerCamelCase:     int   =     (FalconForCausalLM,) if is_torch_available() else ()
						_lowerCamelCase:     List[Any]   =     (
						    {
						        '''feature-extraction''': FalconModel,
						        '''text-classification''': FalconForSequenceClassification,
						        '''text-generation''': FalconForCausalLM,
						        '''question-answering''': FalconForQuestionAnswering,
						        '''token-classification''': FalconForTokenClassification,
						        '''zero-shot''': FalconForSequenceClassification,
						    }
						    if is_torch_available()
						    else {}
						)
						_lowerCamelCase:     int   =     False
						_lowerCamelCase:     str   =     False


						def     _SCREAMING_SNAKE_CASE   (			self  :   Optional[Any]   )  -> Dict:
							A     = FalconModelTester(self   )
							A     = ConfigTester(self	,config_class=A_	,hidden_size=37   )


						def     _SCREAMING_SNAKE_CASE   (			self  :   Any   )  -> Tuple:
							self.config_tester.run_common_tests()


						def     _SCREAMING_SNAKE_CASE   (			self  :   Optional[int]   )  -> Optional[Any]:
							A     = self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_model(*A_   )


						def     _SCREAMING_SNAKE_CASE   (			self  :   List[str]   )  -> str:
							A ,   *A     = self.model_tester.prepare_config_and_inputs()
							for alibi in [True, False]:
								A     = alibi
								self.model_tester.create_and_check_model(A_	,*A_   )


						def     _SCREAMING_SNAKE_CASE   (			self  :   str   )  -> Optional[Any]:
							A ,   A     = self.model_tester.prepare_config_and_inputs_for_common()
							A     = 3
							A     = input_dict['input_ids']
							A     = input_ids.ne(1   ).to(A_   )
							A     = ids_tensor([self.model_tester.batch_size]	,self.model_tester.type_sequence_label_size   )
							A     = FalconForSequenceClassification(A_   )
							model.to(A_   )
							model.eval()
							A     = model(A_	,attention_mask=A_	,labels=A_   )
							self.assertEqual(result.logits.shape	,(self.model_tester.batch_size, self.model_tester.num_labels)   )


						def     _SCREAMING_SNAKE_CASE   (			self  :   Tuple   )  -> Optional[int]:
							A ,   A     = self.model_tester.prepare_config_and_inputs_for_common()
							A     = 3
							A     = 'single_label_classification'
							A     = input_dict['input_ids']
							A     = input_ids.ne(1   ).to(A_   )
							A     = ids_tensor([self.model_tester.batch_size]	,self.model_tester.type_sequence_label_size   )
							A     = FalconForSequenceClassification(A_   )
							model.to(A_   )
							model.eval()
							A     = model(A_	,attention_mask=A_	,labels=A_   )
							self.assertEqual(result.logits.shape	,(self.model_tester.batch_size, self.model_tester.num_labels)   )


						def     _SCREAMING_SNAKE_CASE   (			self  :   List[str]   )  -> Optional[int]:
							A ,   A     = self.model_tester.prepare_config_and_inputs_for_common()
							A     = input_dict['input_ids']
							A     = FalconForCausalLM(A_   )
							model.to(A_   )
							model.eval()
							A     = model(A_	,use_cache=A_   )
							A     = input_ids.shape[0]
							A     = model._convert_to_rw_cache(result.past_key_values   )
							A     = model._convert_cache_to_standard_format(A_	,A_   )
							for layer in range(len(A_   )   ):
								for tensor_idx in range(2   ):
									self.assertTrue(rw_cache[layer][tensor_idx].ndim == 3   )
									self.assertTrue(result.past_key_values[layer][tensor_idx].ndim == 4   )
									self.assertTrue(
									    torch.all(result.past_key_values[layer][tensor_idx] == standard_cache[layer][tensor_idx]   )   )


						def     _SCREAMING_SNAKE_CASE   (			self  :   List[str]   )  -> Optional[Any]:
							A ,   A     = self.model_tester.prepare_config_and_inputs_for_common()
							A     = 3
							A     = 'multi_label_classification'
							A     = input_dict['input_ids']
							A     = input_ids.ne(1   ).to(A_   )
							A     = ids_tensor(
							    [self.model_tester.batch_size, config.num_labels]	,self.model_tester.type_sequence_label_size   ).to(torch.float   )
							A     = FalconForSequenceClassification(A_   )
							model.to(A_   )
							model.eval()
							A     = model(A_	,attention_mask=A_	,labels=A_   )
							self.assertEqual(result.logits.shape	,(self.model_tester.batch_size, self.model_tester.num_labels)   )




						def     _SCREAMING_SNAKE_CASE   (			self  :   Any   )  -> Tuple:
							# Falcon can have different numbers of KV-heads than the number of query heads, so we need
							# to override this test to use the right head counts.
							for model_class in self.all_generative_model_classes:
								A ,   A     = self.model_tester.prepare_config_and_inputs_for_common()

								# If it doesn't support cache, pass the test
								if not hasattr(A_	,'use_cache'   ):
									return

								A     = model_class(A_   ).to(A_   )
								if "use_cache" not in inputs:
									A     = True
								A     = model(**A_   )

								# If "past_key_values" is not returned, pass the test (e.g. RWKV uses a different cache name and format)
								if "past_key_values" not in outputs:
									return

								A     = (
								    getattr(A_	,'decoder_layers'	,A_   )
								    or getattr(A_	,'num_decoder_layers'	,A_   )
								    or config.num_hidden_layers
								)
								A     = getattr(A_	,'num_kv_heads'	,config.num_attention_heads   )
								A     = getattr(A_	,'d_model'	,config.hidden_size   )
								A     = embed_dim // num_attention_heads

								A     = outputs['past_key_values']
								self.assertEqual(len(A_   )	,A_   )

								A ,   A     = inputs['input_ids'].shape
								for i in range(A_   ):
									if config.new_decoder_architecture:
										A     = config.num_attention_heads
									elif config.multi_query:
										A     = 1
									self.assertEqual(len(past_kv[0]   )	,2   )  # K V for the decoder = 2
									self.assertEqual(
									    past_kv[i][0].shape	,(batch_size, num_attention_heads, seq_length, per_head_embed_dim)   )
									self.assertEqual(
									    past_kv[i][1].shape	,(batch_size, num_attention_heads, seq_length, per_head_embed_dim)   )







@require_torch
class 					lowerCAmelCase_					(   unittest.TestCase     ):



						'''simple docstring'''


						@slow
						def     _SCREAMING_SNAKE_CASE   (			self  :   int   )  -> Optional[int]:
							A     = AutoTokenizer.from_pretrained('Rocketknight1/falcon-rw-1b'   )
							A     = FalconForCausalLM.from_pretrained('Rocketknight1/falcon-rw-1b'   )
							model.eval()
							model.to(A_   )
							A     = tokenizer('My favorite food is'	,return_tensors='pt'   ).to(A_   )

							A     = (
							    'My favorite food is pizza. I love it so much that I have a pizza party every year for my birthday.'
							)

							A     = model.generate(**A_	,do_sample=A_	,max_new_tokens=19   )
							A     = tokenizer.batch_decode(A_   )[0]

							self.assertEqual(A_	,A_   )


						@slow
						def     _SCREAMING_SNAKE_CASE   (			self  :   Union[str, Any]   )  -> Tuple:
							# The big models are way too big for the CI, so we use tiny random models that resemble their
							# architectures but with much smaller and fewer layers
							for repo in ["Rocketknight1/tiny-random-falcon-7b", "Rocketknight1/tiny-random-falcon-40b"]:
								A     = AutoTokenizer.from_pretrained(A_   )
								A     = FalconForCausalLM.from_pretrained(A_   )
								model.eval()
								model.to(A_   )
								A     = tokenizer('My favorite food is'	,return_tensors='pt'   ).to(A_   )

								# We just test that these run without errors - the models are randomly initialized
								# and so the actual text outputs will be garbage
								model.generate(**A_	,do_sample=A_	,max_new_tokens=4   )
								model.generate(**A_	,do_sample=A_	,max_new_tokens=4   )
								model.generate(**A_	,num_beams=2	,max_new_tokens=4   )




						@slow
						def     _SCREAMING_SNAKE_CASE   (			self  :   Union[str, Any]   )  -> str:
							# The big models are way too big for the CI, so we use tiny random models that resemble their
							# architectures but with much smaller and fewer layers
							with torch.no_grad():
								for repo in [
								    "Rocketknight1/falcon-rw-1b",
								    "Rocketknight1/tiny-random-falcon-7b",
								    "Rocketknight1/tiny-random-falcon-40b",
								]:
									A     = AutoTokenizer.from_pretrained(A_   )
									A     = FalconForCausalLM.from_pretrained(A_   )
									model.eval()
									model.to(device=A_   )
									A     = tokenizer('My favorite food is'	,return_tensors='pt'   ).to(A_   )

									# Test results are the same with and without cache
									A     = model.generate(**A_	,do_sample=A_	,max_new_tokens=20	,use_cache=A_   )
									A     = model.generate(**A_	,do_sample=A_	,max_new_tokens=20	,use_cache=A_   )
									self.assertTrue((outputs_cache - outputs_no_cache).sum().item() == 0   )






'''simple docstring'''

from dataclasses import asdict, dataclass
from typing import Optional

from ...configuration_utils import PretrainedConfig
from ...utils import logging


a   	=    logging.get_logger(__name__)

# TODO Update this
a   	=    {
    "facebook/esm-1b": "https://huggingface.co/facebook/esm-1b/resolve/main/config.json",
    # See all ESM models at https://huggingface.co/models?filter=esm
}







class __a			( _snake_case      ):
   __UpperCamelCase :			str						   =						'esm'





   def __init__(       self					:      Tuple					,lowerCamelCase					:      List[Any]=None					,lowerCamelCase					:      str=None					,lowerCamelCase					:      Any=None					,lowerCamelCase					:      Union[str, Any]=768					,lowerCamelCase					:      Tuple=12					,lowerCamelCase					:      int=12					,lowerCamelCase					:      Optional[int]=3072					,lowerCamelCase					:      List[Any]=0.1					,lowerCamelCase					:      Optional[int]=0.1					,lowerCamelCase					:      Any=1026					,lowerCamelCase					:      str=0.02					,lowerCamelCase					:      int=1E-1_2					,lowerCamelCase					:      Union[str, Any]="absolute"					,lowerCamelCase					:      Optional[Any]=True					,lowerCamelCase					:      str=None					,lowerCamelCase					:      Optional[int]=False					,lowerCamelCase					:      int=False					,lowerCamelCase					:      Union[str, Any]=None					,lowerCamelCase					:      Any=None					,**lowerCamelCase					:      Any					,):

      '''simple docstring'''

      super().__init__(pad_token_id=lowerCamelCase					,mask_token_id=lowerCamelCase					,**lowerCamelCase     )

      __SCREAMING_SNAKE_CASE   				=							vocab_size
      __SCREAMING_SNAKE_CASE   				=							hidden_size
      __SCREAMING_SNAKE_CASE   				=							num_hidden_layers
      __SCREAMING_SNAKE_CASE   				=							num_attention_heads
      __SCREAMING_SNAKE_CASE   				=							intermediate_size
      __SCREAMING_SNAKE_CASE   				=							hidden_dropout_prob
      __SCREAMING_SNAKE_CASE   				=							attention_probs_dropout_prob
      __SCREAMING_SNAKE_CASE   				=							max_position_embeddings
      __SCREAMING_SNAKE_CASE   				=							initializer_range
      __SCREAMING_SNAKE_CASE   				=							layer_norm_eps
      __SCREAMING_SNAKE_CASE   				=							position_embedding_type
      __SCREAMING_SNAKE_CASE   				=							use_cache
      __SCREAMING_SNAKE_CASE   				=							emb_layer_norm_before
      __SCREAMING_SNAKE_CASE   				=							token_dropout
      __SCREAMING_SNAKE_CASE   				=							is_folding_model
      if is_folding_model:
         if esmfold_config is None:
            logger.info("""No esmfold_config supplied for folding model, using default values."""     )
            __SCREAMING_SNAKE_CASE   				=							EsmFoldConfig()
         elif isinstance(lowerCamelCase					,lowerCamelCase     ):
            __SCREAMING_SNAKE_CASE   				=							EsmFoldConfig(**lowerCamelCase     )
         __SCREAMING_SNAKE_CASE   				=							esmfold_config
         if vocab_list is None:
            logger.warning("""No vocab_list supplied for folding model, assuming the ESM-2 vocabulary!"""     )
            __SCREAMING_SNAKE_CASE   				=							get_default_vocab_list()
         else:
            __SCREAMING_SNAKE_CASE   				=							vocab_list
      else:
         __SCREAMING_SNAKE_CASE   				=							None
         __SCREAMING_SNAKE_CASE   				=							None
      if self.esmfold_config is not None and getattr(self.esmfold_config					,"""use_esm_attn_map"""					,lowerCamelCase     ):
         raise ValueError("""The HuggingFace port of ESMFold does not support use_esm_attn_map at this time!"""     )





   def   UpperCAmelCase__  (       self					:      List[str]     ):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE   				=							super().to_dict()
      if isinstance(self.esmfold_config					,lowerCamelCase     ):
         __SCREAMING_SNAKE_CASE   				=							self.esmfold_config.to_dict()
      return output







@dataclass
class __a			:
   __UpperCamelCase :			str						   =						None
   __UpperCamelCase :			bool						   =						True
   __UpperCamelCase :			bool						   =						False
   __UpperCamelCase :			bool						   =						False
   __UpperCamelCase :			bool						   =						False
   __UpperCamelCase :			float						   =						0

   __UpperCamelCase :			bool						   =						True
   __UpperCamelCase :			bool						   =						False

   __UpperCamelCase :			int						   =						128
   __UpperCamelCase :			"TrunkConfig"						   =						None





   def   UpperCAmelCase__  (       self					:      List[str]     ):

      '''simple docstring'''

      if self.trunk is None:
         __SCREAMING_SNAKE_CASE   				=							TrunkConfig()
      elif isinstance(self.trunk					,lowerCamelCase     ):
         __SCREAMING_SNAKE_CASE   				=							TrunkConfig(**self.trunk     )





   def   UpperCAmelCase__  (       self					:      Optional[int]     ):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE   				=							asdict(self     )
      __SCREAMING_SNAKE_CASE   				=							self.trunk.to_dict()
      return output







@dataclass
class __a			:
   __UpperCamelCase :			int						   =						48
   __UpperCamelCase :			int						   =						1024
   __UpperCamelCase :			int						   =						128
   __UpperCamelCase :			int						   =						32
   __UpperCamelCase :			int						   =						32
   __UpperCamelCase :			int						   =						32
   __UpperCamelCase :			float						   =						0
   __UpperCamelCase :			float						   =						0
   __UpperCamelCase :			bool						   =						False
   __UpperCamelCase :			int						   =						4
   __UpperCamelCase :			Optional[int]						   =						128
   __UpperCamelCase :			"StructureModuleConfig"						   =						None





   def   UpperCAmelCase__  (       self					:      List[Any]     ):

      '''simple docstring'''

      if self.structure_module is None:
         __SCREAMING_SNAKE_CASE   				=							StructureModuleConfig()
      elif isinstance(self.structure_module					,lowerCamelCase     ):
         __SCREAMING_SNAKE_CASE   				=							StructureModuleConfig(**self.structure_module     )

      if self.max_recycles <= 0:
         raise ValueError(f"""`max_recycles` should be positive, got {self.max_recycles}."""     )
      if self.sequence_state_dim % self.sequence_state_dim != 0:
         raise ValueError(
             """`sequence_state_dim` should be a round multiple of `sequence_state_dim`, got"""
             f""" {self.sequence_state_dim} and {self.sequence_state_dim}."""     )
      if self.pairwise_state_dim % self.pairwise_state_dim != 0:
         raise ValueError(
             """`pairwise_state_dim` should be a round multiple of `pairwise_state_dim`, got"""
             f""" {self.pairwise_state_dim} and {self.pairwise_state_dim}."""     )

      __SCREAMING_SNAKE_CASE   				=							self.sequence_state_dim // self.sequence_head_width
      __SCREAMING_SNAKE_CASE   				=							self.pairwise_state_dim // self.pairwise_head_width

      if self.sequence_state_dim != sequence_num_heads * self.sequence_head_width:
         raise ValueError(
             """`sequence_state_dim` should be equal to `sequence_num_heads * sequence_head_width, got"""
             f""" {self.sequence_state_dim} != {sequence_num_heads} * {self.sequence_head_width}."""     )
      if self.pairwise_state_dim != pairwise_num_heads * self.pairwise_head_width:
         raise ValueError(
             """`pairwise_state_dim` should be equal to `pairwise_num_heads * pairwise_head_width, got"""
             f""" {self.pairwise_state_dim} != {pairwise_num_heads} * {self.pairwise_head_width}."""     )
      if self.pairwise_state_dim % 2 != 0:
         raise ValueError(f"""`pairwise_state_dim` should be even, got {self.pairwise_state_dim}."""     )

      if self.dropout >= 0.4:
         raise ValueError(f"""`dropout` should not be greater than 0.4, got {self.dropout}."""     )





   def   UpperCAmelCase__  (       self					:      str     ):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE   				=							asdict(self     )
      __SCREAMING_SNAKE_CASE   				=							self.structure_module.to_dict()
      return output







@dataclass
class __a			:
   __UpperCamelCase :			int						   =						384
   __UpperCamelCase :			int						   =						128
   __UpperCamelCase :			int						   =						16
   __UpperCamelCase :			int						   =						128
   __UpperCamelCase :			int						   =						12
   __UpperCamelCase :			int						   =						4
   __UpperCamelCase :			int						   =						8
   __UpperCamelCase :			float						   =						0.1
   __UpperCamelCase :			int						   =						8
   __UpperCamelCase :			int						   =						1
   __UpperCamelCase :			int						   =						2
   __UpperCamelCase :			int						   =						7
   __UpperCamelCase :			int						   =						10
   __UpperCamelCase :			float						   =						1E-8
   __UpperCamelCase :			float						   =						1E5





   def   UpperCAmelCase__  (       self					:      List[Any]     ):

      '''simple docstring'''

      return asdict(self     )





def     __magic_name__							(				)    ->       Dict:

   '''simple docstring'''







   return (
       "<cls>",
       "<pad>",
       "<eos>",
       "<unk>",
       "L",
       "A",
       "G",
       "V",
       "S",
       "E",
       "R",
       "T",
       "I",
       "D",
       "P",
       "K",
       "Q",
       "N",
       "F",
       "Y",
       "M",
       "H",
       "W",
       "C",
       "X",
       "B",
       "U",
       "Z",
       "O",
       ".",
       "-",
       "<null_1>",
       "<mask>",
   )





