


"""simple docstring"""






import unittest

from transformers import EsmConfig, is_torch_available
from transformers.testing_utils import TestCasePlus, require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
  import torch

  from transformers import EsmForMaskedLM, EsmForSequenceClassification, EsmForTokenClassification, EsmModel
  from transformers.models.esm.modeling_esm import (
      ESM_PRETRAINED_MODEL_ARCHIVE_LIST,
      EsmEmbeddings,
      create_position_ids_from_input_ids,
  )




class 							A_      :


  def __init__(			self:						List[str]							,__lowerCAmelCase:						Any							,__lowerCAmelCase:						Tuple=13							,__lowerCAmelCase:						List[Any]=7							,__lowerCAmelCase:						List[str]=False							,__lowerCAmelCase:						List[str]=True							,__lowerCAmelCase:						Dict=False							,__lowerCAmelCase:						Dict=True							,__lowerCAmelCase:						Tuple=33							,__lowerCAmelCase:						Optional[Any]=32							,__lowerCAmelCase:						int=5							,__lowerCAmelCase:						Tuple=4							,__lowerCAmelCase:						Union[str, Any]=37							,__lowerCAmelCase:						List[Any]="gelu"							,__lowerCAmelCase:						Union[str, Any]=0.1							,__lowerCAmelCase:						str=0.1							,__lowerCAmelCase:						Optional[Any]=512							,__lowerCAmelCase:						str=16							,__lowerCAmelCase:						Union[str, Any]=2							,__lowerCAmelCase:						Optional[Any]=0.02							,__lowerCAmelCase:						List[Any]=3							,__lowerCAmelCase:						List[Any]=4							,__lowerCAmelCase:						Union[str, Any]=None							,):







     '''simple docstring'''







     _lowerCamelCase   :				Dict							=     parent
     _lowerCamelCase   :				Optional[Any]							=     batch_size
     _lowerCamelCase   :				str							=     seq_length
     _lowerCamelCase   :				Union[str, Any]							=     is_training
     _lowerCamelCase   :				Optional[Any]							=     use_input_mask
     _lowerCamelCase   :				Dict							=     use_token_type_ids
     _lowerCamelCase   :				Any							=     use_labels
     _lowerCamelCase   :				Any							=     vocab_size
     _lowerCamelCase   :				Optional[Any]							=     hidden_size
     _lowerCamelCase   :				int							=     num_hidden_layers
     _lowerCamelCase   :				int							=     num_attention_heads
     _lowerCamelCase   :				List[str]							=     intermediate_size
     _lowerCamelCase   :				Union[str, Any]							=     hidden_act
     _lowerCamelCase   :				Optional[Any]							=     hidden_dropout_prob
     _lowerCamelCase   :				Any							=     attention_probs_dropout_prob
     _lowerCamelCase   :				Optional[int]							=     max_position_embeddings
     _lowerCamelCase   :				Tuple							=     type_vocab_size
     _lowerCamelCase   :				Optional[Any]							=     type_sequence_label_size
     _lowerCamelCase   :				str							=     initializer_range
     _lowerCamelCase   :				Optional[Any]							=     num_labels
     _lowerCamelCase   :				Optional[int]							=     num_choices
     _lowerCamelCase   :				Optional[int]							=     scope


  def 							_lowercase					(			self:						List[str]    ):







     '''simple docstring'''







     _lowerCamelCase   :				Optional[Any]							=     ids_tensor([self.batch_size, self.seq_length]							,self.vocab_size    )

     _lowerCamelCase   :				Dict							=     None
     if self.use_input_mask:
        _lowerCamelCase   :				Optional[int]							=     random_attention_mask([self.batch_size, self.seq_length]    )

     _lowerCamelCase   :				Optional[int]							=     None
     _lowerCamelCase   :				int							=     None
     _lowerCamelCase   :				Dict							=     None
     if self.use_labels:
        _lowerCamelCase   :				Union[str, Any]							=     ids_tensor([self.batch_size]							,self.type_sequence_label_size    )
        _lowerCamelCase   :				Optional[int]							=     ids_tensor([self.batch_size, self.seq_length]							,self.num_labels    )
        _lowerCamelCase   :				str							=     ids_tensor([self.batch_size]							,self.num_choices    )

     _lowerCamelCase   :				str							=     self.get_config()

     return config, input_ids, input_mask, sequence_labels, token_labels, choice_labels


  def 							_lowercase					(			self:						List[str]    ):







     '''simple docstring'''







     return EsmConfig(
         vocab_size=self.vocab_size							,hidden_size=self.hidden_size							,pad_token_id=1							,num_hidden_layers=self.num_hidden_layers							,num_attention_heads=self.num_attention_heads							,intermediate_size=self.intermediate_size							,hidden_act=self.hidden_act							,hidden_dropout_prob=self.hidden_dropout_prob							,attention_probs_dropout_prob=self.attention_probs_dropout_prob							,max_position_embeddings=self.max_position_embeddings							,type_vocab_size=self.type_vocab_size							,initializer_range=self.initializer_range							,)


  def 							_lowercase					(			self:						Tuple							,__lowerCAmelCase:						str							,__lowerCAmelCase:						List[Any]							,__lowerCAmelCase:						Tuple							,__lowerCAmelCase:						List[Any]							,__lowerCAmelCase:						Optional[Any]							,__lowerCAmelCase:						Dict    ):







     '''simple docstring'''







     _lowerCamelCase   :				int							=     EsmModel(config=__UpperCAmelCase    )
     model.to(__UpperCAmelCase    )
     model.eval()
     _lowerCamelCase   :				int							=     model(__UpperCAmelCase							,attention_mask=__UpperCAmelCase    )
     _lowerCamelCase   :				str							=     model(__UpperCAmelCase    )
     _lowerCamelCase   :				Dict							=     model(__UpperCAmelCase    )

     self.parent.assertEqual(result.last_hidden_state.shape							,(self.batch_size, self.seq_length, self.hidden_size)    )
     self.parent.assertEqual(result.pooler_output.shape							,(self.batch_size, self.hidden_size)    )


  def 							_lowercase					(			self:						Optional[Any]							,__lowerCAmelCase:						Any							,__lowerCAmelCase:						int							,__lowerCAmelCase:						Union[str, Any]							,__lowerCAmelCase:						str							,__lowerCAmelCase:						List[Any]							,__lowerCAmelCase:						Optional[Any]    ):







     '''simple docstring'''







     _lowerCamelCase   :				Dict							=     EsmForMaskedLM(config=__UpperCAmelCase    )
     model.to(__UpperCAmelCase    )
     model.eval()
     _lowerCamelCase   :				List[Any]							=     model(__UpperCAmelCase							,attention_mask=__UpperCAmelCase							,labels=__UpperCAmelCase    )
     self.parent.assertEqual(result.logits.shape							,(self.batch_size, self.seq_length, self.vocab_size)    )


  def 							_lowercase					(			self:						Dict							,__lowerCAmelCase:						str							,__lowerCAmelCase:						int							,__lowerCAmelCase:						Optional[int]							,__lowerCAmelCase:						Optional[Any]							,__lowerCAmelCase:						str							,__lowerCAmelCase:						Optional[int]    ):







     '''simple docstring'''







     _lowerCamelCase   :				List[Any]							=     self.num_labels
     _lowerCamelCase   :				Union[str, Any]							=     EsmForTokenClassification(config=__UpperCAmelCase    )
     model.to(__UpperCAmelCase    )
     model.eval()
     _lowerCamelCase   :				Optional[Any]							=     model(__UpperCAmelCase							,attention_mask=__UpperCAmelCase							,labels=__UpperCAmelCase    )
     self.parent.assertEqual(result.logits.shape							,(self.batch_size, self.seq_length, self.num_labels)    )






  def 							_lowercase					(			self:						List[Any]    ):







     '''simple docstring'''







     _lowerCamelCase   :				List[Any]							=     self.prepare_config_and_inputs()
     (
         _lowerCamelCase		
     )   :				Tuple							=     config_and_inputs
     _lowerCamelCase   :				List[str]							=     {'''input_ids''': input_ids, '''attention_mask''': input_mask}
     return config, inputs_dict




@require_torch
class 							A_      (       _SCREAMING_SNAKE_CASE     ,	_SCREAMING_SNAKE_CASE     ,	unittest.TestCase ):
  lowerCAmelCase__      			=	False

  lowerCAmelCase__      			=	(
      (
          EsmForMaskedLM,
          EsmModel,
          EsmForSequenceClassification,
          EsmForTokenClassification,
      )
      if is_torch_available()
      else ()
  )
  lowerCAmelCase__      			=	()
  lowerCAmelCase__      			=	(
      {
          '''feature-extraction''': EsmModel,
          '''fill-mask''': EsmForMaskedLM,
          '''text-classification''': EsmForSequenceClassification,
          '''token-classification''': EsmForTokenClassification,
          '''zero-shot''': EsmForSequenceClassification,
      }
      if is_torch_available()
      else {}
  )
  lowerCAmelCase__      			=	True


  def 							_lowercase					(			self:						Union[str, Any]    ):







     '''simple docstring'''







     _lowerCamelCase   :				str							=     EsmModelTester(self    )
     _lowerCamelCase   :				Dict							=     ConfigTester(self							,config_class=__UpperCAmelCase							,hidden_size=37    )


  def 							_lowercase					(			self:						Union[str, Any]    ):







     '''simple docstring'''







     self.config_tester.run_common_tests()


  def 							_lowercase					(			self:						List[str]    ):







     '''simple docstring'''







     _lowerCamelCase   :				Tuple							=     self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_model(*__UpperCAmelCase    )


  def 							_lowercase					(			self:						List[Any]    ):







     '''simple docstring'''







     _lowerCamelCase   :				Optional[int]							=     self.model_tester.prepare_config_and_inputs()
     for type in ["absolute", "relative_key", "relative_key_query"]:
        _lowerCamelCase   :				List[Any]							=     type
        self.model_tester.create_and_check_model(*__UpperCAmelCase    )


  def 							_lowercase					(			self:						int    ):







     '''simple docstring'''







     _lowerCamelCase   :				List[Any]							=     self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_for_masked_lm(*__UpperCAmelCase    )


  def 							_lowercase					(			self:						Tuple    ):







     '''simple docstring'''







     _lowerCamelCase   :				Dict							=     self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_for_token_classification(*__UpperCAmelCase    )


  @slow
  def 							_lowercase					(			self:						Optional[int]    ):







     '''simple docstring'''







     for model_name in ESM_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
        _lowerCamelCase   :				Union[str, Any]							=     EsmModel.from_pretrained(__UpperCAmelCase    )
        self.assertIsNotNone(__UpperCAmelCase    )


  def 							_lowercase					(			self:						List[Any]    ):







     '''simple docstring'''







     _lowerCamelCase   :				Optional[int]							=     self.model_tester.prepare_config_and_inputs()[0]
     _lowerCamelCase   :				Any							=     EsmEmbeddings(config=__UpperCAmelCase    )

     _lowerCamelCase   :				Tuple							=     torch.as_tensor([[12, 31, 13, model.padding_idx]]    )
     _lowerCamelCase   :				str							=     torch.as_tensor(
         [
             [
                 0 + model.padding_idx + 1,
                 1 + model.padding_idx + 1,
                 2 + model.padding_idx + 1,
                 model.padding_idx,
             ]
         ]    )
     _lowerCamelCase   :				str							=     create_position_ids_from_input_ids(__UpperCAmelCase							,model.padding_idx    )
     self.assertEqual(position_ids.shape							,expected_positions.shape    )
     self.assertTrue(torch.all(torch.eq(__UpperCAmelCase							,__UpperCAmelCase    )    )    )


  def 							_lowercase					(			self:						Dict    ):







     '''simple docstring'''







     _lowerCamelCase   :				str							=     self.model_tester.prepare_config_and_inputs()[0]
     _lowerCamelCase   :				Tuple							=     EsmEmbeddings(config=__UpperCAmelCase    )

     _lowerCamelCase   :				List[str]							=     torch.empty(2							,4							,30    )
     _lowerCamelCase   :				Optional[Any]							=     [
         0 + embeddings.padding_idx + 1,
         1 + embeddings.padding_idx + 1,
         2 + embeddings.padding_idx + 1,
         3 + embeddings.padding_idx + 1,
     ]
     _lowerCamelCase   :				int							=     torch.as_tensor([expected_single_positions, expected_single_positions]    )
     _lowerCamelCase   :				Optional[Any]							=     embeddings.create_position_ids_from_inputs_embeds(__UpperCAmelCase    )
     self.assertEqual(position_ids.shape							,expected_positions.shape    )
     self.assertTrue(torch.all(torch.eq(__UpperCAmelCase							,__UpperCAmelCase    )    )    )


  @unittest.skip("Esm does not support embedding resizing"    )
  def 							_lowercase					(			self:						Union[str, Any]    ):







     '''simple docstring'''







     pass


  @unittest.skip("Esm does not support embedding resizing"    )
  def 							_lowercase					(			self:						List[str]    ):







     '''simple docstring'''







     pass






  @unittest.skip("Will be fixed soon by reducing the size of the model used for common tests."    )
  def 							_lowercase					(			self:						int    ):







     '''simple docstring'''







     pass




@require_torch
class 							A_      (       _SCREAMING_SNAKE_CASE ):


  @slow
  def 							_lowercase					(			self:						Any    ):







     '''simple docstring'''







     with torch.no_grad():
        _lowerCamelCase   :				Optional[int]							=     EsmForMaskedLM.from_pretrained("facebook/esm2_t6_8M_UR50D"    )
        model.eval()
        _lowerCamelCase   :				Optional[int]							=     torch.tensor([[0, 1, 2, 3, 4, 5]]    )
        _lowerCamelCase   :				Union[str, Any]							=     model(__UpperCAmelCase    )[0]

        _lowerCamelCase   :				Optional[int]							=     33

        _lowerCamelCase   :				Dict							=     torch.Size((1, 6, vocab_size)    )
        self.assertEqual(output.shape							,__UpperCAmelCase    )

        _lowerCamelCase   :				List[str]							=     torch.tensor(
            [[[8.92_15, -10.5_898, -6.46_71], [-6.39_67, -13.9_114, -1.12_12], [-7.78_12, -13.9_516, -3.74_06]]]    )
        self.assertTrue(torch.allclose(output[:, :3, :3]							,__UpperCAmelCase							,atol=1e-4    )    )






  @slow
  def 							_lowercase					(			self:						Union[str, Any]    ):







     '''simple docstring'''







     with torch.no_grad():
        _lowerCamelCase   :				str							=     EsmModel.from_pretrained("facebook/esm2_t6_8M_UR50D"    )
        model.eval()

        _lowerCamelCase   :				Any							=     torch.tensor([[0, 6, 4, 13, 5, 4, 16, 12, 11, 7, 2]]    )
        _lowerCamelCase   :				Optional[int]							=     model(__UpperCAmelCase    )[0]
        # compare the actual values for a slice.
        _lowerCamelCase   :				str							=     torch.tensor(
            [[[0.14_44, 0.54_13, 0.32_48], [0.30_34, 0.00_53, 0.31_08], [0.32_28, -0.24_99, 0.34_15]]]    )
        self.assertTrue(torch.allclose(output[:, :3, :3]							,__UpperCAmelCase							,atol=1e-4    )    )


from math import factorial







class 					__lowercase  :





							"""simple docstring"""

							def __init__(					self    ,						__UpperCAmelCase    ,						__UpperCAmelCase						)		->       Optional[int]:
														A      :					Union[str, Any]      		=					real
														if isinstance(__UpperCAmelCase    ,						__UpperCAmelCase						):
																					A      :					str      		=					[1] * rank
														else:
																					A      :					Dict      		=					rank




							def __repr__(					self						)		->       List[str]:
														return (
														    f'{self.real}+'
														    f'{"+".join(str(__UpperCAmelCase						)+"E"+str(n+1						)for n,dual in enumerate(self.duals						)						)}'
														)




							def       snake_case			(					self						)		->       Union[str, Any]:
														A      :					List[Any]      		=					self.duals.copy()
														while cur[-1] == 0:
																					cur.pop(-1						)
														return Dual(self.real    ,						__UpperCAmelCase						)




							def __add__(					self    ,						__UpperCAmelCase						)		->       Any:
														if not isinstance(__UpperCAmelCase    ,						__UpperCAmelCase						):
																					return Dual(self.real + other    ,						self.duals						)
														A      :					int      		=					self.duals.copy()
														A      :					Any      		=					other.duals.copy()
														if len(__UpperCAmelCase						) > len(__UpperCAmelCase						):
																					o_dual.extend([1] * (len(__UpperCAmelCase						) - len(__UpperCAmelCase						))						)
														elif len(__UpperCAmelCase						) < len(__UpperCAmelCase						):
																					s_dual.extend([1] * (len(__UpperCAmelCase						) - len(__UpperCAmelCase						))						)
														A      :					List[str]      		=					[]
														for i in range(len(__UpperCAmelCase						)						):
																					new_duals.append(s_dual[i] + o_dual[i]						)
														return Dual(self.real + other.real    ,						__UpperCAmelCase						)

							UpperCAmelCase_						:		int								=	__add__




							def __sub__(					self    ,						__UpperCAmelCase						)		->       List[str]:
														return self + other * -1




							def __mul__(					self    ,						__UpperCAmelCase						)		->       List[str]:
														if not isinstance(__UpperCAmelCase    ,						__UpperCAmelCase						):
																					A      :					Union[str, Any]      		=					[]
																					for i in self.duals:
																												new_duals.append(i * other						)
																					return Dual(self.real * other    ,						__UpperCAmelCase						)
														A      :					Optional[Any]      		=					[0] * (len(self.duals						) + len(other.duals						) + 1)
														for i, item in enumerate(self.duals						):
																					for j, jtem in enumerate(other.duals						):
																												new_duals[i + j + 1] += item * jtem
														for k in range(len(self.duals						)						):
																					new_duals[k] += self.duals[k] * other.real
														for index in range(len(other.duals						)						):
																					new_duals[index] += other.duals[index] * self.real
														return Dual(self.real * other.real    ,						__UpperCAmelCase						)

							UpperCAmelCase_						:		int								=	__mul__




							def __truediv__(					self    ,						__UpperCAmelCase						)		->       Optional[int]:
														if not isinstance(__UpperCAmelCase    ,						__UpperCAmelCase						):
																					A      :					Optional[Any]      		=					[]
																					for i in self.duals:
																												new_duals.append(i / other						)
																					return Dual(self.real / other    ,						__UpperCAmelCase						)
														raise ValueError




							def __floordiv__(					self    ,						__UpperCAmelCase						)		->       Union[str, Any]:
														if not isinstance(__UpperCAmelCase    ,						__UpperCAmelCase						):
																					A      :					List[Any]      		=					[]
																					for i in self.duals:
																												new_duals.append(i // other						)
																					return Dual(self.real // other    ,						__UpperCAmelCase						)
														raise ValueError






							def __pow__(					self    ,						__UpperCAmelCase						)		->       Any:
														if n < 0 or isinstance(__UpperCAmelCase    ,						__UpperCAmelCase						):
																					raise ValueError('''power must be a positive integer'''						)
														if n == 0:
																					return 1
														if n == 1:
																					return self
														A      :					Dict      		=					self
														for _ in range(n - 1						):
																					x *= self
														return x






def        snake_case__		(    lowerCamelCase_     ,				lowerCamelCase_     ,				lowerCamelCase_		):
							if not callable(lowerCamelCase_		):
														raise ValueError('''differentiate() requires a function as input for func'''		)
							if not isinstance(lowerCamelCase_     ,				(float, int)		):
														raise ValueError('''differentiate() requires a float as input for position'''		)
							if not isinstance(lowerCamelCase_     ,				lowerCamelCase_		):
														raise ValueError('''differentiate() requires an int as input for order'''		)
							A      :					Any      		=					Dual(lowerCamelCase_     ,				1		)
							A      :					Dict      		=					func(lowerCamelCase_		)
							if order == 0:
														return result.real
							return result.duals[order - 1] * factorial(lowerCamelCase_		)


if __name__ == "__main__":
		import doctest

		doctest.testmod()






		def        snake_case__		(    lowerCamelCase_		):
									return y**2 * y**4

		print(differentiate(f, 9, 2))






