

import os
import tempfile
import unittest

from transformers import DistilBertConfig, is_torch_available
from transformers.testing_utils import require_torch, require_torch_gpu, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
      import torch

      from transformers import (
          DISTILBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
          DistilBertForMaskedLM,
          DistilBertForMultipleChoice,
          DistilBertForQuestionAnswering,
          DistilBertForSequenceClassification,
          DistilBertForTokenClassification,
          DistilBertModel,
      )

class   _a					(			snake_case_				):




      """simple docstring"""

      def __init__(   self							:				Any						,			UpperCAmelCase							:				int						,			UpperCAmelCase							:				Optional[int]=13						,			UpperCAmelCase							:				List[Any]=7						,			UpperCAmelCase							:				Dict=True						,			UpperCAmelCase							:				Any=True						,			UpperCAmelCase							:				Dict=False						,			UpperCAmelCase							:				Union[str, Any]=True						,			UpperCAmelCase							:				Dict=99						,			UpperCAmelCase							:				Tuple=32						,			UpperCAmelCase							:				str=5						,			UpperCAmelCase							:				List[Any]=4						,			UpperCAmelCase							:				Any=37						,			UpperCAmelCase							:				str="gelu"						,			UpperCAmelCase							:				Any=0.1						,			UpperCAmelCase							:				Optional[int]=0.1						,			UpperCAmelCase							:				int=512						,			UpperCAmelCase							:				Union[str, Any]=16						,			UpperCAmelCase							:				Tuple=2						,			UpperCAmelCase							:				List[Any]=0.02						,			UpperCAmelCase							:				Any=3						,			UpperCAmelCase							:				Any=4						,			UpperCAmelCase							:				Any=None						,			):
            A_						  =				parent
            A_						  =				batch_size
            A_						  =				seq_length
            A_						  =				is_training
            A_						  =				use_input_mask
            A_						  =				use_token_type_ids
            A_						  =				use_labels
            A_						  =				vocab_size
            A_						  =				hidden_size
            A_						  =				num_hidden_layers
            A_						  =				num_attention_heads
            A_						  =				intermediate_size
            A_						  =				hidden_act
            A_						  =				hidden_dropout_prob
            A_						  =				attention_probs_dropout_prob
            A_						  =				max_position_embeddings
            A_						  =				type_vocab_size
            A_						  =				type_sequence_label_size
            A_						  =				initializer_range
            A_						  =				num_labels
            A_						  =				num_choices
            A_						  =				scope





      def 		__A	(   self							:				Optional[int]	):
            A_						  =				ids_tensor([self.batch_size, self.seq_length]						,			self.vocab_size	)

            A_						  =				None
            if self.use_input_mask:
                  A_						  =				random_attention_mask([self.batch_size, self.seq_length]	)

            A_						  =				None
            A_						  =				None
            A_						  =				None
            if self.use_labels:
                  A_						  =				ids_tensor([self.batch_size]						,			self.type_sequence_label_size	)
                  A_						  =				ids_tensor([self.batch_size, self.seq_length]						,			self.num_labels	)
                  A_						  =				ids_tensor([self.batch_size]						,			self.num_choices	)

            A_						  =				self.get_config()

            return config, input_ids, input_mask, sequence_labels, token_labels, choice_labels





      def 		__A	(   self							:				Optional[Any]	):
            return DistilBertConfig(
                vocab_size=self.vocab_size						,			dim=self.hidden_size						,			n_layers=self.num_hidden_layers						,			n_heads=self.num_attention_heads						,			hidden_dim=self.intermediate_size						,			hidden_act=self.hidden_act						,			dropout=self.hidden_dropout_prob						,			attention_dropout=self.attention_probs_dropout_prob						,			max_position_embeddings=self.max_position_embeddings						,			initializer_range=self.initializer_range						,			)





      def 		__A	(   self							:				Union[str, Any]						,			UpperCAmelCase							:				str						,			UpperCAmelCase							:				int						,			UpperCAmelCase							:				int						,			UpperCAmelCase							:				Optional[int]						,			UpperCAmelCase							:				List[Any]						,			UpperCAmelCase							:				List[Any]	):
            A_						  =				DistilBertModel(config=UpperCAmelCase	)
            model.to(UpperCAmelCase	)
            model.eval()
            A_						  =				model(UpperCAmelCase						,			UpperCAmelCase	)
            A_						  =				model(UpperCAmelCase	)
            self.parent.assertEqual(result.last_hidden_state.shape						,			(self.batch_size, self.seq_length, self.hidden_size)	)





      def 		__A	(   self							:				Optional[int]						,			UpperCAmelCase							:				Union[str, Any]						,			UpperCAmelCase							:				List[str]						,			UpperCAmelCase							:				Optional[Any]						,			UpperCAmelCase							:				int						,			UpperCAmelCase							:				Any						,			UpperCAmelCase							:				List[Any]	):
            A_						  =				DistilBertForMaskedLM(config=UpperCAmelCase	)
            model.to(UpperCAmelCase	)
            model.eval()
            A_						  =				model(UpperCAmelCase						,			attention_mask=UpperCAmelCase						,			labels=UpperCAmelCase	)
            self.parent.assertEqual(result.logits.shape						,			(self.batch_size, self.seq_length, self.vocab_size)	)





      def 		__A	(   self							:				str						,			UpperCAmelCase							:				Dict						,			UpperCAmelCase							:				List[str]						,			UpperCAmelCase							:				str						,			UpperCAmelCase							:				int						,			UpperCAmelCase							:				int						,			UpperCAmelCase							:				Optional[int]	):
            A_						  =				DistilBertForQuestionAnswering(config=UpperCAmelCase	)
            model.to(UpperCAmelCase	)
            model.eval()
            A_						  =				model(
                UpperCAmelCase						,			attention_mask=UpperCAmelCase						,			start_positions=UpperCAmelCase						,			end_positions=UpperCAmelCase	)
            self.parent.assertEqual(result.start_logits.shape						,			(self.batch_size, self.seq_length)	)
            self.parent.assertEqual(result.end_logits.shape						,			(self.batch_size, self.seq_length)	)





      def 		__A	(   self							:				Union[str, Any]						,			UpperCAmelCase							:				int						,			UpperCAmelCase							:				List[str]						,			UpperCAmelCase							:				Optional[Any]						,			UpperCAmelCase							:				Union[str, Any]						,			UpperCAmelCase							:				Dict						,			UpperCAmelCase							:				List[Any]	):
            A_						  =				self.num_labels
            A_						  =				DistilBertForSequenceClassification(UpperCAmelCase	)
            model.to(UpperCAmelCase	)
            model.eval()
            A_						  =				model(UpperCAmelCase						,			attention_mask=UpperCAmelCase						,			labels=UpperCAmelCase	)
            self.parent.assertEqual(result.logits.shape						,			(self.batch_size, self.num_labels)	)





      def 		__A	(   self							:				int						,			UpperCAmelCase							:				Any						,			UpperCAmelCase							:				str						,			UpperCAmelCase							:				List[Any]						,			UpperCAmelCase							:				Optional[Any]						,			UpperCAmelCase							:				Optional[int]						,			UpperCAmelCase							:				Any	):
            A_						  =				self.num_labels
            A_						  =				DistilBertForTokenClassification(config=UpperCAmelCase	)
            model.to(UpperCAmelCase	)
            model.eval()

            A_						  =				model(UpperCAmelCase						,			attention_mask=UpperCAmelCase						,			labels=UpperCAmelCase	)
            self.parent.assertEqual(result.logits.shape						,			(self.batch_size, self.seq_length, self.num_labels)	)





      def 		__A	(   self							:				Any						,			UpperCAmelCase							:				List[str]						,			UpperCAmelCase							:				Optional[Any]						,			UpperCAmelCase							:				Union[str, Any]						,			UpperCAmelCase							:				Any						,			UpperCAmelCase							:				List[Any]						,			UpperCAmelCase							:				Optional[int]	):
            A_						  =				self.num_choices
            A_						  =				DistilBertForMultipleChoice(config=UpperCAmelCase	)
            model.to(UpperCAmelCase	)
            model.eval()
            A_						  =				input_ids.unsqueeze(1	).expand(-1						,			self.num_choices						,			-1	).contiguous()
            A_						  =				input_mask.unsqueeze(1	).expand(-1						,			self.num_choices						,			-1	).contiguous()
            A_						  =				model(
                UpperCAmelCase						,			attention_mask=UpperCAmelCase						,			labels=UpperCAmelCase						,			)
            self.parent.assertEqual(result.logits.shape						,			(self.batch_size, self.num_choices)	)







      def 		__A	(   self							:				str	):
            A_						  =				self.prepare_config_and_inputs()
            ((A_)  ,     (A_)  ,     (A_)  ,     (A_)  ,     (A_)  ,     (A_))						  =				config_and_inputs
            A_						  =				{"input_ids": input_ids, "attention_mask": input_mask}
            return config, inputs_dict




@require_torch
class   _a					(			snake_case_     ,				snake_case_     ,				unittest.TestCase				):




      """simple docstring"""

      _lowerCamelCase       :			Optional[Any]      		=       (
          (
              DistilBertModel,
              DistilBertForMaskedLM,
              DistilBertForMultipleChoice,
              DistilBertForQuestionAnswering,
              DistilBertForSequenceClassification,
              DistilBertForTokenClassification,
          )
          if is_torch_available()
          else None
      )
      _lowerCamelCase       :			List[Any]      		=       (
          {
              'feature-extraction': DistilBertModel,
              'fill-mask': DistilBertForMaskedLM,
              'question-answering': DistilBertForQuestionAnswering,
              'text-classification': DistilBertForSequenceClassification,
              'token-classification': DistilBertForTokenClassification,
              'zero-shot': DistilBertForSequenceClassification,
          }
          if is_torch_available()
          else {}
      )
      _lowerCamelCase       :			Any      		=       True
      _lowerCamelCase       :			int      		=       True
      _lowerCamelCase       :			Union[str, Any]      		=       True
      _lowerCamelCase       :			str      		=       True





      def 		__A	(   self							:				Union[str, Any]	):
            A_						  =				DistilBertModelTester(self	)
            A_						  =				ConfigTester(self						,			config_class=UpperCAmelCase						,			dim=37	)





      def 		__A	(   self							:				Dict	):
            self.config_tester.run_common_tests()





      def 		__A	(   self							:				Tuple	):
            A_						  =				self.model_tester.prepare_config_and_inputs()
            self.model_tester.create_and_check_distilbert_model(*UpperCAmelCase	)





      def 		__A	(   self							:				int	):
            A_						  =				self.model_tester.prepare_config_and_inputs()
            self.model_tester.create_and_check_distilbert_for_masked_lm(*UpperCAmelCase	)





      def 		__A	(   self							:				Dict	):
            A_						  =				self.model_tester.prepare_config_and_inputs()
            self.model_tester.create_and_check_distilbert_for_question_answering(*UpperCAmelCase	)





      def 		__A	(   self							:				Any	):
            A_						  =				self.model_tester.prepare_config_and_inputs()
            self.model_tester.create_and_check_distilbert_for_sequence_classification(*UpperCAmelCase	)





      def 		__A	(   self							:				int	):
            A_						  =				self.model_tester.prepare_config_and_inputs()
            self.model_tester.create_and_check_distilbert_for_token_classification(*UpperCAmelCase	)





      def 		__A	(   self							:				List[str]	):
            A_						  =				self.model_tester.prepare_config_and_inputs()
            self.model_tester.create_and_check_distilbert_for_multiple_choice(*UpperCAmelCase	)





      @slow
      def 		__A	(   self							:				Optional[int]	):
            for model_name in DISTILBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
                  A_						  =				DistilBertModel.from_pretrained(UpperCAmelCase	)
                  self.assertIsNotNone(UpperCAmelCase	)







      @slow
      @require_torch_gpu
      def 		__A	(   self							:				Optional[int]	):
            A_  ,     A_						  =				self.model_tester.prepare_config_and_inputs_for_common()
            for model_class in self.all_model_classes:
                  # BertForMultipleChoice behaves incorrectly in JIT environments.
                  if model_class == DistilBertForMultipleChoice:
                        return

                  A_						  =				True
                  A_						  =				model_class(config=UpperCAmelCase	)

                  A_						  =				self._prepare_for_class(UpperCAmelCase						,			UpperCAmelCase	)
                  A_						  =				torch.jit.trace(
                      UpperCAmelCase						,			(inputs_dict["input_ids"].to("cpu"	), inputs_dict["attention_mask"].to("cpu"	))	)

                  with tempfile.TemporaryDirectory() as tmp:
                        torch.jit.save(UpperCAmelCase						,			os.path.join(UpperCAmelCase						,			"traced_model.pt"	)	)
                        A_						  =				torch.jit.load(os.path.join(UpperCAmelCase						,			"traced_model.pt"	)						,			map_location=UpperCAmelCase	)
                        loaded(inputs_dict["input_ids"].to(UpperCAmelCase	)						,			inputs_dict["attention_mask"].to(UpperCAmelCase	)	)




@require_torch
class   _a					(			unittest.TestCase				):




      """simple docstring"""

      @slow
      def 		__A	(   self							:				Optional[int]	):
            A_						  =				DistilBertModel.from_pretrained("distilbert-base-uncased"	)
            A_						  =				torch.tensor([[0, 345, 232, 328, 740, 140, 1695, 69, 6078, 1588, 2]]	)
            A_						  =				torch.tensor([[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]	)
            with torch.no_grad():
                  A_						  =				model(UpperCAmelCase						,			attention_mask=UpperCAmelCase	)[0]
            A_						  =				torch.Size((1, 11, 768)	)
            self.assertEqual(output.shape						,			UpperCAmelCase	)
            A_						  =				torch.tensor(
                [[[-0.1_639, 0.3_299, 0.1_648], [-0.1_746, 0.3_289, 0.1_710], [-0.1_884, 0.3_357, 0.1_810]]]	)

            self.assertTrue(torch.allclose(output[:, 1:4, 1:4]						,			UpperCAmelCase						,			atol=1E-4	)	)






import unittest

from transformers import EsmConfig, is_torch_available
from transformers.testing_utils import TestCasePlus, require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
							import torch

							from transformers import EsmForMaskedLM, EsmForSequenceClassification, EsmForTokenClassification, EsmModel
							from transformers.models.esm.modeling_esm import (
							    ESM_PRETRAINED_MODEL_ARCHIVE_LIST,
							    EsmEmbeddings,
							    create_position_ids_from_input_ids,
							)





class 				a			:






				def __init__(  self			, _lowerCAmelCase			, _lowerCAmelCase=13			, _lowerCAmelCase=7			, _lowerCAmelCase=False			, _lowerCAmelCase=True			, _lowerCAmelCase=False			, _lowerCAmelCase=True			, _lowerCAmelCase=33			, _lowerCAmelCase=32			, _lowerCAmelCase=5			, _lowerCAmelCase=4			, _lowerCAmelCase=37			, _lowerCAmelCase="gelu"			, _lowerCAmelCase=0.1			, _lowerCAmelCase=0.1			, _lowerCAmelCase=512			, _lowerCAmelCase=16			, _lowerCAmelCase=2			, _lowerCAmelCase=0.02			, _lowerCAmelCase=3			, _lowerCAmelCase=4			, _lowerCAmelCase=None			, ):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				Union[str, Any]           =   parent
										__SCREAMING_SNAKE_CASE:				Any           =   batch_size
										__SCREAMING_SNAKE_CASE:				Dict           =   seq_length
										__SCREAMING_SNAKE_CASE:				List[Any]           =   is_training
										__SCREAMING_SNAKE_CASE:				List[Any]           =   use_input_mask
										__SCREAMING_SNAKE_CASE:				str           =   use_token_type_ids
										__SCREAMING_SNAKE_CASE:				Any           =   use_labels
										__SCREAMING_SNAKE_CASE:				Optional[int]           =   vocab_size
										__SCREAMING_SNAKE_CASE:				Dict           =   hidden_size
										__SCREAMING_SNAKE_CASE:				Dict           =   num_hidden_layers
										__SCREAMING_SNAKE_CASE:				Optional[int]           =   num_attention_heads
										__SCREAMING_SNAKE_CASE:				str           =   intermediate_size
										__SCREAMING_SNAKE_CASE:				str           =   hidden_act
										__SCREAMING_SNAKE_CASE:				Optional[int]           =   hidden_dropout_prob
										__SCREAMING_SNAKE_CASE:				Union[str, Any]           =   attention_probs_dropout_prob
										__SCREAMING_SNAKE_CASE:				Any           =   max_position_embeddings
										__SCREAMING_SNAKE_CASE:				Optional[Any]           =   type_vocab_size
										__SCREAMING_SNAKE_CASE:				List[Any]           =   type_sequence_label_size
										__SCREAMING_SNAKE_CASE:				Union[str, Any]           =   initializer_range
										__SCREAMING_SNAKE_CASE:				Union[str, Any]           =   num_labels
										__SCREAMING_SNAKE_CASE:				List[str]           =   num_choices
										__SCREAMING_SNAKE_CASE:				List[Any]           =   scope






				def 			snake_case_    (  self					):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				List[str]           =   ids_tensor([self.batch_size, self.seq_length]			, self.vocab_size					)

										__SCREAMING_SNAKE_CASE:				int           =   None
										if self.use_input_mask:
																__SCREAMING_SNAKE_CASE:				Tuple           =   random_attention_mask([self.batch_size, self.seq_length]					)

										__SCREAMING_SNAKE_CASE:				Optional[Any]           =   None
										__SCREAMING_SNAKE_CASE:				Optional[int]           =   None
										__SCREAMING_SNAKE_CASE:				Optional[Any]           =   None
										if self.use_labels:
																__SCREAMING_SNAKE_CASE:				List[Any]           =   ids_tensor([self.batch_size]			, self.type_sequence_label_size					)
																__SCREAMING_SNAKE_CASE:				List[str]           =   ids_tensor([self.batch_size, self.seq_length]			, self.num_labels					)
																__SCREAMING_SNAKE_CASE:				str           =   ids_tensor([self.batch_size]			, self.num_choices					)

										__SCREAMING_SNAKE_CASE:				Optional[int]           =   self.get_config()

										return config, input_ids, input_mask, sequence_labels, token_labels, choice_labels






				def 			snake_case_    (  self					):

										"""simple docstring"""




										return EsmConfig(
										    vocab_size=self.vocab_size			, hidden_size=self.hidden_size			, pad_token_id=1			, num_hidden_layers=self.num_hidden_layers			, num_attention_heads=self.num_attention_heads			, intermediate_size=self.intermediate_size			, hidden_act=self.hidden_act			, hidden_dropout_prob=self.hidden_dropout_prob			, attention_probs_dropout_prob=self.attention_probs_dropout_prob			, max_position_embeddings=self.max_position_embeddings			, type_vocab_size=self.type_vocab_size			, initializer_range=self.initializer_range			, )






				def 			snake_case_    (  self			, _lowerCAmelCase			, _lowerCAmelCase			, _lowerCAmelCase			, _lowerCAmelCase			, _lowerCAmelCase			, _lowerCAmelCase					):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				List[Any]           =   EsmModel(config=_lowerCAmelCase					)
										model.to(_lowerCAmelCase					)
										model.eval()
										__SCREAMING_SNAKE_CASE:				Union[str, Any]           =   model(_lowerCAmelCase			, attention_mask=_lowerCAmelCase					)
										__SCREAMING_SNAKE_CASE:				Tuple           =   model(_lowerCAmelCase					)
										__SCREAMING_SNAKE_CASE:				int           =   model(_lowerCAmelCase					)

										self.parent.assertEqual(result.last_hidden_state.shape			, (self.batch_size, self.seq_length, self.hidden_size)					)
										self.parent.assertEqual(result.pooler_output.shape			, (self.batch_size, self.hidden_size)					)






				def 			snake_case_    (  self			, _lowerCAmelCase			, _lowerCAmelCase			, _lowerCAmelCase			, _lowerCAmelCase			, _lowerCAmelCase			, _lowerCAmelCase					):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				List[Any]           =   EsmForMaskedLM(config=_lowerCAmelCase					)
										model.to(_lowerCAmelCase					)
										model.eval()
										__SCREAMING_SNAKE_CASE:				Optional[Any]           =   model(_lowerCAmelCase			, attention_mask=_lowerCAmelCase			, labels=_lowerCAmelCase					)
										self.parent.assertEqual(result.logits.shape			, (self.batch_size, self.seq_length, self.vocab_size)					)






				def 			snake_case_    (  self			, _lowerCAmelCase			, _lowerCAmelCase			, _lowerCAmelCase			, _lowerCAmelCase			, _lowerCAmelCase			, _lowerCAmelCase					):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				Any           =   self.num_labels
										__SCREAMING_SNAKE_CASE:				List[Any]           =   EsmForTokenClassification(config=_lowerCAmelCase					)
										model.to(_lowerCAmelCase					)
										model.eval()
										__SCREAMING_SNAKE_CASE:				int           =   model(_lowerCAmelCase			, attention_mask=_lowerCAmelCase			, labels=_lowerCAmelCase					)
										self.parent.assertEqual(result.logits.shape			, (self.batch_size, self.seq_length, self.num_labels)					)






				def 			snake_case_    (  self					):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				str           =   self.prepare_config_and_inputs()
										(
										    (
										    __SCREAMING_SNAKE_CASE
										)       ,(
										    __SCREAMING_SNAKE_CASE
										)       ,(
										    __SCREAMING_SNAKE_CASE
										)       ,(
										    __SCREAMING_SNAKE_CASE
										)       ,(
										    __SCREAMING_SNAKE_CASE
										)       ,(
										    __SCREAMING_SNAKE_CASE
										)       ,
										):				int           =   config_and_inputs
										__SCREAMING_SNAKE_CASE:				Tuple           =   {'''input_ids''': input_ids, '''attention_mask''': input_mask}
										return config, inputs_dict





@require_torch
class 				a			(      __lowercase   ,__lowercase   ,unittest.TestCase			):
				SCREAMING_SNAKE_CASE__     :   Tuple		      =     False

				SCREAMING_SNAKE_CASE__     :   Optional[int]		      =     (
				    (
				        EsmForMaskedLM,
				        EsmModel,
				        EsmForSequenceClassification,
				        EsmForTokenClassification,
				    )
				    if is_torch_available()
				    else ()
				)
				SCREAMING_SNAKE_CASE__     :   int		      =     ()
				SCREAMING_SNAKE_CASE__     :   Optional[int]		      =     (
				    {
				        '''feature-extraction''': EsmModel,
				        '''fill-mask''': EsmForMaskedLM,
				        '''text-classification''': EsmForSequenceClassification,
				        '''token-classification''': EsmForTokenClassification,
				        '''zero-shot''': EsmForSequenceClassification,
				    }
				    if is_torch_available()
				    else {}
				)
				SCREAMING_SNAKE_CASE__     :   Any		      =     True






				def 			snake_case_    (  self					):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				List[str]           =   EsmModelTester(self					)
										__SCREAMING_SNAKE_CASE:				List[str]           =   ConfigTester(self			, config_class=_lowerCAmelCase			, hidden_size=37					)






				def 			snake_case_    (  self					):

										"""simple docstring"""




										self.config_tester.run_common_tests()






				def 			snake_case_    (  self					):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				Dict           =   self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_model(*_lowerCAmelCase					)






				def 			snake_case_    (  self					):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				Optional[int]           =   self.model_tester.prepare_config_and_inputs()
										for type in ["absolute", "relative_key", "relative_key_query"]:
																__SCREAMING_SNAKE_CASE:				Any           =   type
																self.model_tester.create_and_check_model(*_lowerCAmelCase					)






				def 			snake_case_    (  self					):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				Optional[int]           =   self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_for_masked_lm(*_lowerCAmelCase					)






				def 			snake_case_    (  self					):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				Optional[int]           =   self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_for_token_classification(*_lowerCAmelCase					)






				@slow
				def 			snake_case_    (  self					):

										"""simple docstring"""




										for model_name in ESM_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
																__SCREAMING_SNAKE_CASE:				Optional[int]           =   EsmModel.from_pretrained(_lowerCAmelCase					)
																self.assertIsNotNone(_lowerCAmelCase					)






				def 			snake_case_    (  self					):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				Union[str, Any]           =   self.model_tester.prepare_config_and_inputs()[0]
										__SCREAMING_SNAKE_CASE:				int           =   EsmEmbeddings(config=_lowerCAmelCase					)

										__SCREAMING_SNAKE_CASE:				Union[str, Any]           =   torch.as_tensor([[12, 31, 13, model.padding_idx]]					)
										__SCREAMING_SNAKE_CASE:				Optional[Any]           =   torch.as_tensor(
										    [
										        [
										            0 + model.padding_idx + 1,
										            1 + model.padding_idx + 1,
										            2 + model.padding_idx + 1,
										            model.padding_idx,
										        ]
										    ]					)
										__SCREAMING_SNAKE_CASE:				Any           =   create_position_ids_from_input_ids(_lowerCAmelCase			, model.padding_idx					)
										self.assertEqual(position_ids.shape			, expected_positions.shape					)
										self.assertTrue(torch.all(torch.eq(_lowerCAmelCase			, _lowerCAmelCase					)					)					)






				def 			snake_case_    (  self					):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				Tuple           =   self.model_tester.prepare_config_and_inputs()[0]
										__SCREAMING_SNAKE_CASE:				Dict           =   EsmEmbeddings(config=_lowerCAmelCase					)

										__SCREAMING_SNAKE_CASE:				Optional[int]           =   torch.empty(2			, 4			, 30					)
										__SCREAMING_SNAKE_CASE:				List[str]           =   [
										    0 + embeddings.padding_idx + 1,
										    1 + embeddings.padding_idx + 1,
										    2 + embeddings.padding_idx + 1,
										    3 + embeddings.padding_idx + 1,
										]
										__SCREAMING_SNAKE_CASE:				str           =   torch.as_tensor([expected_single_positions, expected_single_positions]					)
										__SCREAMING_SNAKE_CASE:				Optional[Any]           =   embeddings.create_position_ids_from_inputs_embeds(_lowerCAmelCase					)
										self.assertEqual(position_ids.shape			, expected_positions.shape					)
										self.assertTrue(torch.all(torch.eq(_lowerCAmelCase			, _lowerCAmelCase					)					)					)






				@unittest.skip('''Esm does not support embedding resizing'''					)
				def 			snake_case_    (  self					):

										"""simple docstring"""




										pass






				@unittest.skip('''Esm does not support embedding resizing'''					)
				def 			snake_case_    (  self					):

										"""simple docstring"""




										pass






				@unittest.skip('''Will be fixed soon by reducing the size of the model used for common tests.'''					)
				def 			snake_case_    (  self					):

										"""simple docstring"""




										pass





@require_torch
class 				a			(      __lowercase			):






				@slow
				def 			snake_case_    (  self					):

										"""simple docstring"""




										with torch.no_grad():
																__SCREAMING_SNAKE_CASE:				Optional[int]           =   EsmForMaskedLM.from_pretrained('''facebook/esm2_t6_8M_UR50D'''					)
																model.eval()
																__SCREAMING_SNAKE_CASE:				Tuple           =   torch.tensor([[0, 1, 2, 3, 4, 5]]					)
																__SCREAMING_SNAKE_CASE:				str           =   model(_lowerCAmelCase					)[0]

																__SCREAMING_SNAKE_CASE:				Union[str, Any]           =   33

																__SCREAMING_SNAKE_CASE:				Dict           =   torch.Size((1, 6, vocab_size)					)
																self.assertEqual(output.shape			, _lowerCAmelCase					)

																__SCREAMING_SNAKE_CASE:				Optional[int]           =   torch.tensor(
																    [[[8.9215, -10.5898, -6.4671], [-6.3967, -13.9114, -1.1212], [-7.7812, -13.9516, -3.7406]]]					)
																self.assertTrue(torch.allclose(output[:, :3, :3]			, _lowerCAmelCase			, atol=1e-4					)					)






				@slow
				def 			snake_case_    (  self					):

										"""simple docstring"""




										with torch.no_grad():
																__SCREAMING_SNAKE_CASE:				Dict           =   EsmModel.from_pretrained('''facebook/esm2_t6_8M_UR50D'''					)
																model.eval()

																__SCREAMING_SNAKE_CASE:				Dict           =   torch.tensor([[0, 6, 4, 13, 5, 4, 16, 12, 11, 7, 2]]					)
																__SCREAMING_SNAKE_CASE:				Union[str, Any]           =   model(_lowerCAmelCase					)[0]
																# compare the actual values for a slice.
																__SCREAMING_SNAKE_CASE:				Optional[int]           =   torch.tensor(
																    [[[0.1444, 0.5413, 0.3248], [0.3034, 0.0053, 0.3108], [0.3228, -0.2499, 0.3415]]]					)
																self.assertTrue(torch.allclose(output[:, :3, :3]			, _lowerCAmelCase			, atol=1e-4					)					)



