






'''simple docstring'''



from __future__ import annotations

import unittest

from transformers import EsmConfig, is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
 import numpy
 import tensorflow as tf

 from transformers.models.esm.modeling_tf_esm import (
     TF_ESM_PRETRAINED_MODEL_ARCHIVE_LIST,
     TFEsmForMaskedLM,
     TFEsmForSequenceClassification,
     TFEsmForTokenClassification,
     TFEsmModel,
 )
class 				__SCREAMING_SNAKE_CASE							:




      def __init__(	self			, lowerCamelCase			, )				->Optional[int]:



         '''simple docstring'''
         __a					  =     parent
         __a					  =     13
         __a					  =     7
         __a					  =     True
         __a					  =     True
         __a					  =     True
         __a					  =     99
         __a					  =     32
         __a					  =     2
         __a					  =     4
         __a					  =     37
         __a					  =     'gelu'
         __a					  =     0.1
         __a					  =     0.1
         __a					  =     512
         __a					  =     16
         __a					  =     2
         __a					  =     0.02
         __a					  =     3
         __a					  =     4
         __a					  =     None




      def 			__UpperCamelCase				(	self   )				->str:



         '''simple docstring'''
         __a					  =     ids_tensor([self.batch_size, self.seq_length]			, self.vocab_size   )

         __a					  =     None
         if self.use_input_mask:
            __a					  =     random_attention_mask([self.batch_size, self.seq_length]   )

         __a					  =     None
         __a					  =     None
         __a					  =     None
         if self.use_labels:
            __a					  =     ids_tensor([self.batch_size]			, self.type_sequence_label_size   )
            __a					  =     ids_tensor([self.batch_size, self.seq_length]			, self.num_labels   )
            __a					  =     ids_tensor([self.batch_size]			, self.num_choices   )

         __a					  =     EsmConfig(
             vocab_size=self.vocab_size			, hidden_size=self.hidden_size			, num_hidden_layers=self.num_hidden_layers			, pad_token_id=1			, num_attention_heads=self.num_attention_heads			, intermediate_size=self.intermediate_size			, hidden_act=self.hidden_act			, hidden_dropout_prob=self.hidden_dropout_prob			, attention_probs_dropout_prob=self.attention_probs_dropout_prob			, max_position_embeddings=self.max_position_embeddings			, type_vocab_size=self.type_vocab_size			, initializer_range=self.initializer_range			, )

         return config, input_ids, input_mask, sequence_labels, token_labels, choice_labels




      def 			__UpperCamelCase				(	self   )				->Optional[int]:



         '''simple docstring'''
         (
             (
             __a
         )  ,      (
             __a
         )  ,      (
             __a
         )  ,      (
             __a
         )  ,      (
             __a
         )  ,      (
             __a
         )  ,      
         )					  =     self.prepare_config_and_inputs()

         __a					  =     True
         __a					  =     floats_tensor([self.batch_size, self.seq_length, self.hidden_size]   )
         __a					  =     ids_tensor([self.batch_size, self.seq_length]			, vocab_size=2   )

         return (
             config,
             input_ids,
             input_mask,
             sequence_labels,
             token_labels,
             choice_labels,
             encoder_hidden_states,
             encoder_attention_mask,
         )




      def 			__UpperCamelCase				(	self			, lowerCamelCase			, lowerCamelCase			, lowerCamelCase			, lowerCamelCase			, lowerCamelCase			, lowerCamelCase   )				->str:



         '''simple docstring'''
         __a					  =     TFEsmModel(config=lowerCamelCase   )
         __a					  =     {'input_ids': input_ids, 'attention_mask': input_mask}
         __a					  =     model(lowerCamelCase   )

         __a					  =     [input_ids, input_mask]
         __a					  =     model(lowerCamelCase   )

         __a					  =     model(lowerCamelCase   )

         self.parent.assertEqual(result.last_hidden_state.shape			, (self.batch_size, self.seq_length, self.hidden_size)   )




      def 			__UpperCamelCase				(	self			, lowerCamelCase			, lowerCamelCase			, lowerCamelCase			, lowerCamelCase			, lowerCamelCase			, lowerCamelCase			, lowerCamelCase			, lowerCamelCase			, )				->Optional[Any]:



         '''simple docstring'''
         __a					  =     True

         __a					  =     TFEsmModel(config=lowerCamelCase   )
         __a					  =     {
             'input_ids': input_ids,
             'attention_mask': input_mask,
             'encoder_hidden_states': encoder_hidden_states,
             'encoder_attention_mask': encoder_attention_mask,
         }
         __a					  =     model(lowerCamelCase   )

         __a					  =     [input_ids, input_mask]
         __a					  =     model(lowerCamelCase			, encoder_hidden_states=lowerCamelCase   )

         # Also check the case where encoder outputs are not passed
         __a					  =     model(lowerCamelCase			, attention_mask=lowerCamelCase   )

         self.parent.assertEqual(result.last_hidden_state.shape			, (self.batch_size, self.seq_length, self.hidden_size)   )




      def 			__UpperCamelCase				(	self			, lowerCamelCase			, lowerCamelCase			, lowerCamelCase			, lowerCamelCase			, lowerCamelCase			, lowerCamelCase   )				->Union[str, Any]:



         '''simple docstring'''
         __a					  =     TFEsmForMaskedLM(config=lowerCamelCase   )
         __a					  =     model([input_ids, input_mask]   )
         self.parent.assertEqual(result.logits.shape			, (self.batch_size, self.seq_length, self.vocab_size)   )




      def 			__UpperCamelCase				(	self			, lowerCamelCase			, lowerCamelCase			, lowerCamelCase			, lowerCamelCase			, lowerCamelCase			, lowerCamelCase   )				->Optional[int]:



         '''simple docstring'''
         __a					  =     self.num_labels
         __a					  =     TFEsmForTokenClassification(config=lowerCamelCase   )
         __a					  =     {'input_ids': input_ids, 'attention_mask': input_mask}
         __a					  =     model(lowerCamelCase   )
         self.parent.assertEqual(result.logits.shape			, (self.batch_size, self.seq_length, self.num_labels)   )






      def 			__UpperCamelCase				(	self   )				->Optional[int]:



         '''simple docstring'''
         __a					  =     self.prepare_config_and_inputs()
         (
             (
             __a
         )  ,      (
             __a
         )  ,      (
             __a
         )  ,      (
             __a
         )  ,      (
             __a
         )  ,      (
             __a
         )  ,      
         )					  =     config_and_inputs
         __a					  =     {'input_ids': input_ids, 'attention_mask': input_mask}
         return config, inputs_dict







@require_tf
class 				__SCREAMING_SNAKE_CASE							(				_lowerCAmelCase   ,   _lowerCAmelCase   ,   unittest.TestCase  ):
      __a	       =(
          (
              TFEsmModel,
              TFEsmForMaskedLM,
              TFEsmForSequenceClassification,
              TFEsmForTokenClassification,
          )
          if is_tf_available()
          else ()
      )
      __a	       =(
          {
              "feature-extraction": TFEsmModel,
              "fill-mask": TFEsmForMaskedLM,
              "text-classification": TFEsmForSequenceClassification,
              "token-classification": TFEsmForTokenClassification,
              "zero-shot": TFEsmForSequenceClassification,
          }
          if is_tf_available()
          else {}
      )
      __a	       =False
      __a	       =False




      def 			__UpperCamelCase				(	self   )				->Optional[Any]:



         '''simple docstring'''
         __a					  =     TFEsmModelTester(self   )
         __a					  =     ConfigTester(self			, config_class=lowerCamelCase			, hidden_size=37   )




      def 			__UpperCamelCase				(	self   )				->Optional[Any]:



         '''simple docstring'''
         self.config_tester.run_common_tests()




      def 			__UpperCamelCase				(	self   )				->str:



         '''simple docstring'''
         __a					  =     self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_model(*lowerCamelCase   )




      def 			__UpperCamelCase				(	self   )				->Union[str, Any]:



         '''simple docstring'''
         __a					  =     self.model_tester.prepare_config_and_inputs_for_decoder()
         self.model_tester.create_and_check_model_as_decoder(*lowerCamelCase   )




      def 			__UpperCamelCase				(	self   )				->List[Any]:



         '''simple docstring'''
         __a					  =     self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_masked_lm(*lowerCamelCase   )




      def 			__UpperCamelCase				(	self   )				->Dict:



         '''simple docstring'''
         __a					  =     self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_token_classification(*lowerCamelCase   )




      @slow
      def 			__UpperCamelCase				(	self   )				->List[str]:



         '''simple docstring'''
         for model_name in TF_ESM_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
            __a					  =     TFEsmModel.from_pretrained(lowerCamelCase   )
            self.assertIsNotNone(lowerCamelCase   )




      @unittest.skip('Protein models do not support embedding resizing.'   )
      def 			__UpperCamelCase				(	self   )				->List[str]:



         '''simple docstring'''
         pass




      @unittest.skip('Protein models do not support embedding resizing.'   )
      def 			__UpperCamelCase				(	self   )				->Any:



         '''simple docstring'''
         pass






      def 			__UpperCamelCase				(	self   )				->Any:



         '''simple docstring'''
         __a  ,      __a					  =     self.model_tester.prepare_config_and_inputs_for_common()

         for model_class in self.all_model_classes:
            __a					  =     model_class(lowerCamelCase   )
            assert isinstance(model.get_input_embeddings()			, tf.keras.layers.Layer   )
            if model_class is TFEsmForMaskedLM:
               # Output embedding test differs from the main test because they're a matrix, not a layer
               __a					  =     model.get_bias()
               assert isinstance(lowerCamelCase			, lowerCamelCase   )
               for k, v in name.items():
                  assert isinstance(lowerCamelCase			, tf.Variable   )
            else:
               __a					  =     model.get_output_embeddings()
               assert x is None
               __a					  =     model.get_bias()
               assert name is None







@require_tf
class 				__SCREAMING_SNAKE_CASE							(				unittest.TestCase  ):




      @slow
      def 			__UpperCamelCase				(	self   )				->Any:



         '''simple docstring'''
         __a					  =     TFEsmForMaskedLM.from_pretrained('facebook/esm2_t6_8M_UR50D'   )

         __a					  =     tf.constant([[0, 1, 2, 3, 4, 5]]   )
         __a					  =     model(lowerCamelCase   )[0]
         __a					  =     [1, 6, 33]
         self.assertEqual(list(output.numpy().shape   )			, lowerCamelCase   )
         # compare the actual values for a slice.
         __a					  =     tf.constant(
             [
                 [
                     [8.921518, -10.589814, -6.4671307],
                     [-6.3967156, -13.911377, -1.1211915],
                     [-7.781247, -13.951557, -3.740592],
                 ]
             ]   )
         self.assertTrue(numpy.allclose(output[:, :3, :3].numpy()			, expected_slice.numpy()			, atol=1e-2   )   )






      @slow
      def 			__UpperCamelCase				(	self   )				->List[str]:



         '''simple docstring'''
         __a					  =     TFEsmModel.from_pretrained('facebook/esm2_t6_8M_UR50D'   )

         __a					  =     tf.constant([[0, 6, 4, 13, 5, 4, 16, 12, 11, 7, 2]]   )
         __a					  =     model(lowerCamelCase   )[0]
         # compare the actual values for a slice.
         __a					  =     tf.constant(
             [
                 [
                     [0.14443092, 0.54125327, 0.3247739],
                     [0.30340484, 0.00526676, 0.31077722],
                     [0.32278043, -0.24987096, 0.3414628],
                 ]
             ]   )
         self.assertTrue(numpy.allclose(output[:, :3, :3].numpy()			, expected_slice.numpy()			, atol=1e-4   )   )







'''simple docstring'''



from scipy.stats import pearsonr, spearmanr
from sklearn.metrics import fa_score, matthews_corrcoef

import datasets


__UpperCamelCase   :							Optional[int]         =   """\
@inproceedings{wang2019glue,
  title={{GLUE}: A Multi-Task Benchmark and Analysis Platform for Natural Language Understanding},
  author={Wang, Alex and Singh, Amanpreet and Michael, Julian and Hill, Felix and Levy, Omer and Bowman, Samuel R.},
  note={In the Proceedings of ICLR.},
  year={2019}
}
"""

__UpperCamelCase   :							Tuple         =   """\
GLUE, the General Language Understanding Evaluation benchmark
(https://gluebenchmark.com/) is a collection of resources for training,
evaluating, and analyzing natural language understanding systems.
"""

__UpperCamelCase   :							Optional[int]         =   """
Compute GLUE evaluation metric associated to each GLUE dataset.
Args:
    predictions: list of predictions to score.
        Each translation should be tokenized into a list of tokens.
    references: list of lists of references for each translation.
        Each reference should be tokenized into a list of tokens.
Returns: depending on the GLUE subset, one or several of:
    \"accuracy\": Accuracy
    \"f1\": F1 score
    \"pearson\": Pearson Correlation
    \"spearmanr\": Spearman Correlation
    \"matthews_correlation\": Matthew Correlation
Examples:

    >>> glue_metric = datasets.load_metric('glue', 'sst2')  # 'sst2' or any of [\"mnli\", \"mnli_mismatched\", \"mnli_matched\", \"qnli\", \"rte\", \"wnli\", \"hans\"]
    >>> references = [0, 1]
    >>> predictions = [0, 1]
    >>> results = glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {'accuracy': 1.0}

    >>> glue_metric = datasets.load_metric('glue', 'mrpc')  # 'mrpc' or 'qqp'
    >>> references = [0, 1]
    >>> predictions = [0, 1]
    >>> results = glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {'accuracy': 1.0, 'f1': 1.0}

    >>> glue_metric = datasets.load_metric('glue', 'stsb')
    >>> references = [0., 1., 2., 3., 4., 5.]
    >>> predictions = [0., 1., 2., 3., 4., 5.]
    >>> results = glue_metric.compute(predictions=predictions, references=references)
    >>> print({\"pearson\": round(results[\"pearson\"], 2), \"spearmanr\": round(results[\"spearmanr\"], 2)})
    {'pearson': 1.0, 'spearmanr': 1.0}

    >>> glue_metric = datasets.load_metric('glue', 'cola')
    >>> references = [0, 1]
    >>> predictions = [0, 1]
    >>> results = glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {'matthews_correlation': 1.0}
"""



def  __UpperCAmelCase						(					SCREAMING_SNAKE_CASE__:						Optional[int],       SCREAMING_SNAKE_CASE__:						str     )	->      List[Any]:

   """simple docstring"""
   return float((preds == labels).mean()     )



def  __UpperCAmelCase						(					SCREAMING_SNAKE_CASE__:						Optional[Any],       SCREAMING_SNAKE_CASE__:						List[str]     )	->      Optional[Any]:

   """simple docstring"""
   __a					  =     simple_accuracy(SCREAMING_SNAKE_CASE__,       SCREAMING_SNAKE_CASE__     )
   __a					  =     float(fa_score(y_true=SCREAMING_SNAKE_CASE__,       y_pred=SCREAMING_SNAKE_CASE__     )     )
   return {
       "accuracy": acc,
       "f1": fa,
   }



def  __UpperCAmelCase						(					SCREAMING_SNAKE_CASE__:						int,       SCREAMING_SNAKE_CASE__:						Optional[Any]     )	->      int:

   """simple docstring"""
   __a					  =     float(pearsonr(SCREAMING_SNAKE_CASE__,       SCREAMING_SNAKE_CASE__     )[0]     )
   __a					  =     float(spearmanr(SCREAMING_SNAKE_CASE__,       SCREAMING_SNAKE_CASE__     )[0]     )
   return {
       "pearson": pearson_corr,
       "spearmanr": spearman_corr,
   }



@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION   ,   _KWARGS_DESCRIPTION  )
class 				__SCREAMING_SNAKE_CASE							(				datasets.Metric  ):




      def 			__UpperCamelCase				(	self   )				->int:



         '''simple docstring'''
         if self.config_name not in [
             "sst2",
             "mnli",
             "mnli_mismatched",
             "mnli_matched",
             "cola",
             "stsb",
             "mrpc",
             "qqp",
             "qnli",
             "rte",
             "wnli",
             "hans",
         ]:
            raise KeyError(
                'You should supply a configuration name selected in '
                '["sst2", "mnli", "mnli_mismatched", "mnli_matched", '
                '"cola", "stsb", "mrpc", "qqp", "qnli", "rte", "wnli", "hans"]'   )
         return datasets.MetricInfo(
             description=_DESCRIPTION			, citation=_CITATION			, inputs_description=_KWARGS_DESCRIPTION			, features=datasets.Features(
                 {
                     'predictions': datasets.Value('int64' if self.config_name != 'stsb' else 'float32'   ),
                     'references': datasets.Value('int64' if self.config_name != 'stsb' else 'float32'   ),
                 }   )			, codebase_urls=[]			, reference_urls=[]			, format='numpy'			, )






      def 			__UpperCamelCase				(	self			, lowerCamelCase			, lowerCamelCase   )				->Tuple:



         '''simple docstring'''
         if self.config_name == "cola":
            return {"matthews_correlation": matthews_corrcoef(lowerCamelCase			, lowerCamelCase   )}
         elif self.config_name == "stsb":
            return pearson_and_spearman(lowerCamelCase			, lowerCamelCase   )
         elif self.config_name in ["mrpc", "qqp"]:
            return acc_and_fa(lowerCamelCase			, lowerCamelCase   )
         elif self.config_name in ["sst2", "mnli", "mnli_mismatched", "mnli_matched", "qnli", "rte", "wnli", "hans"]:
            return {"accuracy": simple_accuracy(lowerCamelCase			, lowerCamelCase   )}
         else:
            raise KeyError(
                'You should supply a configuration name selected in '
                '["sst2", "mnli", "mnli_mismatched", "mnli_matched", '
                '"cola", "stsb", "mrpc", "qqp", "qnli", "rte", "wnli", "hans"]'   )