
"""simple docstring"""



from scipy.stats import pearsonr, spearmanr
from sklearn.metrics import fa_score, matthews_corrcoef

import datasets


UpperCAmelCase__        =					'\\n@inproceedings{wang2019glue,\n  title={{GLUE}: A Multi-Task Benchmark and Analysis Platform for Natural Language Understanding},\n  author={Wang, Alex and Singh, Amanpreet and Michael, Julian and Hill, Felix and Levy, Omer and Bowman, Samuel R.},\n  note={In the Proceedings of ICLR.},\n  year={2019}\n}\n'

UpperCAmelCase__        =					'\\nGLUE, the General Language Understanding Evaluation benchmark\n(https://gluebenchmark.com/) is a collection of resources for training,\nevaluating, and analyzing natural language understanding systems.\n'

UpperCAmelCase__        =					'\nCompute GLUE evaluation metric associated to each GLUE dataset.\nArgs:\n    predictions: list of predictions to score.\n        Each translation should be tokenized into a list of tokens.\n    references: list of lists of references for each translation.\n        Each reference should be tokenized into a list of tokens.\nReturns: depending on the GLUE subset, one or several of:\n    "accuracy": Accuracy\n    "f1": F1 score\n    "pearson": Pearson Correlation\n    "spearmanr": Spearman Correlation\n    "matthews_correlation": Matthew Correlation\nExamples:\n\n    >>> glue_metric = datasets.load_metric(\'glue\', \'sst2\')  # \'sst2\' or any of ["mnli", "mnli_mismatched", "mnli_matched", "qnli", "rte", "wnli", "hans"]\n    >>> references = [0, 1]\n    >>> predictions = [0, 1]\n    >>> results = glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {\'accuracy\': 1.0}\n\n    >>> glue_metric = datasets.load_metric(\'glue\', \'mrpc\')  # \'mrpc\' or \'qqp\'\n    >>> references = [0, 1]\n    >>> predictions = [0, 1]\n    >>> results = glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {\'accuracy\': 1.0, \'f1\': 1.0}\n\n    >>> glue_metric = datasets.load_metric(\'glue\', \'stsb\')\n    >>> references = [0., 1., 2., 3., 4., 5.]\n    >>> predictions = [0., 1., 2., 3., 4., 5.]\n    >>> results = glue_metric.compute(predictions=predictions, references=references)\n    >>> print({"pearson": round(results["pearson"], 2), "spearmanr": round(results["spearmanr"], 2)})\n    {\'pearson\': 1.0, \'spearmanr\': 1.0}\n\n    >>> glue_metric = datasets.load_metric(\'glue\', \'cola\')\n    >>> references = [0, 1]\n    >>> predictions = [0, 1]\n    >>> results = glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {\'matthews_correlation\': 1.0}\n'
def    _UpperCAmelCase (					__lowerCamelCase  :			Union[str, Any]					,   __lowerCamelCase  :			str						)	->   Any:
  return float((preds == labels).mean()						)
def    _UpperCAmelCase (					__lowerCamelCase  :			int					,   __lowerCamelCase  :			Union[str, Any]						)	->   Any:
  _snake_case   	=			simple_accuracy(__UpperCAmelCase					,   __UpperCAmelCase						)
  _snake_case   	=			float(fa_score(y_true=__UpperCAmelCase					,   y_pred=__UpperCAmelCase						)						)
  return {
      "accuracy": acc,
      "f1": fa,
  }
def    _UpperCAmelCase (					__lowerCamelCase  :			int					,   __lowerCamelCase  :			Tuple						)	->   Optional[int]:
  _snake_case   	=			float(pearsonr(__UpperCAmelCase					,   __UpperCAmelCase						)[0]						)
  _snake_case   	=			float(spearmanr(__UpperCAmelCase					,   __UpperCAmelCase						)[0]						)
  return {
      "pearson": pearson_corr,
      "spearmanr": spearman_corr,
  }

@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION					,      _KWARGS_DESCRIPTION      )
class  lowerCAmelCase__							(		datasets.Metric      ):







     def 	lowercase      (				self     :     List[Any]     ):
       if self.config_name not in [
           "sst2",
           "mnli",
           "mnli_mismatched",
           "mnli_matched",
           "cola",
           "stsb",
           "mrpc",
           "qqp",
           "qnli",
           "rte",
           "wnli",
           "hans",
       ]:
         raise KeyError(
             '''You should supply a configuration name selected in '''
             '''["sst2", "mnli", "mnli_mismatched", "mnli_matched", '''
             '''"cola", "stsb", "mrpc", "qqp", "qnli", "rte", "wnli", "hans"]'''     )
       return datasets.MetricInfo(
           description=_DESCRIPTION       , citation=_CITATION       , inputs_description=_KWARGS_DESCRIPTION       , features=datasets.Features(
               {
                   '''predictions''': datasets.Value('''int64''' if self.config_name != '''stsb''' else '''float32'''     ),
                   '''references''': datasets.Value('''int64''' if self.config_name != '''stsb''' else '''float32'''     ),
               }     )       , codebase_urls=[]       , reference_urls=[]       , format='''numpy'''       , )







     def 	lowercase      (				self     :     Optional[Any]       , _lowerCamelCase     :     str       , _lowerCamelCase     :     List[str]     ):
       if self.config_name == "cola":
         return {"matthews_correlation": matthews_corrcoef(_lowerCamelCase       , _lowerCamelCase     )}
       elif self.config_name == "stsb":
         return pearson_and_spearman(_lowerCamelCase       , _lowerCamelCase     )
       elif self.config_name in ["mrpc", "qqp"]:
         return acc_and_fa(_lowerCamelCase       , _lowerCamelCase     )
       elif self.config_name in ["sst2", "mnli", "mnli_mismatched", "mnli_matched", "qnli", "rte", "wnli", "hans"]:
         return {"accuracy": simple_accuracy(_lowerCamelCase       , _lowerCamelCase     )}
       else:
         raise KeyError(
             '''You should supply a configuration name selected in '''
             '''["sst2", "mnli", "mnli_mismatched", "mnli_matched", '''
             '''"cola", "stsb", "mrpc", "qqp", "qnli", "rte", "wnli", "hans"]'''     )








import unittest

from transformers import AlbertConfig, is_torch_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
     import torch

     from transformers import (
         MODEL_FOR_PRETRAINING_MAPPING,
         AlbertForMaskedLM,
         AlbertForMultipleChoice,
         AlbertForPreTraining,
         AlbertForQuestionAnswering,
         AlbertForSequenceClassification,
         AlbertForTokenClassification,
         AlbertModel,
     )
     from transformers.models.albert.modeling_albert import ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST
class    __UpperCamelCase			:

 """simple docstring"""



 def __init__(	self		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE=1_3		,				SCREAMING_SNAKE_CASE=7		,				SCREAMING_SNAKE_CASE=True		,				SCREAMING_SNAKE_CASE=True		,				SCREAMING_SNAKE_CASE=True		,				SCREAMING_SNAKE_CASE=True		,				SCREAMING_SNAKE_CASE=9_9		,				SCREAMING_SNAKE_CASE=1_6		,				SCREAMING_SNAKE_CASE=3_6		,				SCREAMING_SNAKE_CASE=6		,				SCREAMING_SNAKE_CASE=6		,				SCREAMING_SNAKE_CASE=6		,				SCREAMING_SNAKE_CASE=3_7		,				SCREAMING_SNAKE_CASE="gelu"		,				SCREAMING_SNAKE_CASE=0.1		,				SCREAMING_SNAKE_CASE=0.1		,				SCREAMING_SNAKE_CASE=5_1_2		,				SCREAMING_SNAKE_CASE=1_6		,				SCREAMING_SNAKE_CASE=2		,				SCREAMING_SNAKE_CASE=0.02		,				SCREAMING_SNAKE_CASE=3		,				SCREAMING_SNAKE_CASE=4		,				SCREAMING_SNAKE_CASE=None		,				) ->			Union[str, Any]:
  a__												= parent
  a__												= batch_size
  a__												= seq_length
  a__												= is_training
  a__												= use_input_mask
  a__												= use_token_type_ids
  a__												= use_labels
  a__												= vocab_size
  a__												= embedding_size
  a__												= hidden_size
  a__												= num_hidden_layers
  a__												= num_hidden_groups
  a__												= num_attention_heads
  a__												= intermediate_size
  a__												= hidden_act
  a__												= hidden_dropout_prob
  a__												= attention_probs_dropout_prob
  a__												= max_position_embeddings
  a__												= type_vocab_size
  a__												= type_sequence_label_size
  a__												= initializer_range
  a__												= num_labels
  a__												= num_choices
  a__												= scope

 def 						_UpperCAmelCase       (	self      ) ->			List[str]:
  a__												= ids_tensor([self.batch_size, self.seq_length]		,				self.vocab_size      )

  a__												= None
  if self.use_input_mask:
   a__												= random_attention_mask([self.batch_size, self.seq_length]      )

  a__												= None
  if self.use_token_type_ids:
   a__												= ids_tensor([self.batch_size, self.seq_length]		,				self.type_vocab_size      )

  a__												= None
  a__												= None
  a__												= None
  if self.use_labels:
   a__												= ids_tensor([self.batch_size]		,				self.type_sequence_label_size      )
   a__												= ids_tensor([self.batch_size, self.seq_length]		,				self.num_labels      )
   a__												= ids_tensor([self.batch_size]		,				self.num_choices      )

  a__												= self.get_config()

  return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels

 def 						_UpperCAmelCase       (	self      ) ->			Any:
  return AlbertConfig(
      vocab_size=self.vocab_size		,				hidden_size=self.hidden_size		,				num_hidden_layers=self.num_hidden_layers		,				num_attention_heads=self.num_attention_heads		,				intermediate_size=self.intermediate_size		,				hidden_act=self.hidden_act		,				hidden_dropout_prob=self.hidden_dropout_prob		,				attention_probs_dropout_prob=self.attention_probs_dropout_prob		,				max_position_embeddings=self.max_position_embeddings		,				type_vocab_size=self.type_vocab_size		,				initializer_range=self.initializer_range		,				num_hidden_groups=self.num_hidden_groups		,				)

 def 						_UpperCAmelCase       (	self		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE      ) ->			Optional[int]:
  a__												= AlbertModel(config=SCREAMING_SNAKE_CASE      )
  model.to(SCREAMING_SNAKE_CASE      )
  model.eval()
  a__												= model(SCREAMING_SNAKE_CASE		,				attention_mask=SCREAMING_SNAKE_CASE		,				token_type_ids=SCREAMING_SNAKE_CASE      )
  a__												= model(SCREAMING_SNAKE_CASE		,				token_type_ids=SCREAMING_SNAKE_CASE      )
  a__												= model(SCREAMING_SNAKE_CASE      )
  self.parent.assertEqual(result.last_hidden_state.shape		,				(self.batch_size, self.seq_length, self.hidden_size)      )
  self.parent.assertEqual(result.pooler_output.shape		,				(self.batch_size, self.hidden_size)      )

 def 						_UpperCAmelCase       (	self		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE      ) ->			List[Any]:
  a__												= AlbertForPreTraining(config=SCREAMING_SNAKE_CASE      )
  model.to(SCREAMING_SNAKE_CASE      )
  model.eval()
  a__												= model(
      SCREAMING_SNAKE_CASE		,				attention_mask=SCREAMING_SNAKE_CASE		,				token_type_ids=SCREAMING_SNAKE_CASE		,				labels=SCREAMING_SNAKE_CASE		,				sentence_order_label=SCREAMING_SNAKE_CASE		,				)
  self.parent.assertEqual(result.prediction_logits.shape		,				(self.batch_size, self.seq_length, self.vocab_size)      )
  self.parent.assertEqual(result.sop_logits.shape		,				(self.batch_size, config.num_labels)      )

 def 						_UpperCAmelCase       (	self		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE      ) ->			Dict:
  a__												= AlbertForMaskedLM(config=SCREAMING_SNAKE_CASE      )
  model.to(SCREAMING_SNAKE_CASE      )
  model.eval()
  a__												= model(SCREAMING_SNAKE_CASE		,				attention_mask=SCREAMING_SNAKE_CASE		,				token_type_ids=SCREAMING_SNAKE_CASE		,				labels=SCREAMING_SNAKE_CASE      )
  self.parent.assertEqual(result.logits.shape		,				(self.batch_size, self.seq_length, self.vocab_size)      )

 def 						_UpperCAmelCase       (	self		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE      ) ->			Any:
  a__												= AlbertForQuestionAnswering(config=SCREAMING_SNAKE_CASE      )
  model.to(SCREAMING_SNAKE_CASE      )
  model.eval()
  a__												= model(
      SCREAMING_SNAKE_CASE		,				attention_mask=SCREAMING_SNAKE_CASE		,				token_type_ids=SCREAMING_SNAKE_CASE		,				start_positions=SCREAMING_SNAKE_CASE		,				end_positions=SCREAMING_SNAKE_CASE		,				)
  self.parent.assertEqual(result.start_logits.shape		,				(self.batch_size, self.seq_length)      )
  self.parent.assertEqual(result.end_logits.shape		,				(self.batch_size, self.seq_length)      )

 def 						_UpperCAmelCase       (	self		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE      ) ->			Union[str, Any]:
  a__												= self.num_labels
  a__												= AlbertForSequenceClassification(SCREAMING_SNAKE_CASE      )
  model.to(SCREAMING_SNAKE_CASE      )
  model.eval()
  a__												= model(SCREAMING_SNAKE_CASE		,				attention_mask=SCREAMING_SNAKE_CASE		,				token_type_ids=SCREAMING_SNAKE_CASE		,				labels=SCREAMING_SNAKE_CASE      )
  self.parent.assertEqual(result.logits.shape		,				(self.batch_size, self.num_labels)      )

 def 						_UpperCAmelCase       (	self		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE      ) ->			Tuple:
  a__												= self.num_labels
  a__												= AlbertForTokenClassification(config=SCREAMING_SNAKE_CASE      )
  model.to(SCREAMING_SNAKE_CASE      )
  model.eval()
  a__												= model(SCREAMING_SNAKE_CASE		,				attention_mask=SCREAMING_SNAKE_CASE		,				token_type_ids=SCREAMING_SNAKE_CASE		,				labels=SCREAMING_SNAKE_CASE      )
  self.parent.assertEqual(result.logits.shape		,				(self.batch_size, self.seq_length, self.num_labels)      )

 def 						_UpperCAmelCase       (	self		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE      ) ->			Optional[int]:
  a__												= self.num_choices
  a__												= AlbertForMultipleChoice(config=SCREAMING_SNAKE_CASE      )
  model.to(SCREAMING_SNAKE_CASE      )
  model.eval()
  a__												= input_ids.unsqueeze(1      ).expand(-1		,				self.num_choices		,				-1      ).contiguous()
  a__												= token_type_ids.unsqueeze(1      ).expand(-1		,				self.num_choices		,				-1      ).contiguous()
  a__												= input_mask.unsqueeze(1      ).expand(-1		,				self.num_choices		,				-1      ).contiguous()
  a__												= model(
      SCREAMING_SNAKE_CASE		,				attention_mask=SCREAMING_SNAKE_CASE		,				token_type_ids=SCREAMING_SNAKE_CASE		,				labels=SCREAMING_SNAKE_CASE		,				)
  self.parent.assertEqual(result.logits.shape		,				(self.batch_size, self.num_choices)      )



 def 						_UpperCAmelCase       (	self      ) ->			Tuple:
  a__												= self.prepare_config_and_inputs()
  (
      (
      a__
  )							,			(
      a__
  )							,			(
      a__
  )							,			(
      a__
  )							,			(
      a__
  )							,			(
      a__
  )							,			(
      a__
  )							,			
  )												= config_and_inputs
  a__												= {'''input_ids''': input_ids, '''token_type_ids''': token_type_ids, '''attention_mask''': input_mask}
  return config, inputs_dict





@require_torch
class    __UpperCamelCase			(       _lowercase	,	_lowercase	,	unittest.TestCase				):

 """simple docstring"""



 _lowercase   :    List[str]      						=					(
     (
         AlbertModel,
         AlbertForPreTraining,
         AlbertForMaskedLM,
         AlbertForMultipleChoice,
         AlbertForSequenceClassification,
         AlbertForTokenClassification,
         AlbertForQuestionAnswering,
     )
     if is_torch_available()
     else ()
 )
 _lowercase   :    List[str]      						=					(
     {
         '''feature-extraction''': AlbertModel,
         '''fill-mask''': AlbertForMaskedLM,
         '''question-answering''': AlbertForQuestionAnswering,
         '''text-classification''': AlbertForSequenceClassification,
         '''token-classification''': AlbertForTokenClassification,
         '''zero-shot''': AlbertForSequenceClassification,
     }
     if is_torch_available()
     else {}
 )
 _lowercase   :    List[str]      						=					True

 def 						_UpperCAmelCase       (	self		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE=False      ) ->			List[str]:
  a__												= super()._prepare_for_class(SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE		,				return_labels=SCREAMING_SNAKE_CASE      )

  if return_labels:
   if model_class in get_values(SCREAMING_SNAKE_CASE      ):
    a__												= torch.zeros(
        (self.model_tester.batch_size, self.model_tester.seq_length)		,				dtype=torch.long		,				device=SCREAMING_SNAKE_CASE      )
    a__												= torch.zeros(
        self.model_tester.batch_size		,				dtype=torch.long		,				device=SCREAMING_SNAKE_CASE      )
  return inputs_dict

 def 						_UpperCAmelCase       (	self      ) ->			Dict:
  a__												= AlbertModelTester(self      )
  a__												= ConfigTester(self		,				config_class=SCREAMING_SNAKE_CASE		,				hidden_size=3_7      )

 def 						_UpperCAmelCase       (	self      ) ->			Union[str, Any]:
  self.config_tester.run_common_tests()

 def 						_UpperCAmelCase       (	self      ) ->			Union[str, Any]:
  a__												= self.model_tester.prepare_config_and_inputs()
  self.model_tester.create_and_check_model(*SCREAMING_SNAKE_CASE      )

 def 						_UpperCAmelCase       (	self      ) ->			Union[str, Any]:
  a__												= self.model_tester.prepare_config_and_inputs()
  self.model_tester.create_and_check_for_pretraining(*SCREAMING_SNAKE_CASE      )

 def 						_UpperCAmelCase       (	self      ) ->			Any:
  a__												= self.model_tester.prepare_config_and_inputs()
  self.model_tester.create_and_check_for_masked_lm(*SCREAMING_SNAKE_CASE      )

 def 						_UpperCAmelCase       (	self      ) ->			List[Any]:
  a__												= self.model_tester.prepare_config_and_inputs()
  self.model_tester.create_and_check_for_multiple_choice(*SCREAMING_SNAKE_CASE      )

 def 						_UpperCAmelCase       (	self      ) ->			Union[str, Any]:
  a__												= self.model_tester.prepare_config_and_inputs()
  self.model_tester.create_and_check_for_question_answering(*SCREAMING_SNAKE_CASE      )

 def 						_UpperCAmelCase       (	self      ) ->			Optional[Any]:
  a__												= self.model_tester.prepare_config_and_inputs()
  self.model_tester.create_and_check_for_sequence_classification(*SCREAMING_SNAKE_CASE      )

 def 						_UpperCAmelCase       (	self      ) ->			List[Any]:
  a__												= self.model_tester.prepare_config_and_inputs()
  for type in ["absolute", "relative_key", "relative_key_query"]:
   a__												= type
   self.model_tester.create_and_check_model(*SCREAMING_SNAKE_CASE      )



 @slow
 def 						_UpperCAmelCase       (	self      ) ->			List[str]:
  for model_name in ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
   a__												= AlbertModel.from_pretrained(SCREAMING_SNAKE_CASE      )
   self.assertIsNotNone(SCREAMING_SNAKE_CASE      )





@require_torch
class    __UpperCamelCase			(       unittest.TestCase				):

 """simple docstring"""



 @slow
 def 						_UpperCAmelCase       (	self      ) ->			Optional[Any]:
  a__												= AlbertModel.from_pretrained('''albert-base-v2'''      )
  a__												= torch.tensor([[0, 3_4_5, 2_3_2, 3_2_8, 7_4_0, 1_4_0, 1_6_9_5, 6_9, 6_0_7_8, 1_5_8_8, 2]]      )
  a__												= torch.tensor([[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]      )
  with torch.no_grad():
   a__												= model(SCREAMING_SNAKE_CASE		,				attention_mask=SCREAMING_SNAKE_CASE      )[0]
  a__												= torch.Size((1, 1_1, 7_6_8)      )
  self.assertEqual(output.shape		,				SCREAMING_SNAKE_CASE      )
  a__												= torch.tensor(
      [[[-0.65_13, 1.50_35, -0.27_66], [-0.65_15, 1.50_46, -0.27_80], [-0.65_12, 1.50_49, -0.27_84]]]      )

  self.assertTrue(torch.allclose(output[:, 1:4, 1:4]		,				SCREAMING_SNAKE_CASE		,				atol=1e-4      )      )





