




'''simple docstring'''





import unittest

from transformers import AutoTokenizer, NystromformerConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
   import torch

   from transformers import (
       NystromformerForMaskedLM,
       NystromformerForMultipleChoice,
       NystromformerForQuestionAnswering,
       NystromformerForSequenceClassification,
       NystromformerForTokenClassification,
       NystromformerModel,
   )
   from transformers.models.nystromformer.modeling_nystromformer import NYSTROMFORMER_PRETRAINED_MODEL_ARCHIVE_LIST

class   _a							:






     def __init__(					self   ,_SCREAMING_SNAKE_CASE   ,_SCREAMING_SNAKE_CASE=13   ,_SCREAMING_SNAKE_CASE=7   ,_SCREAMING_SNAKE_CASE=True   ,_SCREAMING_SNAKE_CASE=True   ,_SCREAMING_SNAKE_CASE=True   ,_SCREAMING_SNAKE_CASE=True   ,_SCREAMING_SNAKE_CASE=99   ,_SCREAMING_SNAKE_CASE=32   ,_SCREAMING_SNAKE_CASE=5   ,_SCREAMING_SNAKE_CASE=4   ,_SCREAMING_SNAKE_CASE=37   ,_SCREAMING_SNAKE_CASE="gelu"   ,_SCREAMING_SNAKE_CASE=0.1   ,_SCREAMING_SNAKE_CASE=0.1   ,_SCREAMING_SNAKE_CASE=512   ,_SCREAMING_SNAKE_CASE=16   ,_SCREAMING_SNAKE_CASE=2   ,_SCREAMING_SNAKE_CASE=0.0_2   ,_SCREAMING_SNAKE_CASE=3   ,_SCREAMING_SNAKE_CASE=4   ,_SCREAMING_SNAKE_CASE=None   ,)				->     Dict:
          _snake_case				= parent
          _snake_case				= batch_size
          _snake_case				= seq_length
          _snake_case				= is_training
          _snake_case				= use_input_mask
          _snake_case				= use_token_type_ids
          _snake_case				= use_labels
          _snake_case				= vocab_size
          _snake_case				= hidden_size
          _snake_case				= num_hidden_layers
          _snake_case				= num_attention_heads
          _snake_case				= intermediate_size
          _snake_case				= hidden_act
          _snake_case				= hidden_dropout_prob
          _snake_case				= attention_probs_dropout_prob
          _snake_case				= max_position_embeddings
          _snake_case				= type_vocab_size
          _snake_case				= type_sequence_label_size
          _snake_case				= initializer_range
          _snake_case				= num_labels
          _snake_case				= num_choices
          _snake_case				= scope






     def 							_lowercase      (					self       )				->     Union[str, Any]:
          _snake_case				= ids_tensor([self.batch_size, self.seq_length]   ,self.vocab_size       )

          _snake_case				= None
          if self.use_input_mask:
               _snake_case				= random_attention_mask([self.batch_size, self.seq_length]       )

          _snake_case				= None
          if self.use_token_type_ids:
               _snake_case				= ids_tensor([self.batch_size, self.seq_length]   ,self.type_vocab_size       )

          _snake_case				= None
          _snake_case				= None
          _snake_case				= None
          if self.use_labels:
               _snake_case				= ids_tensor([self.batch_size]   ,self.type_sequence_label_size       )
               _snake_case				= ids_tensor([self.batch_size, self.seq_length]   ,self.num_labels       )
               _snake_case				= ids_tensor([self.batch_size]   ,self.num_choices       )

          _snake_case				= self.get_config()

          return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels






     def 							_lowercase      (					self       )				->     int:
          return NystromformerConfig(
              vocab_size=self.vocab_size   ,hidden_size=self.hidden_size   ,num_hidden_layers=self.num_hidden_layers   ,num_attention_heads=self.num_attention_heads   ,intermediate_size=self.intermediate_size   ,hidden_act=self.hidden_act   ,hidden_dropout_prob=self.hidden_dropout_prob   ,attention_probs_dropout_prob=self.attention_probs_dropout_prob   ,max_position_embeddings=self.max_position_embeddings   ,type_vocab_size=self.type_vocab_size   ,is_decoder=_SCREAMING_SNAKE_CASE   ,initializer_range=self.initializer_range   ,)






     def 							_lowercase      (					self   ,_SCREAMING_SNAKE_CASE   ,_SCREAMING_SNAKE_CASE   ,_SCREAMING_SNAKE_CASE   ,_SCREAMING_SNAKE_CASE   ,_SCREAMING_SNAKE_CASE   ,_SCREAMING_SNAKE_CASE   ,_SCREAMING_SNAKE_CASE       )				->     Optional[Any]:
          _snake_case				= NystromformerModel(config=_SCREAMING_SNAKE_CASE       )
          model.to(_SCREAMING_SNAKE_CASE       )
          model.eval()
          _snake_case				= model(_SCREAMING_SNAKE_CASE   ,attention_mask=_SCREAMING_SNAKE_CASE   ,token_type_ids=_SCREAMING_SNAKE_CASE       )
          _snake_case				= model(_SCREAMING_SNAKE_CASE   ,token_type_ids=_SCREAMING_SNAKE_CASE       )
          _snake_case				= model(_SCREAMING_SNAKE_CASE       )
          self.parent.assertEqual(result.last_hidden_state.shape   ,(self.batch_size, self.seq_length, self.hidden_size)       )






     def 							_lowercase      (					self   ,_SCREAMING_SNAKE_CASE   ,_SCREAMING_SNAKE_CASE   ,_SCREAMING_SNAKE_CASE   ,_SCREAMING_SNAKE_CASE   ,_SCREAMING_SNAKE_CASE   ,_SCREAMING_SNAKE_CASE   ,_SCREAMING_SNAKE_CASE       )				->     Optional[Any]:
          _snake_case				= NystromformerForMaskedLM(config=_SCREAMING_SNAKE_CASE       )
          model.to(_SCREAMING_SNAKE_CASE       )
          model.eval()
          _snake_case				= model(_SCREAMING_SNAKE_CASE   ,attention_mask=_SCREAMING_SNAKE_CASE   ,token_type_ids=_SCREAMING_SNAKE_CASE   ,labels=_SCREAMING_SNAKE_CASE       )
          self.parent.assertEqual(result.logits.shape   ,(self.batch_size, self.seq_length, self.vocab_size)       )






     def 							_lowercase      (					self   ,_SCREAMING_SNAKE_CASE   ,_SCREAMING_SNAKE_CASE   ,_SCREAMING_SNAKE_CASE   ,_SCREAMING_SNAKE_CASE   ,_SCREAMING_SNAKE_CASE   ,_SCREAMING_SNAKE_CASE   ,_SCREAMING_SNAKE_CASE       )				->     Any:
          _snake_case				= NystromformerForQuestionAnswering(config=_SCREAMING_SNAKE_CASE       )
          model.to(_SCREAMING_SNAKE_CASE       )
          model.eval()
          _snake_case				= model(
              _SCREAMING_SNAKE_CASE   ,attention_mask=_SCREAMING_SNAKE_CASE   ,token_type_ids=_SCREAMING_SNAKE_CASE   ,start_positions=_SCREAMING_SNAKE_CASE   ,end_positions=_SCREAMING_SNAKE_CASE   ,)
          self.parent.assertEqual(result.start_logits.shape   ,(self.batch_size, self.seq_length)       )
          self.parent.assertEqual(result.end_logits.shape   ,(self.batch_size, self.seq_length)       )






     def 							_lowercase      (					self   ,_SCREAMING_SNAKE_CASE   ,_SCREAMING_SNAKE_CASE   ,_SCREAMING_SNAKE_CASE   ,_SCREAMING_SNAKE_CASE   ,_SCREAMING_SNAKE_CASE   ,_SCREAMING_SNAKE_CASE   ,_SCREAMING_SNAKE_CASE       )				->     Optional[int]:
          _snake_case				= self.num_labels
          _snake_case				= NystromformerForSequenceClassification(_SCREAMING_SNAKE_CASE       )
          model.to(_SCREAMING_SNAKE_CASE       )
          model.eval()
          _snake_case				= model(_SCREAMING_SNAKE_CASE   ,attention_mask=_SCREAMING_SNAKE_CASE   ,token_type_ids=_SCREAMING_SNAKE_CASE   ,labels=_SCREAMING_SNAKE_CASE       )
          self.parent.assertEqual(result.logits.shape   ,(self.batch_size, self.num_labels)       )






     def 							_lowercase      (					self   ,_SCREAMING_SNAKE_CASE   ,_SCREAMING_SNAKE_CASE   ,_SCREAMING_SNAKE_CASE   ,_SCREAMING_SNAKE_CASE   ,_SCREAMING_SNAKE_CASE   ,_SCREAMING_SNAKE_CASE   ,_SCREAMING_SNAKE_CASE       )				->     Optional[int]:
          _snake_case				= self.num_labels
          _snake_case				= NystromformerForTokenClassification(config=_SCREAMING_SNAKE_CASE       )
          model.to(_SCREAMING_SNAKE_CASE       )
          model.eval()
          _snake_case				= model(_SCREAMING_SNAKE_CASE   ,attention_mask=_SCREAMING_SNAKE_CASE   ,token_type_ids=_SCREAMING_SNAKE_CASE   ,labels=_SCREAMING_SNAKE_CASE       )
          self.parent.assertEqual(result.logits.shape   ,(self.batch_size, self.seq_length, self.num_labels)       )






     def 							_lowercase      (					self   ,_SCREAMING_SNAKE_CASE   ,_SCREAMING_SNAKE_CASE   ,_SCREAMING_SNAKE_CASE   ,_SCREAMING_SNAKE_CASE   ,_SCREAMING_SNAKE_CASE   ,_SCREAMING_SNAKE_CASE   ,_SCREAMING_SNAKE_CASE       )				->     List[Any]:
          _snake_case				= self.num_choices
          _snake_case				= NystromformerForMultipleChoice(config=_SCREAMING_SNAKE_CASE       )
          model.to(_SCREAMING_SNAKE_CASE       )
          model.eval()
          _snake_case				= input_ids.unsqueeze(1       ).expand(-1   ,self.num_choices   ,-1       ).contiguous()
          _snake_case				= token_type_ids.unsqueeze(1       ).expand(-1   ,self.num_choices   ,-1       ).contiguous()
          _snake_case				= input_mask.unsqueeze(1       ).expand(-1   ,self.num_choices   ,-1       ).contiguous()
          _snake_case				= model(
              _SCREAMING_SNAKE_CASE   ,attention_mask=_SCREAMING_SNAKE_CASE   ,token_type_ids=_SCREAMING_SNAKE_CASE   ,labels=_SCREAMING_SNAKE_CASE   ,)
          self.parent.assertEqual(result.logits.shape   ,(self.batch_size, self.num_choices)       )






     def 							_lowercase      (					self       )				->     Optional[int]:
          _snake_case				= self.prepare_config_and_inputs()
          (
              (
              _snake_case
          )				,   (
              _snake_case
          )				,   (
              _snake_case
          )				,   (
              _snake_case
          )				,   (
              _snake_case
          )				,   (
              _snake_case
          )				,   (
              _snake_case
          )				,   
          )				= config_and_inputs
          _snake_case				= {"input_ids": input_ids, "token_type_ids": token_type_ids, "attention_mask": input_mask}
          return config, inputs_dict







@require_torch
class   _a							(		__lowerCAmelCase   ,   __lowerCAmelCase   ,   unittest.TestCase				):
     SCREAMING_SNAKE_CASE_				:       Union[str, Any]       							=		(
         (
             NystromformerModel,
             NystromformerForMaskedLM,
             NystromformerForMultipleChoice,
             NystromformerForQuestionAnswering,
             NystromformerForSequenceClassification,
             NystromformerForTokenClassification,
         )
         if is_torch_available()
         else ()
     )
     SCREAMING_SNAKE_CASE_				:       List[str]       							=		(
         {
             """feature-extraction""": NystromformerModel,
             """fill-mask""": NystromformerForMaskedLM,
             """question-answering""": NystromformerForQuestionAnswering,
             """text-classification""": NystromformerForSequenceClassification,
             """token-classification""": NystromformerForTokenClassification,
             """zero-shot""": NystromformerForSequenceClassification,
         }
         if is_torch_available()
         else {}
     )
     SCREAMING_SNAKE_CASE_				:       Optional[Any]       							=		False
     SCREAMING_SNAKE_CASE_				:       Optional[int]       							=		False






     def 							_lowercase      (					self       )				->     str:
          _snake_case				= NystromformerModelTester(self       )
          _snake_case				= ConfigTester(self   ,config_class=_SCREAMING_SNAKE_CASE   ,hidden_size=37       )






     def 							_lowercase      (					self       )				->     Dict:
          self.config_tester.run_common_tests()






     def 							_lowercase      (					self       )				->     str:
          _snake_case				= self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_model(*_SCREAMING_SNAKE_CASE       )






     def 							_lowercase      (					self       )				->     Dict:
          _snake_case				= self.model_tester.prepare_config_and_inputs()
          for type in ["absolute", "relative_key", "relative_key_query"]:
               _snake_case				= type
               self.model_tester.create_and_check_model(*_SCREAMING_SNAKE_CASE       )






     def 							_lowercase      (					self       )				->     str:
          _snake_case				= self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_for_masked_lm(*_SCREAMING_SNAKE_CASE       )






     def 							_lowercase      (					self       )				->     int:
          _snake_case				= self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_for_multiple_choice(*_SCREAMING_SNAKE_CASE       )






     def 							_lowercase      (					self       )				->     Union[str, Any]:
          _snake_case				= self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_for_question_answering(*_SCREAMING_SNAKE_CASE       )






     def 							_lowercase      (					self       )				->     List[Any]:
          _snake_case				= self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_for_sequence_classification(*_SCREAMING_SNAKE_CASE       )






     def 							_lowercase      (					self       )				->     Dict:
          _snake_case				= self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_for_token_classification(*_SCREAMING_SNAKE_CASE       )






     @slow
     def 							_lowercase      (					self       )				->     Optional[Any]:
          for model_name in NYSTROMFORMER_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
               _snake_case				= NystromformerModel.from_pretrained(_SCREAMING_SNAKE_CASE       )
               self.assertIsNotNone(_SCREAMING_SNAKE_CASE       )







@require_torch
class   _a							(		unittest.TestCase				):






     @slow
     def 							_lowercase      (					self       )				->     Optional[Any]:
          _snake_case				= NystromformerModel.from_pretrained("uw-madison/nystromformer-512"       )
          _snake_case				= torch.tensor([[0, 1, 2, 3, 4, 5]]       )

          with torch.no_grad():
               _snake_case				= model(_SCREAMING_SNAKE_CASE       )[0]

          _snake_case				= torch.Size((1, 6, 768)       )
          self.assertEqual(output.shape   ,_SCREAMING_SNAKE_CASE       )

          _snake_case				= torch.tensor(
              [[[-0.4_5_3_2, -0.0_9_3_6, 0.5_1_3_7], [-0.2_6_7_6, 0.0_6_2_8, 0.6_1_8_6], [-0.3_6_2_9, -0.1_7_2_6, 0.4_7_1_6]]]       )

          self.assertTrue(torch.allclose(output[:, :3, :3]   ,_SCREAMING_SNAKE_CASE   ,atol=1e-4       )       )






     @slow
     def 							_lowercase      (					self       )				->     str:
          _snake_case				= "the [MASK] of Belgium is Brussels"

          _snake_case				= AutoTokenizer.from_pretrained("uw-madison/nystromformer-512"       )
          _snake_case				= NystromformerForMaskedLM.from_pretrained("uw-madison/nystromformer-512"       )

          _snake_case				= tokenizer(_SCREAMING_SNAKE_CASE   ,return_tensors="pt"       )

          with torch.no_grad():
               _snake_case				= model(encoding.input_ids       ).logits

          _snake_case				= token_logits[:, 2, :].argmax(-1       )[0]

          self.assertEqual(tokenizer.decode(_SCREAMING_SNAKE_CASE       )   ,"capital"       )








'''simple docstring'''





from sklearn.metrics import fa_score, matthews_corrcoef

import datasets

from .record_evaluation import evaluate as evaluate_record


UpperCamelCase_		:     Tuple          =		'''\
@article{wang2019superglue,
  title={SuperGLUE: A Stickier Benchmark for General-Purpose Language Understanding Systems},
  author={Wang, Alex and Pruksachatkun, Yada and Nangia, Nikita and Singh, Amanpreet and Michael, Julian and Hill, Felix and Levy, Omer and Bowman, Samuel R},
  journal={arXiv preprint arXiv:1905.00537},
  year={2019}
}
'''

UpperCamelCase_		:     Optional[Any]          =		'''\
SuperGLUE (https://super.gluebenchmark.com/) is a new benchmark styled after
GLUE with a new set of more difficult language understanding tasks, improved
resources, and a new public leaderboard.
'''

UpperCamelCase_		:     Dict          =		'''
Compute SuperGLUE evaluation metric associated to each SuperGLUE dataset.
Args:
    predictions: list of predictions to score. Depending on the SuperGlUE subset:
        - for \'record\': list of question-answer dictionaries with the following keys:
            - \'idx\': index of the question as specified by the dataset
            - \'prediction_text\': the predicted answer text
        - for \'multirc\': list of question-answer dictionaries with the following keys:
            - \'idx\': index of the question-answer pair as specified by the dataset
            - \'prediction\': the predicted answer label
        - otherwise: list of predicted labels
    references: list of reference labels. Depending on the SuperGLUE subset:
        - for \'record\': list of question-answers dictionaries with the following keys:
            - \'idx\': index of the question as specified by the dataset
            - \'answers\': list of possible answers
        - otherwise: list of reference labels
Returns: depending on the SuperGLUE subset:
    - for \'record\':
        - \'exact_match\': Exact match between answer and gold answer
        - \'f1\': F1 score
    - for \'multirc\':
        - \'exact_match\': Exact match between answer and gold answer
        - \'f1_m\': Per-question macro-F1 score
        - \'f1_a\': Average F1 score over all answers
    - for \'axb\':
        \'matthews_correlation\': Matthew Correlation
    - for \'cb\':
        - \'accuracy\': Accuracy
        - \'f1\': F1 score
    - for all others:
        - \'accuracy\': Accuracy
Examples:

    >>> super_glue_metric = datasets.load_metric(\'super_glue\', \'copa\')  # any of ["copa", "rte", "wic", "wsc", "wsc.fixed", "boolq", "axg"]
    >>> predictions = [0, 1]
    >>> references = [0, 1]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {\'accuracy\': 1.0}

    >>> super_glue_metric = datasets.load_metric(\'super_glue\', \'cb\')
    >>> predictions = [0, 1]
    >>> references = [0, 1]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {\'accuracy\': 1.0, \'f1\': 1.0}

    >>> super_glue_metric = datasets.load_metric(\'super_glue\', \'record\')
    >>> predictions = [{\'idx\': {\'passage\': 0, \'query\': 0}, \'prediction_text\': \'answer\'}]
    >>> references = [{\'idx\': {\'passage\': 0, \'query\': 0}, \'answers\': [\'answer\', \'another_answer\']}]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {\'exact_match\': 1.0, \'f1\': 1.0}

    >>> super_glue_metric = datasets.load_metric(\'super_glue\', \'multirc\')
    >>> predictions = [{\'idx\': {\'answer\': 0, \'paragraph\': 0, \'question\': 0}, \'prediction\': 0}, {\'idx\': {\'answer\': 1, \'paragraph\': 2, \'question\': 3}, \'prediction\': 1}]
    >>> references = [0, 1]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {\'exact_match\': 1.0, \'f1_m\': 1.0, \'f1_a\': 1.0}

    >>> super_glue_metric = datasets.load_metric(\'super_glue\', \'axb\')
    >>> references = [0, 1]
    >>> predictions = [0, 1]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {\'matthews_correlation\': 1.0}
'''





def       __a      (  _UpperCamelCase:       Tuple   ,		_UpperCamelCase:       int       )		->					Optional[Any]:

     """simple docstring"""




     return float((preds == labels).mean()       )





def       __a      (  _UpperCamelCase:       Union[str, Any]   ,		_UpperCamelCase:       Optional[int]   ,		_UpperCamelCase:       int="binary"       )		->					Any:

     """simple docstring"""




     _snake_case				= simple_accuracy(_UpperCamelCase   ,		_UpperCamelCase       )
     _snake_case				= float(fa_score(y_true=_UpperCamelCase   ,		y_pred=_UpperCamelCase   ,		average=_UpperCamelCase       )       )
     return {
         "accuracy": acc,
         "f1": fa,
     }





def       __a      (  _UpperCamelCase:       str   ,		_UpperCamelCase:       Any       )		->					Tuple:

     """simple docstring"""




     _snake_case				= {}
     for id_pred, label in zip(_UpperCamelCase   ,		_UpperCamelCase       ):
          _snake_case				= F"""{id_pred['idx']['paragraph']}-{id_pred['idx']['question']}"""
          _snake_case				= id_pred["prediction"]
          if question_id in question_map:
               question_map[question_id].append((pred, label)       )
          else:
               _snake_case				= [(pred, label)]
     _snake_case				,   _snake_case				= [], []
     for question, preds_labels in question_map.items():
          _snake_case				,   _snake_case				= zip(*_UpperCamelCase       )
          _snake_case				= fa_score(y_true=_UpperCamelCase   ,		y_pred=_UpperCamelCase   ,		average="macro"       )
          fas.append(_UpperCamelCase       )
          _snake_case				= int(sum(pred == label for pred, label in preds_labels       ) == len(_UpperCamelCase       )       )
          ems.append(_UpperCamelCase       )
     _snake_case				= float(sum(_UpperCamelCase       ) / len(_UpperCamelCase       )       )
     _snake_case				= sum(_UpperCamelCase       ) / len(_UpperCamelCase       )
     _snake_case				= float(fa_score(y_true=_UpperCamelCase   ,		y_pred=[id_pred["prediction"] for id_pred in ids_preds]       )       )
     return {"exact_match": em, "f1_m": fa_m, "f1_a": fa_a}





@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION   ,   _KWARGS_DESCRIPTION				)
class   _a							(		datasets.Metric				):






     def 							_lowercase      (					self       )				->     Union[str, Any]:
          if self.config_name not in [
              "boolq",
              "cb",
              "copa",
              "multirc",
              "record",
              "rte",
              "wic",
              "wsc",
              "wsc.fixed",
              "axb",
              "axg",
          ]:
               raise KeyError(
                   "You should supply a configuration name selected in "
                   "[\"boolq\", \"cb\", \"copa\", \"multirc\", \"record\", \"rte\", \"wic\", \"wsc\", \"wsc.fixed\", \"axb\", \"axg\",]"       )
          return datasets.MetricInfo(
              description=_DESCRIPTION   ,citation=_CITATION   ,inputs_description=_KWARGS_DESCRIPTION   ,features=datasets.Features(self._get_feature_types()       )   ,codebase_urls=[]   ,reference_urls=[]   ,format="numpy" if not self.config_name == "record" and not self.config_name == "multirc" else None   ,)






     def 							_lowercase      (					self       )				->     Union[str, Any]:
          if self.config_name == "record":
               return {
                   "predictions": {
                       "idx": {
                           "passage": datasets.Value("int64"       ),
                           "query": datasets.Value("int64"       ),
                       },
                       "prediction_text": datasets.Value("string"       ),
                   },
                   "references": {
                       "idx": {
                           "passage": datasets.Value("int64"       ),
                           "query": datasets.Value("int64"       ),
                       },
                       "answers": datasets.Sequence(datasets.Value("string"       )       ),
                   },
               }
          elif self.config_name == "multirc":
               return {
                   "predictions": {
                       "idx": {
                           "answer": datasets.Value("int64"       ),
                           "paragraph": datasets.Value("int64"       ),
                           "question": datasets.Value("int64"       ),
                       },
                       "prediction": datasets.Value("int64"       ),
                   },
                   "references": datasets.Value("int64"       ),
               }
          else:
               return {
                   "predictions": datasets.Value("int64"       ),
                   "references": datasets.Value("int64"       ),
               }






     def 							_lowercase      (					self   ,_SCREAMING_SNAKE_CASE   ,_SCREAMING_SNAKE_CASE       )				->     Optional[Any]:
          if self.config_name == "axb":
               return {"matthews_correlation": matthews_corrcoef(_SCREAMING_SNAKE_CASE   ,_SCREAMING_SNAKE_CASE       )}
          elif self.config_name == "cb":
               return acc_and_fa(_SCREAMING_SNAKE_CASE   ,_SCREAMING_SNAKE_CASE   ,fa_avg="macro"       )
          elif self.config_name == "record":
               _snake_case				= [
                   {
                       "qas": [
                           {"id": ref["idx"]["query"], "answers": [{"text": ans} for ans in ref["answers"]]}
                           for ref in references
                       ]
                   }
               ]
               _snake_case				= {pred["idx"]["query"]: pred["prediction_text"] for pred in predictions}
               return evaluate_record(_SCREAMING_SNAKE_CASE   ,_SCREAMING_SNAKE_CASE       )[0]
          elif self.config_name == "multirc":
               return evaluate_multirc(_SCREAMING_SNAKE_CASE   ,_SCREAMING_SNAKE_CASE       )
          elif self.config_name in ["copa", "rte", "wic", "wsc", "wsc.fixed", "boolq", "axg"]:
               return {"accuracy": simple_accuracy(_SCREAMING_SNAKE_CASE   ,_SCREAMING_SNAKE_CASE       )}
          else:
               raise KeyError(
                   "You should supply a configuration name selected in "
                   "[\"boolq\", \"cb\", \"copa\", \"multirc\", \"record\", \"rte\", \"wic\", \"wsc\", \"wsc.fixed\", \"axb\", \"axg\",]"       )


