





"""simple docstring"""





from sklearn.metrics import fa_score, matthews_corrcoef

import datasets

from .record_evaluation import evaluate as evaluate_record


lowercase__				:List[str]				   =       '\\n@article{wang2019superglue,\n  title={SuperGLUE: A Stickier Benchmark for General-Purpose Language Understanding Systems},\n  author={Wang, Alex and Pruksachatkun, Yada and Nangia, Nikita and Singh, Amanpreet and Michael, Julian and Hill, Felix and Levy, Omer and Bowman, Samuel R},\n  journal={arXiv preprint arXiv:1905.00537},\n  year={2019}\n}\n'

lowercase__				:int				   =       '\\nSuperGLUE (https://super.gluebenchmark.com/) is a new benchmark styled after\nGLUE with a new set of more difficult language understanding tasks, improved\nresources, and a new public leaderboard.\n'

lowercase__				:Optional[int]				   =       '\nCompute SuperGLUE evaluation metric associated to each SuperGLUE dataset.\nArgs:\n    predictions: list of predictions to score. Depending on the SuperGlUE subset:\n        - for \'record\': list of question-answer dictionaries with the following keys:\n            - \'idx\': index of the question as specified by the dataset\n            - \'prediction_text\': the predicted answer text\n        - for \'multirc\': list of question-answer dictionaries with the following keys:\n            - \'idx\': index of the question-answer pair as specified by the dataset\n            - \'prediction\': the predicted answer label\n        - otherwise: list of predicted labels\n    references: list of reference labels. Depending on the SuperGLUE subset:\n        - for \'record\': list of question-answers dictionaries with the following keys:\n            - \'idx\': index of the question as specified by the dataset\n            - \'answers\': list of possible answers\n        - otherwise: list of reference labels\nReturns: depending on the SuperGLUE subset:\n    - for \'record\':\n        - \'exact_match\': Exact match between answer and gold answer\n        - \'f1\': F1 score\n    - for \'multirc\':\n        - \'exact_match\': Exact match between answer and gold answer\n        - \'f1_m\': Per-question macro-F1 score\n        - \'f1_a\': Average F1 score over all answers\n    - for \'axb\':\n        \'matthews_correlation\': Matthew Correlation\n    - for \'cb\':\n        - \'accuracy\': Accuracy\n        - \'f1\': F1 score\n    - for all others:\n        - \'accuracy\': Accuracy\nExamples:\n\n    >>> super_glue_metric = datasets.load_metric(\'super_glue\', \'copa\')  # any of ["copa", "rte", "wic", "wsc", "wsc.fixed", "boolq", "axg"]\n    >>> predictions = [0, 1]\n    >>> references = [0, 1]\n    >>> results = super_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {\'accuracy\': 1.0}\n\n    >>> super_glue_metric = datasets.load_metric(\'super_glue\', \'cb\')\n    >>> predictions = [0, 1]\n    >>> references = [0, 1]\n    >>> results = super_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {\'accuracy\': 1.0, \'f1\': 1.0}\n\n    >>> super_glue_metric = datasets.load_metric(\'super_glue\', \'record\')\n    >>> predictions = [{\'idx\': {\'passage\': 0, \'query\': 0}, \'prediction_text\': \'answer\'}]\n    >>> references = [{\'idx\': {\'passage\': 0, \'query\': 0}, \'answers\': [\'answer\', \'another_answer\']}]\n    >>> results = super_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {\'exact_match\': 1.0, \'f1\': 1.0}\n\n    >>> super_glue_metric = datasets.load_metric(\'super_glue\', \'multirc\')\n    >>> predictions = [{\'idx\': {\'answer\': 0, \'paragraph\': 0, \'question\': 0}, \'prediction\': 0}, {\'idx\': {\'answer\': 1, \'paragraph\': 2, \'question\': 3}, \'prediction\': 1}]\n    >>> references = [0, 1]\n    >>> results = super_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {\'exact_match\': 1.0, \'f1_m\': 1.0, \'f1_a\': 1.0}\n\n    >>> super_glue_metric = datasets.load_metric(\'super_glue\', \'axb\')\n    >>> references = [0, 1]\n    >>> predictions = [0, 1]\n    >>> results = super_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {\'matthews_correlation\': 1.0}\n'


def       lowerCamelCase_       (   UpperCAmelCase_  ,			UpperCAmelCase_  )						->List[Any]:





  """simple docstring"""


  return float((preds == labels).mean()  )


def       lowerCamelCase_       (   UpperCAmelCase_  ,			UpperCAmelCase_  ,			UpperCAmelCase_="binary"  )						->int:





  """simple docstring"""


  __UpperCAmelCase    :						Tuple       =						simple_accuracy(UpperCAmelCase_  ,			UpperCAmelCase_  )
  __UpperCAmelCase    :						str       =						float(fa_score(y_true=UpperCAmelCase_  ,			y_pred=UpperCAmelCase_  ,			average=UpperCAmelCase_  )  )
  return {
      "accuracy": acc,
      "f1": fa,
  }


def       lowerCamelCase_       (   UpperCAmelCase_  ,			UpperCAmelCase_  )						->Any:





  """simple docstring"""


  __UpperCAmelCase    :						str       =						{}
  for id_pred, label in zip(UpperCAmelCase_  ,			UpperCAmelCase_  ):
    __UpperCAmelCase    :						Union[str, Any]       =						f'''{id_pred['idx']['paragraph']}-{id_pred['idx']['question']}'''
    __UpperCAmelCase    :						Optional[Any]       =						id_pred['''prediction''']
    if question_id in question_map:
      question_map[question_id].append((pred, label)  )
    else:
      __UpperCAmelCase    :						Union[str, Any]       =						[(pred, label)]
  __UpperCAmelCase       ,     __UpperCAmelCase    :						Any       =						[], []
  for question, preds_labels in question_map.items():
    __UpperCAmelCase       ,     __UpperCAmelCase    :						Optional[int]       =						zip(*UpperCAmelCase_  )
    __UpperCAmelCase    :						Tuple       =						fa_score(y_true=UpperCAmelCase_  ,			y_pred=UpperCAmelCase_  ,			average='''macro'''  )
    fas.append(UpperCAmelCase_  )
    __UpperCAmelCase    :						int       =						int(sum(pred == label for pred, label in preds_labels  ) == len(UpperCAmelCase_  )  )
    ems.append(UpperCAmelCase_  )
  __UpperCAmelCase    :						int       =						float(sum(UpperCAmelCase_  ) / len(UpperCAmelCase_  )  )
  __UpperCAmelCase    :						List[Any]       =						sum(UpperCAmelCase_  ) / len(UpperCAmelCase_  )
  __UpperCAmelCase    :						List[Any]       =						float(fa_score(y_true=UpperCAmelCase_  ,			y_pred=[id_pred['''prediction'''] for id_pred in ids_preds]  )  )
  return {"exact_match": em, "f1_m": fa_m, "f1_a": fa_a}







@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION      ,  _KWARGS_DESCRIPTION )
class 							snake_case							(			datasets.Metric ):

      '''simple docstring'''



      def       A_					(							self       :      int ):


        '''simple docstring'''




        if self.config_name not in [
            "boolq",
            "cb",
            "copa",
            "multirc",
            "record",
            "rte",
            "wic",
            "wsc",
            "wsc.fixed",
            "axb",
            "axg",
        ]:
          raise KeyError(
              '''You should supply a configuration name selected in '''
              '''["boolq", "cb", "copa", "multirc", "record", "rte", "wic", "wsc", "wsc.fixed", "axb", "axg",]''' )
        return datasets.MetricInfo(
            description=_DESCRIPTION						,				citation=_CITATION						,				inputs_description=_KWARGS_DESCRIPTION						,				features=datasets.Features(self._get_feature_types() )						,				codebase_urls=[]						,				reference_urls=[]						,				format='''numpy''' if not self.config_name == '''record''' and not self.config_name == '''multirc''' else None						,				)

      def       A_					(							self       :      List[Any] ):


        '''simple docstring'''




        if self.config_name == "record":
          return {
              "predictions": {
                  "idx": {
                      "passage": datasets.Value('''int64''' ),
                      "query": datasets.Value('''int64''' ),
                  },
                  "prediction_text": datasets.Value('''string''' ),
              },
              "references": {
                  "idx": {
                      "passage": datasets.Value('''int64''' ),
                      "query": datasets.Value('''int64''' ),
                  },
                  "answers": datasets.Sequence(datasets.Value('''string''' ) ),
              },
          }
        elif self.config_name == "multirc":
          return {
              "predictions": {
                  "idx": {
                      "answer": datasets.Value('''int64''' ),
                      "paragraph": datasets.Value('''int64''' ),
                      "question": datasets.Value('''int64''' ),
                  },
                  "prediction": datasets.Value('''int64''' ),
              },
              "references": datasets.Value('''int64''' ),
          }
        else:
          return {
              "predictions": datasets.Value('''int64''' ),
              "references": datasets.Value('''int64''' ),
          }







      def       A_					(							self       :      int						,				__lowercase       :      Optional[int]						,				__lowercase       :      Dict ):


        '''simple docstring'''




        if self.config_name == "axb":
          return {"matthews_correlation": matthews_corrcoef(__lowercase						,				__lowercase )}
        elif self.config_name == "cb":
          return acc_and_fa(__lowercase						,				__lowercase						,				fa_avg='''macro''' )
        elif self.config_name == "record":
          __UpperCAmelCase    :						List[str]       =						[
              {
                  '''qas''': [
                      {'''id''': ref['''idx''']['''query'''], '''answers''': [{'''text''': ans} for ans in ref['''answers''']]}
                      for ref in references
                  ]
              }
          ]
          __UpperCAmelCase    :						List[Any]       =						{pred['''idx''']['''query''']: pred['''prediction_text'''] for pred in predictions}
          return evaluate_record(__lowercase						,				__lowercase )[0]
        elif self.config_name == "multirc":
          return evaluate_multirc(__lowercase						,				__lowercase )
        elif self.config_name in ["copa", "rte", "wic", "wsc", "wsc.fixed", "boolq", "axg"]:
          return {"accuracy": simple_accuracy(__lowercase						,				__lowercase )}
        else:
          raise KeyError(
              '''You should supply a configuration name selected in '''
              '''["boolq", "cb", "copa", "multirc", "record", "rte", "wic", "wsc", "wsc.fixed", "axb", "axg",]''' )






"""simple docstring"""





import argparse
import os

from transformers.utils import direct_transformers_import


# All paths are set with the intent you should run this script from the root of the repo with the command
# python utils/check_task_guides.py
lowercase__				:int				   =       'src/transformers'
lowercase__				:List[str]				   =       'docs/source/en/tasks'


def       lowerCamelCase_       (   UpperCAmelCase_  ,			UpperCAmelCase_  ,			UpperCAmelCase_  )						->str:





  """simple docstring"""


  with open(UpperCAmelCase_  ,			'''r'''  ,			encoding='''utf-8'''  ,			newline='''\n'''  ) as f:
    __UpperCAmelCase    :						Union[str, Any]       =						f.readlines()
  # Find the start prompt.
  __UpperCAmelCase    :						Any       =						0
  while not lines[start_index].startswith(UpperCAmelCase_  ):
    start_index += 1
  start_index += 1

  __UpperCAmelCase    :						Optional[Any]       =						start_index
  while not lines[end_index].startswith(UpperCAmelCase_  ):
    end_index += 1
  end_index -= 1

  while len(lines[start_index]  ) <= 1:
    start_index += 1
  while len(lines[end_index]  ) <= 1:
    end_index -= 1
  end_index += 1
  return "".join(lines[start_index:end_index]  ), start_index, end_index, lines


# This is to make sure the transformers module imported is the one in the repo.
lowercase__				:Any				   =       direct_transformers_import(TRANSFORMERS_PATH)

lowercase__				:List[Any]				   =       {
    'asr.md': transformers_module.models.auto.modeling_auto.MODEL_FOR_CTC_MAPPING_NAMES,
    'audio_classification.md': transformers_module.models.auto.modeling_auto.MODEL_FOR_AUDIO_CLASSIFICATION_MAPPING_NAMES,
    'language_modeling.md': transformers_module.models.auto.modeling_auto.MODEL_FOR_CAUSAL_LM_MAPPING_NAMES,
    'image_classification.md': transformers_module.models.auto.modeling_auto.MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING_NAMES,
    'masked_language_modeling.md': transformers_module.models.auto.modeling_auto.MODEL_FOR_MASKED_LM_MAPPING_NAMES,
    'multiple_choice.md': transformers_module.models.auto.modeling_auto.MODEL_FOR_MULTIPLE_CHOICE_MAPPING_NAMES,
    'object_detection.md': transformers_module.models.auto.modeling_auto.MODEL_FOR_OBJECT_DETECTION_MAPPING_NAMES,
    'question_answering.md': transformers_module.models.auto.modeling_auto.MODEL_FOR_QUESTION_ANSWERING_MAPPING_NAMES,
    'semantic_segmentation.md': transformers_module.models.auto.modeling_auto.MODEL_FOR_SEMANTIC_SEGMENTATION_MAPPING_NAMES,
    'sequence_classification.md': transformers_module.models.auto.modeling_auto.MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING_NAMES,
    'summarization.md': transformers_module.models.auto.modeling_auto.MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING_NAMES,
    'token_classification.md': transformers_module.models.auto.modeling_auto.MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING_NAMES,
    'translation.md': transformers_module.models.auto.modeling_auto.MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING_NAMES,
    'video_classification.md': transformers_module.models.auto.modeling_auto.MODEL_FOR_VIDEO_CLASSIFICATION_MAPPING_NAMES,
    'document_question_answering.md': transformers_module.models.auto.modeling_auto.MODEL_FOR_DOCUMENT_QUESTION_ANSWERING_MAPPING_NAMES,
    'monocular_depth_estimation.md': transformers_module.models.auto.modeling_auto.MODEL_FOR_DEPTH_ESTIMATION_MAPPING_NAMES,
}

# This list contains model types used in some task guides that are not in `CONFIG_MAPPING_NAMES` (therefore not in any
# `MODEL_MAPPING_NAMES` or any `MODEL_FOR_XXX_MAPPING_NAMES`).
lowercase__				:Union[str, Any]				   =       {
    'summarization.md': ('nllb',),
    'translation.md': ('nllb',),
}


def       lowerCamelCase_       (   UpperCAmelCase_  )						->Union[str, Any]:





  """simple docstring"""


  __UpperCAmelCase    :						List[str]       =						TASK_GUIDE_TO_MODELS[task_guide]
  __UpperCAmelCase    :						Dict       =						SPECIAL_TASK_GUIDE_TO_MODEL_TYPES.get(UpperCAmelCase_  ,			set()  )
  __UpperCAmelCase    :						List[Any]       =						{
      code: name
      for code, name in transformers_module.MODEL_NAMES_MAPPING.items()
      if (code in model_maping_names or code in special_model_types)
  }
  return ", ".join([f'''[{name}](../model_doc/{code})''' for code, name in model_names.items()]  ) + "\n"


def       lowerCamelCase_       (   UpperCAmelCase_  ,			UpperCAmelCase_=False  )						->Tuple:





  """simple docstring"""


  __UpperCAmelCase       ,     __UpperCAmelCase       ,     __UpperCAmelCase       ,     __UpperCAmelCase    :						Dict       =						_find_text_in_file(
      filename=os.path.join(UpperCAmelCase_  ,			UpperCAmelCase_  )  ,			start_prompt='''<!--This tip is automatically generated by `make fix-copies`, do not fill manually!-->'''  ,			end_prompt='''<!--End of the generated tip-->'''  ,			)

  __UpperCAmelCase    :						List[str]       =						get_model_list_for_task(UpperCAmelCase_  )

  if current_list != new_list:
    if overwrite:
      with open(os.path.join(UpperCAmelCase_  ,			UpperCAmelCase_  )  ,			'''w'''  ,			encoding='''utf-8'''  ,			newline='''\n'''  ) as f:
        f.writelines(lines[:start_index] + [new_list] + lines[end_index:]  )
    else:
      raise ValueError(
          f'''The list of models that can be used in the {task_guide} guide needs an update. Run `make fix-copies`'''
          ''' to fix this.'''  )


if __name__ == "__main__":
  lowercase__				:int				   =       argparse.ArgumentParser()
  parser.add_argument('--fix_and_overwrite', action='store_true', help='Whether to fix inconsistencies.')
  lowercase__				:Optional[Any]				   =       parser.parse_args()

  for task_guide in TASK_GUIDE_TO_MODELS.keys():
    check_model_list_for_task(task_guide, args.fix_and_overwrite)