




"""simple docstring"""

from sklearn.metrics import fa_score, matthews_corrcoef

import datasets

from .record_evaluation import evaluate as evaluate_record


a         =      '''\\n@article{wang2019superglue,\n  title={SuperGLUE: A Stickier Benchmark for General-Purpose Language Understanding Systems},\n  author={Wang, Alex and Pruksachatkun, Yada and Nangia, Nikita and Singh, Amanpreet and Michael, Julian and Hill, Felix and Levy, Omer and Bowman, Samuel R},\n  journal={arXiv preprint arXiv:1905.00537},\n  year={2019}\n}\n'''

a         =      '''\\nSuperGLUE (https://super.gluebenchmark.com/) is a new benchmark styled after\nGLUE with a new set of more difficult language understanding tasks, improved\nresources, and a new public leaderboard.\n'''

a         =      '''\nCompute SuperGLUE evaluation metric associated to each SuperGLUE dataset.\nArgs:\n    predictions: list of predictions to score. Depending on the SuperGlUE subset:\n        - for \'record\': list of question-answer dictionaries with the following keys:\n            - \'idx\': index of the question as specified by the dataset\n            - \'prediction_text\': the predicted answer text\n        - for \'multirc\': list of question-answer dictionaries with the following keys:\n            - \'idx\': index of the question-answer pair as specified by the dataset\n            - \'prediction\': the predicted answer label\n        - otherwise: list of predicted labels\n    references: list of reference labels. Depending on the SuperGLUE subset:\n        - for \'record\': list of question-answers dictionaries with the following keys:\n            - \'idx\': index of the question as specified by the dataset\n            - \'answers\': list of possible answers\n        - otherwise: list of reference labels\nReturns: depending on the SuperGLUE subset:\n    - for \'record\':\n        - \'exact_match\': Exact match between answer and gold answer\n        - \'f1\': F1 score\n    - for \'multirc\':\n        - \'exact_match\': Exact match between answer and gold answer\n        - \'f1_m\': Per-question macro-F1 score\n        - \'f1_a\': Average F1 score over all answers\n    - for \'axb\':\n        \'matthews_correlation\': Matthew Correlation\n    - for \'cb\':\n        - \'accuracy\': Accuracy\n        - \'f1\': F1 score\n    - for all others:\n        - \'accuracy\': Accuracy\nExamples:\n\n    >>> super_glue_metric = datasets.load_metric(\'super_glue\', \'copa\')  # any of ["copa", "rte", "wic", "wsc", "wsc.fixed", "boolq", "axg"]\n    >>> predictions = [0, 1]\n    >>> references = [0, 1]\n    >>> results = super_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {\'accuracy\': 1.0}\n\n    >>> super_glue_metric = datasets.load_metric(\'super_glue\', \'cb\')\n    >>> predictions = [0, 1]\n    >>> references = [0, 1]\n    >>> results = super_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {\'accuracy\': 1.0, \'f1\': 1.0}\n\n    >>> super_glue_metric = datasets.load_metric(\'super_glue\', \'record\')\n    >>> predictions = [{\'idx\': {\'passage\': 0, \'query\': 0}, \'prediction_text\': \'answer\'}]\n    >>> references = [{\'idx\': {\'passage\': 0, \'query\': 0}, \'answers\': [\'answer\', \'another_answer\']}]\n    >>> results = super_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {\'exact_match\': 1.0, \'f1\': 1.0}\n\n    >>> super_glue_metric = datasets.load_metric(\'super_glue\', \'multirc\')\n    >>> predictions = [{\'idx\': {\'answer\': 0, \'paragraph\': 0, \'question\': 0}, \'prediction\': 0}, {\'idx\': {\'answer\': 1, \'paragraph\': 2, \'question\': 3}, \'prediction\': 1}]\n    >>> references = [0, 1]\n    >>> results = super_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {\'exact_match\': 1.0, \'f1_m\': 1.0, \'f1_a\': 1.0}\n\n    >>> super_glue_metric = datasets.load_metric(\'super_glue\', \'axb\')\n    >>> references = [0, 1]\n    >>> predictions = [0, 1]\n    >>> results = super_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {\'matthews_correlation\': 1.0}\n'''






def 						_snake_case					( _snake_case							: Tuple    ,  _snake_case							: Dict							)   -> Optional[Any]:






						'''simple docstring'''


						return float((preds == labels).mean()							)






def 						_snake_case					( _snake_case							: Dict    ,  _snake_case							: Tuple    ,  _snake_case							: Union[str, Any]="binary"							)   -> int:






						'''simple docstring'''


						_A        =       simple_accuracy(A__    ,  A__							)
						_A        =       float(fa_score(y_true=A__    ,  y_pred=A__    ,  average=A__							)							)
						return {
						    "accuracy": acc,
						    "f1": fa,
						}






def 						_snake_case					( _snake_case							: str    ,  _snake_case							: int							)   -> Optional[Any]:






						'''simple docstring'''


						_A        =       {}
						for id_pred, label in zip(A__    ,  A__							):
												_A        =       F'''{id_pred["idx"]["paragraph"]}-{id_pred["idx"]["question"]}'''
												_A        =       id_pred['prediction']
												if question_id in question_map:
																		question_map[question_id].append((pred, label)							)
												else:
																		_A        =       [(pred, label)]
						_A       ,   _A        =       [], []
						for question, preds_labels in question_map.items():
												_A       ,   _A        =       zip(*A__							)
												_A        =       fa_score(y_true=A__    ,  y_pred=A__    ,  average='macro'							)
												fas.append(A__							)
												_A        =       int(sum(pred == label for pred, label in preds_labels							) == len(A__							)							)
												ems.append(A__							)
						_A        =       float(sum(A__							) / len(A__							)							)
						_A        =       sum(A__							) / len(A__							)
						_A        =       float(fa_score(y_true=A__    ,  y_pred=[id_pred['prediction'] for id_pred in ids_preds]							)							)
						return {"exact_match": em, "f1_m": fa_m, "f1_a": fa_a}







@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION      ,     _KWARGS_DESCRIPTION							)
class     lowercase_	(					datasets.Metric							):



	'''simple docstring'''

	def       lowerCAmelCase_							(			self	:	Any		):
							if self.config_name not in [
							    "boolq",
							    "cb",
							    "copa",
							    "multirc",
							    "record",
							    "rte",
							    "wic",
							    "wsc",
							    "wsc.fixed",
							    "axb",
							    "axg",
							]:
													raise KeyError(
													    'You should supply a configuration name selected in '
													    '[\"boolq\", \"cb\", \"copa\", \"multirc\", \"record\", \"rte\", \"wic\", \"wsc\", \"wsc.fixed\", \"axb\", \"axg\",]'		)
							return datasets.MetricInfo(
							    description=_DESCRIPTION   ,							citation=_CITATION   ,							inputs_description=_KWARGS_DESCRIPTION   ,							features=datasets.Features(self._get_feature_types()		)   ,							codebase_urls=[]   ,							reference_urls=[]   ,							format='numpy' if not self.config_name == 'record' and not self.config_name == 'multirc' else None   ,							)

	def       lowerCAmelCase_							(			self	:	List[str]		):
							if self.config_name == "record":
													return {
													    "predictions": {
													        "idx": {
													            "passage": datasets.Value('int64'		),
													            "query": datasets.Value('int64'		),
													        },
													        "prediction_text": datasets.Value('string'		),
													    },
													    "references": {
													        "idx": {
													            "passage": datasets.Value('int64'		),
													            "query": datasets.Value('int64'		),
													        },
													        "answers": datasets.Sequence(datasets.Value('string'		)		),
													    },
													}
							elif self.config_name == "multirc":
													return {
													    "predictions": {
													        "idx": {
													            "answer": datasets.Value('int64'		),
													            "paragraph": datasets.Value('int64'		),
													            "question": datasets.Value('int64'		),
													        },
													        "prediction": datasets.Value('int64'		),
													    },
													    "references": datasets.Value('int64'		),
													}
							else:
													return {
													    "predictions": datasets.Value('int64'		),
													    "references": datasets.Value('int64'		),
													}



	def       lowerCAmelCase_							(			self	:	Optional[int]   ,							_UpperCAmelCase	:	Dict   ,							_UpperCAmelCase	:	int		):
							if self.config_name == "axb":
													return {"matthews_correlation": matthews_corrcoef(UpperCamelCase_   ,							UpperCamelCase_		)}
							elif self.config_name == "cb":
													return acc_and_fa(UpperCamelCase_   ,							UpperCamelCase_   ,							fa_avg='macro'		)
							elif self.config_name == "record":
													_A        =       [
													    {
													        'qas': [
													            {'id': ref['idx']['query'], 'answers': [{'text': ans} for ans in ref['answers']]}
													            for ref in references
													        ]
													    }
													]
													_A        =       {pred['idx']['query']: pred['prediction_text'] for pred in predictions}
													return evaluate_record(UpperCamelCase_   ,							UpperCamelCase_		)[0]
							elif self.config_name == "multirc":
													return evaluate_multirc(UpperCamelCase_   ,							UpperCamelCase_		)
							elif self.config_name in ["copa", "rte", "wic", "wsc", "wsc.fixed", "boolq", "axg"]:
													return {"accuracy": simple_accuracy(UpperCamelCase_   ,							UpperCamelCase_		)}
							else:
													raise KeyError(
													    'You should supply a configuration name selected in '
													    '[\"boolq\", \"cb\", \"copa\", \"multirc\", \"record\", \"rte\", \"wic\", \"wsc\", \"wsc.fixed\", \"axb\", \"axg\",]'		)







"""simple docstring"""

import argparse
from collections import defaultdict

import yaml


a         =      '''docs/source/en/_toctree.yml'''






def 						_snake_case					( _snake_case							: List[Any]							)   -> Optional[Any]:






      '''simple docstring'''


      _A        =       defaultdict(_snake_case							)
      _A        =       []
      _A        =       []
      for doc in doc_list:
            if "local" in doc:
                  counts[doc["local"]] += 1

            if doc["title"].lower() == "overview":
                  overview_doc.append({'local': doc['local'], 'title': doc['title']}							)
            else:
                  new_doc_list.append(_snake_case							)

      _A        =       new_doc_list
      _A        =       [key for key, value in counts.items() if value > 1]

      _A        =       []
      for duplicate_key in duplicates:
            _A        =       list({doc['title'] for doc in doc_list if doc['local'] == duplicate_key}							)
            if len(_snake_case							) > 1:
                  raise ValueError(
                      F'''{duplicate_key} is present several times in the documentation table of content at '''
                      '`docs/source/en/_toctree.yml` with different *Title* values. Choose one of those and remove the '
                      'others.'							)
            # Only add this once
            new_doc.append({'local': duplicate_key, 'title': titles[0]}							)

      # Add none duplicate-keys
      new_doc.extend([doc for doc in doc_list if 'local' not in counts or counts[doc['local']] == 1]							)
      _A        =       sorted(_snake_case    ,  key=lambda _snake_case							: s["title"].lower()							)

      # "overview" gets special treatment and is always first
      if len(_snake_case							) > 1:
            raise ValueError('{doc_list} has two \'overview\' docs which is not allowed.'							)

      overview_doc.extend(_snake_case							)

      # Sort
      return overview_doc






def 						_snake_case					( _snake_case							: Tuple=False							)   -> List[Any]:






      '''simple docstring'''


      with open(_snake_case    ,  encoding='utf-8'							) as f:
            _A        =       yaml.safe_load(f.read()							)

      # Get to the API doc
      _A        =       0
      while content[api_idx]["title"] != "API":
            api_idx += 1
      _A        =       content[api_idx]['sections']

      # Then to the model doc
      _A        =       0
      while api_doc[scheduler_idx]["title"] != "Schedulers":
            scheduler_idx += 1

      _A        =       api_doc[scheduler_idx]['sections']
      _A        =       clean_doc_toc(_snake_case							)

      _A        =       False
      if new_scheduler_doc != scheduler_doc:
            _A        =       True
            if overwrite:
                  _A        =       new_scheduler_doc

      if diff:
            if overwrite:
                  _A        =       api_doc
                  with open(_snake_case    ,  'w'    ,  encoding='utf-8'							) as f:
                        f.write(yaml.dump(_snake_case    ,  allow_unicode=_snake_case							)							)
            else:
                  raise ValueError(
                      'The model doc part of the table of content is not properly sorted, run `make style` to fix this.'							)






def 						_snake_case					( _snake_case							: str=False							)   -> Union[str, Any]:






      '''simple docstring'''


      with open(_snake_case    ,  encoding='utf-8'							) as f:
            _A        =       yaml.safe_load(f.read()							)

      # Get to the API doc
      _A        =       0
      while content[api_idx]["title"] != "API":
            api_idx += 1
      _A        =       content[api_idx]['sections']

      # Then to the model doc
      _A        =       0
      while api_doc[pipeline_idx]["title"] != "Pipelines":
            pipeline_idx += 1

      _A        =       False
      _A        =       api_doc[pipeline_idx]['sections']
      _A        =       []

      # sort sub pipeline docs
      for pipeline_doc in pipeline_docs:
            if "section" in pipeline_doc:
                  _A        =       pipeline_doc['section']
                  _A        =       clean_doc_toc(_snake_case							)
                  if overwrite:
                        _A        =       new_sub_pipeline_doc
            new_pipeline_docs.append(_snake_case							)

      # sort overall pipeline doc
      _A        =       clean_doc_toc(_snake_case							)

      if new_pipeline_docs != pipeline_docs:
            _A        =       True
            if overwrite:
                  _A        =       new_pipeline_docs

      if diff:
            if overwrite:
                  _A        =       api_doc
                  with open(_snake_case    ,  'w'    ,  encoding='utf-8'							) as f:
                        f.write(yaml.dump(_snake_case    ,  allow_unicode=_snake_case							)							)
            else:
                  raise ValueError(
                      'The model doc part of the table of content is not properly sorted, run `make style` to fix this.'							)


if __name__ == "__main__":
       a         =      argparse.ArgumentParser()
       parser.add_argument('''--fix_and_overwrite''', action='''store_true''', help='''Whether to fix inconsistencies.''')
       a         =      parser.parse_args()

       check_scheduler_doc(args.fix_and_overwrite)
       check_pipeline_doc(args.fix_and_overwrite)

