



'''simple docstring'''






import re
import time
from typing import Optional

import IPython.display as disp

from ..trainer_callback import TrainerCallback
from ..trainer_utils import IntervalStrategy, has_length


def 				snake_case_  (      _lowerCAmelCase						:		Optional[Any]						)   ->     Dict:
       UpperCAmelCase				: List[Any]											=	int(lowerCamelCase_						)
       UpperCAmelCase							,	UpperCAmelCase							,	UpperCAmelCase				: Optional[Any]											=	t // 3600, (t // 60) % 60, t % 60
       return f"""{h}:{m:02d}:{s:02d}""" if h != 0 else f"""{m:02d}:{s:02d}"""


def 				snake_case_  (      _lowerCAmelCase						:		Union[str, Any]     ,   _lowerCAmelCase						:		Union[str, Any]     ,   _lowerCAmelCase						:		Any     ,   _lowerCAmelCase						:		Tuple     ,   _lowerCAmelCase						:		Tuple=300						)   ->     Optional[Any]:
       # docstyle-ignore
       return f"""\n    <div>\n      {prefix}\n      <progress value='{value}' max='{total}' style='width:{width}px; height:20px; vertical-align: middle;'></progress>\n      {label}\n    </div>\n    """


def 				snake_case_  (      _lowerCAmelCase						:		Optional[Any]						)   ->     str:
       UpperCAmelCase				: Any											=	'''<table border="1" class="dataframe">\n'''
       html_code += """  <thead>\n <tr style="text-align: left;">\n"""
       for i in items[0]:
              html_code += f"""      <th>{i}</th>\n"""
       html_code += "    </tr>\n  </thead>\n  <tbody>\n"
       for line in items[1:]:
              html_code += "    <tr>\n"
              for elt in line:
                     UpperCAmelCase				: Optional[Any]											=	f"""{elt:.6f}""" if isinstance(lowerCamelCase_     ,   lowerCamelCase_						) else str(lowerCamelCase_						)
                     html_code += f"""      <td>{elt}</td>\n"""
              html_code += "    </tr>\n"
       html_code += "  </tbody>\n</table><p>"
       return html_code

class        SCREAMING_SNAKE_CASE:







 """simple docstring"""






 lowerCamelCase__    				= 5
 lowerCamelCase__    				= 0.2







 def __init__(		self    :  int	,    __snake_case    :  Union[str, Any]	,    __snake_case    :  Union[str, Any] = None	,    __snake_case    :  str = True	,    __snake_case    :  int = None	,    __snake_case    :  List[Any] = 300	,    )    ->	List[str]:
        UpperCAmelCase				: List[Any]											=	total
        UpperCAmelCase				: str											=	'''''' if prefix is None else prefix
        UpperCAmelCase				: Dict											=	leave
        UpperCAmelCase				: Optional[Any]											=	parent
        UpperCAmelCase				: List[Any]											=	width
        UpperCAmelCase				: List[str]											=	None
        UpperCAmelCase				: int											=	None
        UpperCAmelCase				: Union[str, Any]											=	None







 def   A			(		self    :  Any	,    __snake_case    :  str	,    __snake_case    :  str = False	,    __snake_case    :  Tuple = None	)    ->	int:
        UpperCAmelCase				: List[Any]											=	value
        if comment is not None:
               UpperCAmelCase				: Tuple											=	comment
        if self.last_value is None:
               UpperCAmelCase				: Optional[Any]											=	time.time()
               UpperCAmelCase				: str											=	value
               UpperCAmelCase				: Tuple											=	None
               UpperCAmelCase				: int											=	self.warmup
               UpperCAmelCase				: List[Any]											=	1
               self.update_bar(_lowerCamelCase	)
        elif value <= self.last_value and not force_update:
               return
        elif force_update or self.first_calls > 0 or value >= min(self.last_value + self.wait_for	,    self.total	):
               if self.first_calls > 0:
                      self.first_calls -= 1
               UpperCAmelCase				: List[Any]											=	time.time()
               UpperCAmelCase				: List[str]											=	current_time - self.start_time
               # We could have value = self.start_value if the update is called twixe with the same start value.
               if value > self.start_value:
                      UpperCAmelCase				: str											=	self.elapsed_time / (value - self.start_value)
               else:
                      UpperCAmelCase				: Tuple											=	None
               if value >= self.total:
                      UpperCAmelCase				: Dict											=	self.total
                      UpperCAmelCase				: Dict											=	None
                      if not self.leave:
                             self.close()
               elif self.average_time_per_item is not None:
                      UpperCAmelCase				: Union[str, Any]											=	self.average_time_per_item * (self.total - value)
               self.update_bar(_lowerCamelCase	)
               UpperCAmelCase				: Dict											=	value
               UpperCAmelCase				: Union[str, Any]											=	current_time
               if self.average_time_per_item is None:
                      UpperCAmelCase				: List[Any]											=	1
               else:
                      UpperCAmelCase				: List[str]											=	max(int(self.update_every / self.average_time_per_item	)	,    1	)







 def   A			(		self    :  Any	,    __snake_case    :  Optional[Any]	,    __snake_case    :  Tuple=None	)    ->	Dict:
        UpperCAmelCase				: Any											=	''' ''' * (len(str(self.total	)	) - len(str(_lowerCamelCase	)	)) + str(_lowerCamelCase	)
        if self.elapsed_time is None:
               UpperCAmelCase				: Optional[int]											=	F"""[{spaced_value}/{self.total} : < :"""
        elif self.predicted_remaining is None:
               UpperCAmelCase				: Any											=	F"""[{spaced_value}/{self.total} {format_time(self.elapsed_time	)}"""
        else:
               UpperCAmelCase				: str											=	(
                   F"""[{spaced_value}/{self.total} {format_time(self.elapsed_time	)} <"""
                   F""" {format_time(self.predicted_remaining	)}"""
               )
               self.label += F""", {1/self.average_time_per_item:.2f} it/s"""
        self.label += "]" if self.comment is None or len(self.comment	) == 0 else F""", {self.comment}]"""
        self.display()







 def   A			(		self    :  Optional[Any]	)    ->	Optional[Any]:
        UpperCAmelCase				: Any											=	html_progress_bar(self.value	,    self.total	,    self.prefix	,    self.label	,    self.width	)
        if self.parent is not None:
               # If this is a child bar, the parent will take care of the display.
               self.parent.display()
               return
        if self.output is None:
               UpperCAmelCase				: List[str]											=	disp.display(disp.HTML(self.html_code	)	,    display_id=_lowerCamelCase	)
        else:
               self.output.update(disp.HTML(self.html_code	)	)







 def   A			(		self    :  Optional[int]	)    ->	Any:





        if self.parent is None and self.output is not None:
               self.output.update(disp.HTML(''''''	)	)

class        SCREAMING_SNAKE_CASE(							lowerCAmelCase__    ):







 """simple docstring"""






 def __init__(		self    :  int	,    __snake_case    :  Any	,    __snake_case    :  List[str]=None	)    ->	Any:
        super().__init__(_lowerCamelCase	)
        UpperCAmelCase				: Optional[Any]											=	None if column_names is None else [column_names]
        UpperCAmelCase				: Optional[Any]											=	None







 def   A			(		self    :  Optional[int]	)    ->	Any:
        UpperCAmelCase				: Optional[Any]											=	html_progress_bar(self.value	,    self.total	,    self.prefix	,    self.label	,    self.width	)
        if self.inner_table is not None:
               self.html_code += text_to_html_table(self.inner_table	)
        if self.child_bar is not None:
               self.html_code += self.child_bar.html_code
        if self.output is None:
               UpperCAmelCase				: List[Any]											=	disp.display(disp.HTML(self.html_code	)	,    display_id=_lowerCamelCase	)
        else:
               self.output.update(disp.HTML(self.html_code	)	)







 def   A			(		self    :  Optional[int]	,    __snake_case    :  Optional[int]	)    ->	Union[str, Any]:





        if self.inner_table is None:
               UpperCAmelCase				: str											=	[list(values.keys()	), list(values.values()	)]
        else:
               UpperCAmelCase				: Optional[int]											=	self.inner_table[0]
               if len(self.inner_table	) == 1:
                      # We give a chance to update the column names at the first iteration
                      for key in values.keys():
                             if key not in columns:
                                    columns.append(_lowerCamelCase	)
                      UpperCAmelCase				: str											=	columns
               self.inner_table.append([values[c] for c in columns]	)







 def   A			(		self    :  Dict	,    __snake_case    :  int	,    __snake_case    :  Optional[int]=None	,    __snake_case    :  Optional[Any]=300	)    ->	int:
        UpperCAmelCase				: Tuple											=	NotebookProgressBar(_lowerCamelCase	,    prefix=_lowerCamelCase	,    parent=self	,    width=_lowerCamelCase	)
        return self.child_bar







 def   A			(		self    :  Dict	)    ->	List[str]:
        UpperCAmelCase				: Tuple											=	None
        self.display()

class        SCREAMING_SNAKE_CASE(							lowerCAmelCase__    ):







 """simple docstring"""






 def __init__(		self    :  List[str]	)    ->	List[Any]:
        UpperCAmelCase				: Optional[int]											=	None
        UpperCAmelCase				: Optional[Any]											=	None
        UpperCAmelCase				: Union[str, Any]											=	False







 def   A			(		self    :  Any	,    __snake_case    :  str	,    __snake_case    :  str	,    __snake_case    :  List[Any]	,    **__snake_case    :  int	)    ->	Union[str, Any]:
        UpperCAmelCase				: Tuple											=	'''Epoch''' if args.evaluation_strategy == IntervalStrategy.EPOCH else '''Step'''
        UpperCAmelCase				: List[Any]											=	0
        UpperCAmelCase				: List[Any]											=	0
        UpperCAmelCase				: int											=	[self.first_column] + ['''Training Loss''']
        if args.evaluation_strategy != IntervalStrategy.NO:
               column_names.append('''Validation Loss'''	)
        UpperCAmelCase				: Union[str, Any]											=	NotebookTrainingTracker(state.max_steps	,    _lowerCamelCase	)







 def   A			(		self    :  Tuple	,    __snake_case    :  List[Any]	,    __snake_case    :  List[str]	,    __snake_case    :  Optional[int]	,    **__snake_case    :  List[str]	)    ->	Tuple:
        UpperCAmelCase				: int											=	int(state.epoch	) if int(state.epoch	) == state.epoch else F"""{state.epoch:.2f}"""
        self.training_tracker.update(
            state.global_step + 1	,    comment=F"""Epoch {epoch}/{state.num_train_epochs}"""	,    force_update=self._force_next_update	,    )
        UpperCAmelCase				: int											=	False







 def   A			(		self    :  int	,    __snake_case    :  Optional[Any]	,    __snake_case    :  List[Any]	,    __snake_case    :  str	,    __snake_case    :  Dict=None	,    **__snake_case    :  Tuple	)    ->	Optional[int]:





        if not has_length(_lowerCamelCase	):
               return
        if self.prediction_bar is None:
               if self.training_tracker is not None:
                      UpperCAmelCase				: List[Any]											=	self.training_tracker.add_child(len(_lowerCamelCase	)	)
               else:
                      UpperCAmelCase				: Optional[Any]											=	NotebookProgressBar(len(_lowerCamelCase	)	)
               self.prediction_bar.update(1	)
        else:
               self.prediction_bar.update(self.prediction_bar.value + 1	)







 def   A			(		self    :  Any	,    __snake_case    :  Optional[Any]	,    __snake_case    :  Any	,    __snake_case    :  Any	,    **__snake_case    :  int	)    ->	Optional[Any]:





        if self.prediction_bar is not None:
               self.prediction_bar.close()
        UpperCAmelCase				: Any											=	None







 def   A			(		self    :  str	,    __snake_case    :  Optional[int]	,    __snake_case    :  Union[str, Any]	,    __snake_case    :  Any	,    __snake_case    :  List[str]=None	,    **__snake_case    :  str	)    ->	Optional[int]:





        if args.evaluation_strategy == IntervalStrategy.NO and "loss" in logs:
               UpperCAmelCase				: Optional[int]											=	{'''Training Loss''': logs['''loss''']}
               # First column is necessarily Step sine we're not in epoch eval strategy
               UpperCAmelCase				: Optional[Any]											=	state.global_step
               self.training_tracker.write_line(_lowerCamelCase	)







 def   A			(		self    :  Optional[Any]	,    __snake_case    :  Optional[int]	,    __snake_case    :  Optional[int]	,    __snake_case    :  Optional[int]	,    __snake_case    :  Tuple=None	,    **__snake_case    :  Optional[Any]	)    ->	Any:





        if self.training_tracker is not None:
               UpperCAmelCase				: Dict											=	{'''Training Loss''': '''No log''', '''Validation Loss''': '''No log'''}
               for log in reversed(state.log_history	):
                      if "loss" in log:
                             UpperCAmelCase				: Optional[Any]											=	log['''loss''']
                             break

               if self.first_column == "Epoch":
                      UpperCAmelCase				: Any											=	int(state.epoch	)
               else:
                      UpperCAmelCase				: Any											=	state.global_step
               UpperCAmelCase				: str											=	'''eval'''
               for k in metrics:
                      if k.endswith('''_loss'''	):
                             UpperCAmelCase				: str											=	re.sub(r'''\_loss$'''	,    ''''''	,    _lowerCamelCase	)
               UpperCAmelCase				: Any											=	metrics.pop('''total_flos'''	,    _lowerCamelCase	)
               UpperCAmelCase				: Optional[Any]											=	metrics.pop('''epoch'''	,    _lowerCamelCase	)
               UpperCAmelCase				: List[str]											=	metrics.pop(F"""{metric_key_prefix}_runtime"""	,    _lowerCamelCase	)
               UpperCAmelCase				: List[str]											=	metrics.pop(F"""{metric_key_prefix}_samples_per_second"""	,    _lowerCamelCase	)
               UpperCAmelCase				: Tuple											=	metrics.pop(F"""{metric_key_prefix}_steps_per_second"""	,    _lowerCamelCase	)
               UpperCAmelCase				: Optional[Any]											=	metrics.pop(F"""{metric_key_prefix}_jit_compilation_time"""	,    _lowerCamelCase	)
               for k, v in metrics.items():
                      if k == F"""{metric_key_prefix}_loss""":
                             UpperCAmelCase				: Tuple											=	v
                      else:
                             UpperCAmelCase				: List[Any]											=	k.split('''_'''	)
                             UpperCAmelCase				: Tuple											=	''' '''.join([part.capitalize() for part in splits[1:]]	)
                             UpperCAmelCase				: List[Any]											=	v
               self.training_tracker.write_line(_lowerCamelCase	)
               self.training_tracker.remove_child()
               UpperCAmelCase				: List[str]											=	None
               # Evaluation takes a long time so we should force the next update.
               UpperCAmelCase				: str											=	True







 def   A			(		self    :  Union[str, Any]	,    __snake_case    :  Dict	,    __snake_case    :  Union[str, Any]	,    __snake_case    :  List[Any]	,    **__snake_case    :  Dict	)    ->	Optional[int]:
        self.training_tracker.update(
            state.global_step	,    comment=F"""Epoch {int(state.epoch	)}/{state.num_train_epochs}"""	,    force_update=_lowerCamelCase	)
        UpperCAmelCase				: List[Any]											=	None










'''simple docstring'''






from sklearn.metrics import fa_score, matthews_corrcoef

import datasets

from .record_evaluation import evaluate as evaluate_record


UpperCamelCase__:     int										=    "\\n@article{wang2019superglue,\n  title={SuperGLUE: A Stickier Benchmark for General-Purpose Language Understanding Systems},\n  author={Wang, Alex and Pruksachatkun, Yada and Nangia, Nikita and Singh, Amanpreet and Michael, Julian and Hill, Felix and Levy, Omer and Bowman, Samuel R},\n  journal={arXiv preprint arXiv:1905.00537},\n  year={2019}\n}\n"

UpperCamelCase__:     Dict										=    "\\nSuperGLUE (https://super.gluebenchmark.com/) is a new benchmark styled after\nGLUE with a new set of more difficult language understanding tasks, improved\nresources, and a new public leaderboard.\n"

UpperCamelCase__:     str										=    "\nCompute SuperGLUE evaluation metric associated to each SuperGLUE dataset.\nArgs:\n    predictions: list of predictions to score. Depending on the SuperGlUE subset:\n        - for 'record': list of question-answer dictionaries with the following keys:\n            - 'idx': index of the question as specified by the dataset\n            - 'prediction_text': the predicted answer text\n        - for 'multirc': list of question-answer dictionaries with the following keys:\n            - 'idx': index of the question-answer pair as specified by the dataset\n            - 'prediction': the predicted answer label\n        - otherwise: list of predicted labels\n    references: list of reference labels. Depending on the SuperGLUE subset:\n        - for 'record': list of question-answers dictionaries with the following keys:\n            - 'idx': index of the question as specified by the dataset\n            - 'answers': list of possible answers\n        - otherwise: list of reference labels\nReturns: depending on the SuperGLUE subset:\n    - for 'record':\n        - 'exact_match': Exact match between answer and gold answer\n        - 'f1': F1 score\n    - for 'multirc':\n        - 'exact_match': Exact match between answer and gold answer\n        - 'f1_m': Per-question macro-F1 score\n        - 'f1_a': Average F1 score over all answers\n    - for 'axb':\n        'matthews_correlation': Matthew Correlation\n    - for 'cb':\n        - 'accuracy': Accuracy\n        - 'f1': F1 score\n    - for all others:\n        - 'accuracy': Accuracy\nExamples:\n\n    >>> super_glue_metric = datasets.load_metric('super_glue', 'copa')  # any of [\"copa\", \"rte\", \"wic\", \"wsc\", \"wsc.fixed\", \"boolq\", \"axg\"]\n    >>> predictions = [0, 1]\n    >>> references = [0, 1]\n    >>> results = super_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {'accuracy': 1.0}\n\n    >>> super_glue_metric = datasets.load_metric('super_glue', 'cb')\n    >>> predictions = [0, 1]\n    >>> references = [0, 1]\n    >>> results = super_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {'accuracy': 1.0, 'f1': 1.0}\n\n    >>> super_glue_metric = datasets.load_metric('super_glue', 'record')\n    >>> predictions = [{'idx': {'passage': 0, 'query': 0}, 'prediction_text': 'answer'}]\n    >>> references = [{'idx': {'passage': 0, 'query': 0}, 'answers': ['answer', 'another_answer']}]\n    >>> results = super_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {'exact_match': 1.0, 'f1': 1.0}\n\n    >>> super_glue_metric = datasets.load_metric('super_glue', 'multirc')\n    >>> predictions = [{'idx': {'answer': 0, 'paragraph': 0, 'question': 0}, 'prediction': 0}, {'idx': {'answer': 1, 'paragraph': 2, 'question': 3}, 'prediction': 1}]\n    >>> references = [0, 1]\n    >>> results = super_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {'exact_match': 1.0, 'f1_m': 1.0, 'f1_a': 1.0}\n\n    >>> super_glue_metric = datasets.load_metric('super_glue', 'axb')\n    >>> references = [0, 1]\n    >>> predictions = [0, 1]\n    >>> results = super_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {'matthews_correlation': 1.0}\n"


def 				snake_case_  (      _lowerCAmelCase						:		str     ,   _lowerCAmelCase						:		Tuple						)   ->     Dict:
							return float((preds == labels).mean()						)


def 				snake_case_  (      _lowerCAmelCase						:		List[Any]     ,   _lowerCAmelCase						:		Union[str, Any]     ,   _lowerCAmelCase						:		Dict="binary"						)   ->     Any:
							UpperCAmelCase				: str											=	simple_accuracy(_lowerCAmelCase     ,   _lowerCAmelCase						)
							UpperCAmelCase				: str											=	float(fa_score(y_true=_lowerCAmelCase     ,   y_pred=_lowerCAmelCase     ,   average=_lowerCAmelCase						)						)
							return {
							    "accuracy": acc,
							    "f1": fa,
							}


def 				snake_case_  (      _lowerCAmelCase						:		int     ,   _lowerCAmelCase						:		str						)   ->     List[str]:
							UpperCAmelCase				: Optional[int]											=	{}
							for id_pred, label in zip(_lowerCAmelCase     ,   _lowerCAmelCase						):
														UpperCAmelCase				: Union[str, Any]											=	f"""{id_pred["idx"]["paragraph"]}-{id_pred["idx"]["question"]}"""
														UpperCAmelCase				: List[str]											=	id_pred['''prediction''']
														if question_id in question_map:
																					question_map[question_id].append((pred, label)						)
														else:
																					UpperCAmelCase				: Union[str, Any]											=	[(pred, label)]
							UpperCAmelCase							,	UpperCAmelCase				: Dict											=	[], []
							for question, preds_labels in question_map.items():
														UpperCAmelCase							,	UpperCAmelCase				: List[Any]											=	zip(*_lowerCAmelCase						)
														UpperCAmelCase				: int											=	fa_score(y_true=_lowerCAmelCase     ,   y_pred=_lowerCAmelCase     ,   average='''macro'''						)
														fas.append(_lowerCAmelCase						)
														UpperCAmelCase				: Dict											=	int(sum(pred == label for pred, label in preds_labels						) == len(_lowerCAmelCase						)						)
														ems.append(_lowerCAmelCase						)
							UpperCAmelCase				: int											=	float(sum(_lowerCAmelCase						) / len(_lowerCAmelCase						)						)
							UpperCAmelCase				: Optional[int]											=	sum(_lowerCAmelCase						) / len(_lowerCAmelCase						)
							UpperCAmelCase				: str											=	float(fa_score(y_true=_lowerCAmelCase     ,   y_pred=[id_pred['''prediction'''] for id_pred in ids_preds]						)						)
							return {"exact_match": em, "f1_m": fa_m, "f1_a": fa_a}




@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION							,	_KWARGS_DESCRIPTION    )
class        SCREAMING_SNAKE_CASE(							datasets.Metric    ):







	"""simple docstring"""






	def   A			(		self    :  Optional[Any]	)    ->	Dict:
								if self.config_name not in [
								    "boolq",
								    "cb",
								    "copa",
								    "multirc",
								    "record",
								    "rte",
								    "wic",
								    "wsc",
								    "wsc.fixed",
								    "axb",
								    "axg",
								]:
															raise KeyError(
															    '''You should supply a configuration name selected in '''
															    '''["boolq", "cb", "copa", "multirc", "record", "rte", "wic", "wsc", "wsc.fixed", "axb", "axg",]'''	)
								return datasets.MetricInfo(
								    description=_DESCRIPTION	,    citation=_CITATION	,    inputs_description=_KWARGS_DESCRIPTION	,    features=datasets.Features(self._get_feature_types()	)	,    codebase_urls=[]	,    reference_urls=[]	,    format='''numpy''' if not self.config_name == '''record''' and not self.config_name == '''multirc''' else None	,    )







	def   A			(		self    :  Union[str, Any]	)    ->	Dict:
								if self.config_name == "record":
															return {
															    "predictions": {
															        "idx": {
															            "passage": datasets.Value('''int64'''	),
															            "query": datasets.Value('''int64'''	),
															        },
															        "prediction_text": datasets.Value('''string'''	),
															    },
															    "references": {
															        "idx": {
															            "passage": datasets.Value('''int64'''	),
															            "query": datasets.Value('''int64'''	),
															        },
															        "answers": datasets.Sequence(datasets.Value('''string'''	)	),
															    },
															}
								elif self.config_name == "multirc":
															return {
															    "predictions": {
															        "idx": {
															            "answer": datasets.Value('''int64'''	),
															            "paragraph": datasets.Value('''int64'''	),
															            "question": datasets.Value('''int64'''	),
															        },
															        "prediction": datasets.Value('''int64'''	),
															    },
															    "references": datasets.Value('''int64'''	),
															}
								else:
															return {
															    "predictions": datasets.Value('''int64'''	),
															    "references": datasets.Value('''int64'''	),
															}







	def   A			(		self    :  Dict	,    __snake_case    :  str	,    __snake_case    :  str	)    ->	Optional[int]:
								if self.config_name == "axb":
															return {"matthews_correlation": matthews_corrcoef(__snake_case	,    __snake_case	)}
								elif self.config_name == "cb":
															return acc_and_fa(__snake_case	,    __snake_case	,    fa_avg='''macro'''	)
								elif self.config_name == "record":
															UpperCAmelCase				: Tuple											=	[
															    {
															        '''qas''': [
															            {'''id''': ref['''idx''']['''query'''], '''answers''': [{'''text''': ans} for ans in ref['''answers''']]}
															            for ref in references
															        ]
															    }
															]
															UpperCAmelCase				: int											=	{pred['''idx''']['''query''']: pred['''prediction_text'''] for pred in predictions}
															return evaluate_record(__snake_case	,    __snake_case	)[0]
								elif self.config_name == "multirc":
															return evaluate_multirc(__snake_case	,    __snake_case	)
								elif self.config_name in ["copa", "rte", "wic", "wsc", "wsc.fixed", "boolq", "axg"]:
															return {"accuracy": simple_accuracy(__snake_case	,    __snake_case	)}
								else:
															raise KeyError(
															    '''You should supply a configuration name selected in '''
															    '''["boolq", "cb", "copa", "multirc", "record", "rte", "wic", "wsc", "wsc.fixed", "axb", "axg",]'''	)





