


import gc
import random
import unittest

import numpy as np
import torch
from PIL import Image
from transformers import CLIPTextConfig, CLIPTextModel, CLIPTokenizer

from diffusers import AutoencoderKL, DDIMScheduler, DDPMScheduler, StableDiffusionUpscalePipeline, UNetaDConditionModel
from diffusers.utils import floats_tensor, load_image, load_numpy, slow, torch_device
from diffusers.utils.testing_utils import enable_full_determinism, require_torch_gpu


enable_full_determinism()

class        a	(unittest.TestCase   ):




							"""simple docstring"""


							def 					__snake_case      (							self :						str       )			->					Optional[Any]:
														# clean up the VRAM after each test
														super().tearDown()
														gc.collect()
														torch.cuda.empty_cache()

							@property
							def 					__snake_case      (							self :						Union[str, Any]       )			->					List[str]:
														__snake_case	:    Dict											=     1
														__snake_case	:    List[Any]											=     3
														__snake_case	:    Any											=     (32, 32)

														__snake_case	:    List[str]											=     floats_tensor((batch_size, num_channels) + sizes					,			rng=random.Random(0       )       ).to(lowerCamelCase       )
														return image

							@property
							def 					__snake_case      (							self :						List[Any]       )			->					Tuple:
														torch.manual_seed(0       )
														__snake_case	:    Any											=     UNetaDConditionModel(
														    block_out_channels=(32, 32, 64)					,			layers_per_block=2					,			sample_size=32					,			in_channels=7					,			out_channels=4					,			down_block_types=("DownBlock2D", "CrossAttnDownBlock2D", "CrossAttnDownBlock2D")					,			up_block_types=("CrossAttnUpBlock2D", "CrossAttnUpBlock2D", "UpBlock2D")					,			cross_attention_dim=32					,			attention_head_dim=8					,			use_linear_projection=lowerCamelCase					,			only_cross_attention=(True, True, False)					,			num_class_embeds=100					,			)
														return model

							@property
							def 					__snake_case      (							self :						Dict       )			->					Union[str, Any]:
														torch.manual_seed(0       )
														__snake_case	:    str											=     AutoencoderKL(
														    block_out_channels=[32, 32, 64]					,			in_channels=3					,			out_channels=3					,			down_block_types=["DownEncoderBlock2D", "DownEncoderBlock2D", "DownEncoderBlock2D"]					,			up_block_types=["UpDecoderBlock2D", "UpDecoderBlock2D", "UpDecoderBlock2D"]					,			latent_channels=4					,			)
														return model

							@property
							def 					__snake_case      (							self :						Any       )			->					List[str]:
														torch.manual_seed(0       )
														__snake_case	:    str											=     CLIPTextConfig(
														    bos_token_id=0					,			eos_token_id=2					,			hidden_size=32					,			intermediate_size=37					,			layer_norm_eps=1E-05					,			num_attention_heads=4					,			num_hidden_layers=5					,			pad_token_id=1					,			vocab_size=1000					,			hidden_act="gelu"					,			projection_dim=512					,			)
														return CLIPTextModel(lowerCamelCase       )

							def 					__snake_case      (							self :						List[str]       )			->					Dict:
														__snake_case	:    Optional[int]											=     "cpu"  # ensure determinism for the device-dependent torch.Generator
														__snake_case	:    str											=     self.dummy_cond_unet_upscale
														__snake_case	:    Tuple											=     DDPMScheduler()
														__snake_case	:    Tuple											=     DDIMScheduler(prediction_type="v_prediction"       )
														__snake_case	:    Any											=     self.dummy_vae
														__snake_case	:    Any											=     self.dummy_text_encoder
														__snake_case	:    int											=     CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip"       )

														__snake_case	:    List[Any]											=     self.dummy_image.cpu().permute(0					,			2					,			3					,			1       )[0]
														__snake_case	:    Optional[Any]											=     Image.fromarray(np.uinta(lowerCamelCase       )       ).convert("RGB"       ).resize((64, 64)       )

														# make sure here that pndm scheduler skips prk
														__snake_case	:    Optional[Any]											=     StableDiffusionUpscalePipeline(
														    unet=lowerCamelCase					,			low_res_scheduler=lowerCamelCase					,			scheduler=lowerCamelCase					,			vae=lowerCamelCase					,			text_encoder=lowerCamelCase					,			tokenizer=lowerCamelCase					,			max_noise_level=350					,			)
														__snake_case	:    Union[str, Any]											=     sd_pipe.to(lowerCamelCase       )
														sd_pipe.set_progress_bar_config(disable=lowerCamelCase       )

														__snake_case	:    Union[str, Any]											=     "A painting of a squirrel eating a burger"
														__snake_case	:    Dict											=     torch.Generator(device=lowerCamelCase       ).manual_seed(0       )
														__snake_case	:    Union[str, Any]											=     sd_pipe(
														    [prompt]					,			image=lowerCamelCase					,			generator=lowerCamelCase					,			guidance_scale=6.0					,			noise_level=20					,			num_inference_steps=2					,			output_type="np"					,			)

														__snake_case	:    Dict											=     output.images

														__snake_case	:    Any											=     torch.Generator(device=lowerCamelCase       ).manual_seed(0       )
														__snake_case	:    Optional[int]											=     sd_pipe(
														    [prompt]					,			image=lowerCamelCase					,			generator=lowerCamelCase					,			guidance_scale=6.0					,			noise_level=20					,			num_inference_steps=2					,			output_type="np"					,			return_dict=lowerCamelCase					,			)[0]

														__snake_case	:    List[Any]											=     image[0, -3:, -3:, -1]
														__snake_case	:    int											=     image_from_tuple[0, -3:, -3:, -1]

														__snake_case	:    Tuple											=     low_res_image.size[0] * 4
														assert image.shape == (1, expected_height_width, expected_height_width, 3)
														__snake_case	:    List[Any]											=     np.array([0.31_13, 0.39_10, 0.42_72, 0.48_59, 0.50_61, 0.46_52, 0.53_62, 0.57_15, 0.56_61]       )

														assert np.abs(image_slice.flatten() - expected_slice       ).max() < 1E-2
														assert np.abs(image_from_tuple_slice.flatten() - expected_slice       ).max() < 1E-2

							def 					__snake_case      (							self :						List[str]       )			->					Optional[Any]:
														__snake_case	:    Any											=     "cpu"  # ensure determinism for the device-dependent torch.Generator
														__snake_case	:    List[Any]											=     self.dummy_cond_unet_upscale
														__snake_case	:    Optional[Any]											=     DDPMScheduler()
														__snake_case	:    Any											=     DDIMScheduler(prediction_type="v_prediction"       )
														__snake_case	:    str											=     self.dummy_vae
														__snake_case	:    Dict											=     self.dummy_text_encoder
														__snake_case	:    str											=     CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip"       )

														__snake_case	:    Dict											=     self.dummy_image.cpu().permute(0					,			2					,			3					,			1       )[0]
														__snake_case	:    str											=     Image.fromarray(np.uinta(lowerCamelCase       )       ).convert("RGB"       ).resize((64, 64)       )

														# make sure here that pndm scheduler skips prk
														__snake_case	:    List[Any]											=     StableDiffusionUpscalePipeline(
														    unet=lowerCamelCase					,			low_res_scheduler=lowerCamelCase					,			scheduler=lowerCamelCase					,			vae=lowerCamelCase					,			text_encoder=lowerCamelCase					,			tokenizer=lowerCamelCase					,			max_noise_level=350					,			)
														__snake_case	:    Union[str, Any]											=     sd_pipe.to(lowerCamelCase       )
														sd_pipe.set_progress_bar_config(disable=lowerCamelCase       )

														__snake_case	:    Any											=     "A painting of a squirrel eating a burger"
														__snake_case	:    Tuple											=     sd_pipe(
														    2 * [prompt]					,			image=2 * [low_res_image]					,			guidance_scale=6.0					,			noise_level=20					,			num_inference_steps=2					,			output_type="np"					,			)
														__snake_case	:    str											=     output.images
														assert image.shape[0] == 2

														__snake_case	:    int											=     torch.Generator(device=lowerCamelCase       ).manual_seed(0       )
														__snake_case	:    Union[str, Any]											=     sd_pipe(
														    [prompt]					,			image=lowerCamelCase					,			generator=lowerCamelCase					,			num_images_per_prompt=2					,			guidance_scale=6.0					,			noise_level=20					,			num_inference_steps=2					,			output_type="np"					,			)
														__snake_case	:    Optional[Any]											=     output.images
														assert image.shape[0] == 2






							@unittest.skipIf(torch_device != "cuda"					,			"This test requires a GPU"       )
							def 					__snake_case      (							self :						List[Any]       )			->					Tuple:
														__snake_case	:    Optional[int]											=     self.dummy_cond_unet_upscale
														__snake_case	:    Any											=     DDPMScheduler()
														__snake_case	:    Optional[Any]											=     DDIMScheduler(prediction_type="v_prediction"       )
														__snake_case	:    Union[str, Any]											=     self.dummy_vae
														__snake_case	:    Union[str, Any]											=     self.dummy_text_encoder
														__snake_case	:    Optional[int]											=     CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip"       )

														__snake_case	:    List[str]											=     self.dummy_image.cpu().permute(0					,			2					,			3					,			1       )[0]
														__snake_case	:    Union[str, Any]											=     Image.fromarray(np.uinta(lowerCamelCase       )       ).convert("RGB"       ).resize((64, 64)       )

														# put models in fp16, except vae as it overflows in fp16
														__snake_case	:    Any											=     unet.half()
														__snake_case	:    Tuple											=     text_encoder.half()

														# make sure here that pndm scheduler skips prk
														__snake_case	:    List[Any]											=     StableDiffusionUpscalePipeline(
														    unet=lowerCamelCase					,			low_res_scheduler=lowerCamelCase					,			scheduler=lowerCamelCase					,			vae=lowerCamelCase					,			text_encoder=lowerCamelCase					,			tokenizer=lowerCamelCase					,			max_noise_level=350					,			)
														__snake_case	:    Any											=     sd_pipe.to(lowerCamelCase       )
														sd_pipe.set_progress_bar_config(disable=lowerCamelCase       )

														__snake_case	:    int											=     "A painting of a squirrel eating a burger"
														__snake_case	:    Optional[Any]											=     torch.manual_seed(0       )
														__snake_case	:    Any											=     sd_pipe(
														    [prompt]					,			image=lowerCamelCase					,			generator=lowerCamelCase					,			num_inference_steps=2					,			output_type="np"					,			).images

														__snake_case	:    List[Any]											=     low_res_image.size[0] * 4
														assert image.shape == (1, expected_height_width, expected_height_width, 3)

@slow
@require_torch_gpu
class        a	(unittest.TestCase   ):




							"""simple docstring"""


							def 					__snake_case      (							self :						Optional[Any]       )			->					List[Any]:
														# clean up the VRAM after each test
														super().tearDown()
														gc.collect()
														torch.cuda.empty_cache()

							def 					__snake_case      (							self :						Dict       )			->					List[Any]:
														__snake_case	:    str											=     load_image(
														    "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main"
														    "/sd2-upscale/low_res_cat.png"       )
														__snake_case	:    Any											=     load_numpy(
														    "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/sd2-upscale"
														    "/upsampled_cat.npy"       )

														__snake_case	:    Dict											=     "stabilityai/stable-diffusion-x4-upscaler"
														__snake_case	:    Optional[int]											=     StableDiffusionUpscalePipeline.from_pretrained(lowerCamelCase       )
														pipe.to(lowerCamelCase       )
														pipe.set_progress_bar_config(disable=lowerCamelCase       )
														pipe.enable_attention_slicing()

														__snake_case	:    Any											=     "a cat sitting on a park bench"

														__snake_case	:    List[str]											=     torch.manual_seed(0       )
														__snake_case	:    List[str]											=     pipe(
														    prompt=lowerCamelCase					,			image=lowerCamelCase					,			generator=lowerCamelCase					,			output_type="np"					,			)
														__snake_case	:    str											=     output.images[0]

														assert image.shape == (512, 512, 3)
														assert np.abs(expected_image - image       ).max() < 1E-3

							def 					__snake_case      (							self :						Any       )			->					Any:
														__snake_case	:    Any											=     load_image(
														    "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main"
														    "/sd2-upscale/low_res_cat.png"       )
														__snake_case	:    Optional[Any]											=     load_numpy(
														    "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/sd2-upscale"
														    "/upsampled_cat_fp16.npy"       )

														__snake_case	:    Tuple											=     "stabilityai/stable-diffusion-x4-upscaler"
														__snake_case	:    List[str]											=     StableDiffusionUpscalePipeline.from_pretrained(
														    lowerCamelCase					,			torch_dtype=torch.floataa					,			)
														pipe.to(lowerCamelCase       )
														pipe.set_progress_bar_config(disable=lowerCamelCase       )
														pipe.enable_attention_slicing()

														__snake_case	:    int											=     "a cat sitting on a park bench"

														__snake_case	:    List[Any]											=     torch.manual_seed(0       )
														__snake_case	:    Dict											=     pipe(
														    prompt=lowerCamelCase					,			image=lowerCamelCase					,			generator=lowerCamelCase					,			output_type="np"					,			)
														__snake_case	:    Optional[int]											=     output.images[0]

														assert image.shape == (512, 512, 3)
														assert np.abs(expected_image - image       ).max() < 5E-1






							def 					__snake_case      (							self :						List[str]       )			->					Dict:
														torch.cuda.empty_cache()
														torch.cuda.reset_max_memory_allocated()
														torch.cuda.reset_peak_memory_stats()

														__snake_case	:    Dict											=     load_image(
														    "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main"
														    "/sd2-upscale/low_res_cat.png"       )

														__snake_case	:    int											=     "stabilityai/stable-diffusion-x4-upscaler"
														__snake_case	:    Dict											=     StableDiffusionUpscalePipeline.from_pretrained(
														    lowerCamelCase					,			torch_dtype=torch.floataa					,			)
														pipe.to(lowerCamelCase       )
														pipe.set_progress_bar_config(disable=lowerCamelCase       )
														pipe.enable_attention_slicing(1       )
														pipe.enable_sequential_cpu_offload()

														__snake_case	:    Union[str, Any]											=     "a cat sitting on a park bench"

														__snake_case	:    List[Any]											=     torch.manual_seed(0       )
														__snake_case	:    Union[str, Any]											=     pipe(
														    prompt=lowerCamelCase					,			image=lowerCamelCase					,			generator=lowerCamelCase					,			num_inference_steps=5					,			output_type="np"					,			)

														__snake_case	:    Optional[Any]											=     torch.cuda.max_memory_allocated()
														# make sure that less than 2.9 GB is allocated
														assert mem_bytes < 2.9 * 10**9





from sklearn.metrics import fa_score, matthews_corrcoef

import datasets

from .record_evaluation import evaluate as evaluate_record


_snake_case  :	List[Any]			      =       "\\n@article{wang2019superglue,\n  title={SuperGLUE: A Stickier Benchmark for General-Purpose Language Understanding Systems},\n  author={Wang, Alex and Pruksachatkun, Yada and Nangia, Nikita and Singh, Amanpreet and Michael, Julian and Hill, Felix and Levy, Omer and Bowman, Samuel R},\n  journal={arXiv preprint arXiv:1905.00537},\n  year={2019}\n}\n"

_snake_case  :	Any			      =       "\\nSuperGLUE (https://super.gluebenchmark.com/) is a new benchmark styled after\nGLUE with a new set of more difficult language understanding tasks, improved\nresources, and a new public leaderboard.\n"

_snake_case  :	str			      =       "\nCompute SuperGLUE evaluation metric associated to each SuperGLUE dataset.\nArgs:\n    predictions: list of predictions to score. Depending on the SuperGlUE subset:\n        - for 'record': list of question-answer dictionaries with the following keys:\n            - 'idx': index of the question as specified by the dataset\n            - 'prediction_text': the predicted answer text\n        - for 'multirc': list of question-answer dictionaries with the following keys:\n            - 'idx': index of the question-answer pair as specified by the dataset\n            - 'prediction': the predicted answer label\n        - otherwise: list of predicted labels\n    references: list of reference labels. Depending on the SuperGLUE subset:\n        - for 'record': list of question-answers dictionaries with the following keys:\n            - 'idx': index of the question as specified by the dataset\n            - 'answers': list of possible answers\n        - otherwise: list of reference labels\nReturns: depending on the SuperGLUE subset:\n    - for 'record':\n        - 'exact_match': Exact match between answer and gold answer\n        - 'f1': F1 score\n    - for 'multirc':\n        - 'exact_match': Exact match between answer and gold answer\n        - 'f1_m': Per-question macro-F1 score\n        - 'f1_a': Average F1 score over all answers\n    - for 'axb':\n        'matthews_correlation': Matthew Correlation\n    - for 'cb':\n        - 'accuracy': Accuracy\n        - 'f1': F1 score\n    - for all others:\n        - 'accuracy': Accuracy\nExamples:\n\n    >>> super_glue_metric = datasets.load_metric('super_glue', 'copa')  # any of [\"copa\", \"rte\", \"wic\", \"wsc\", \"wsc.fixed\", \"boolq\", \"axg\"]\n    >>> predictions = [0, 1]\n    >>> references = [0, 1]\n    >>> results = super_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {'accuracy': 1.0}\n\n    >>> super_glue_metric = datasets.load_metric('super_glue', 'cb')\n    >>> predictions = [0, 1]\n    >>> references = [0, 1]\n    >>> results = super_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {'accuracy': 1.0, 'f1': 1.0}\n\n    >>> super_glue_metric = datasets.load_metric('super_glue', 'record')\n    >>> predictions = [{'idx': {'passage': 0, 'query': 0}, 'prediction_text': 'answer'}]\n    >>> references = [{'idx': {'passage': 0, 'query': 0}, 'answers': ['answer', 'another_answer']}]\n    >>> results = super_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {'exact_match': 1.0, 'f1': 1.0}\n\n    >>> super_glue_metric = datasets.load_metric('super_glue', 'multirc')\n    >>> predictions = [{'idx': {'answer': 0, 'paragraph': 0, 'question': 0}, 'prediction': 0}, {'idx': {'answer': 1, 'paragraph': 2, 'question': 3}, 'prediction': 1}]\n    >>> references = [0, 1]\n    >>> results = super_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {'exact_match': 1.0, 'f1_m': 1.0, 'f1_a': 1.0}\n\n    >>> super_glue_metric = datasets.load_metric('super_glue', 'axb')\n    >>> references = [0, 1]\n    >>> predictions = [0, 1]\n    >>> results = super_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {'matthews_correlation': 1.0}\n"







def        lowerCAmelCase_  (      __lowerCamelCase							,			__lowerCamelCase			):
							return float((preds == labels).mean()			)







def        lowerCAmelCase_  (      __lowerCamelCase							,			__lowerCamelCase							,			__lowerCamelCase="binary"			):
							__snake_case	:    Union[str, Any]											=     simple_accuracy(__lowerCamelCase							,			__lowerCamelCase			)
							__snake_case	:    List[str]											=     float(fa_score(y_true=__lowerCamelCase							,			y_pred=__lowerCamelCase							,			average=__lowerCamelCase			)			)
							return {
							    "accuracy": acc,
							    "f1": fa,
							}







def        lowerCAmelCase_  (      __lowerCamelCase							,			__lowerCamelCase			):
							__snake_case	:    List[Any]											=     {}
							for id_pred, label in zip(__lowerCamelCase							,			__lowerCamelCase			):
														__snake_case	:    int											=     F'{id_pred["idx"]["paragraph"]}-{id_pred["idx"]["question"]}'
														__snake_case	:    str											=     id_pred["prediction"]
														if question_id in question_map:
																					question_map[question_id].append((pred, label)			)
														else:
																					__snake_case	:    Optional[Any]											=     [(pred, label)]
							__snake_case				,    __snake_case	:    Dict											=     [], []
							for question, preds_labels in question_map.items():
														__snake_case				,    __snake_case	:    int											=     zip(*__lowerCamelCase			)
														__snake_case	:    Optional[Any]											=     fa_score(y_true=__lowerCamelCase							,			y_pred=__lowerCamelCase							,			average="macro"			)
														fas.append(__lowerCamelCase			)
														__snake_case	:    Union[str, Any]											=     int(sum(pred == label for pred, label in preds_labels			) == len(__lowerCamelCase			)			)
														ems.append(__lowerCamelCase			)
							__snake_case	:    Tuple											=     float(sum(__lowerCamelCase			) / len(__lowerCamelCase			)			)
							__snake_case	:    Any											=     sum(__lowerCamelCase			) / len(__lowerCamelCase			)
							__snake_case	:    int											=     float(fa_score(y_true=__lowerCamelCase							,			y_pred=[id_pred["prediction"] for id_pred in ids_preds]			)			)
							return {"exact_match": em, "f1_m": fa_m, "f1_a": fa_a}

@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION  ,       _KWARGS_DESCRIPTION   )
class        a	(datasets.Metric   ):




							"""simple docstring"""


							def 					__snake_case      (							self :						List[Any]       )			->					Union[str, Any]:
														if self.config_name not in [
														    "boolq",
														    "cb",
														    "copa",
														    "multirc",
														    "record",
														    "rte",
														    "wic",
														    "wsc",
														    "wsc.fixed",
														    "axb",
														    "axg",
														]:
																					raise KeyError(
																					    "You should supply a configuration name selected in "
																					    "[\"boolq\", \"cb\", \"copa\", \"multirc\", \"record\", \"rte\", \"wic\", \"wsc\", \"wsc.fixed\", \"axb\", \"axg\",]"       )
														return datasets.MetricInfo(
														    description=_DESCRIPTION					,			citation=_CITATION					,			inputs_description=_KWARGS_DESCRIPTION					,			features=datasets.Features(self._get_feature_types()       )					,			codebase_urls=[]					,			reference_urls=[]					,			format="numpy" if not self.config_name == "record" and not self.config_name == "multirc" else None					,			)

							def 					__snake_case      (							self :						Any       )			->					int:
														if self.config_name == "record":
																					return {
																					    "predictions": {
																					        "idx": {
																					            "passage": datasets.Value("int64"       ),
																					            "query": datasets.Value("int64"       ),
																					        },
																					        "prediction_text": datasets.Value("string"       ),
																					    },
																					    "references": {
																					        "idx": {
																					            "passage": datasets.Value("int64"       ),
																					            "query": datasets.Value("int64"       ),
																					        },
																					        "answers": datasets.Sequence(datasets.Value("string"       )       ),
																					    },
																					}
														elif self.config_name == "multirc":
																					return {
																					    "predictions": {
																					        "idx": {
																					            "answer": datasets.Value("int64"       ),
																					            "paragraph": datasets.Value("int64"       ),
																					            "question": datasets.Value("int64"       ),
																					        },
																					        "prediction": datasets.Value("int64"       ),
																					    },
																					    "references": datasets.Value("int64"       ),
																					}
														else:
																					return {
																					    "predictions": datasets.Value("int64"       ),
																					    "references": datasets.Value("int64"       ),
																					}






							def 					__snake_case      (							self :						Optional[Any]					,			lowerCamelCase :						str					,			lowerCamelCase :						Optional[Any]       )			->					str:
														if self.config_name == "axb":
																					return {"matthews_correlation": matthews_corrcoef(lowerCamelCase					,			lowerCamelCase       )}
														elif self.config_name == "cb":
																					return acc_and_fa(lowerCamelCase					,			lowerCamelCase					,			fa_avg="macro"       )
														elif self.config_name == "record":
																					__snake_case	:    Tuple											=     [
																					    {
																					        "qas": [
																					            {"id": ref["idx"]["query"], "answers": [{"text": ans} for ans in ref["answers"]]}
																					            for ref in references
																					        ]
																					    }
																					]
																					__snake_case	:    List[str]											=     {pred["idx"]["query"]: pred["prediction_text"] for pred in predictions}
																					return evaluate_record(lowerCamelCase					,			lowerCamelCase       )[0]
														elif self.config_name == "multirc":
																					return evaluate_multirc(lowerCamelCase					,			lowerCamelCase       )
														elif self.config_name in ["copa", "rte", "wic", "wsc", "wsc.fixed", "boolq", "axg"]:
																					return {"accuracy": simple_accuracy(lowerCamelCase					,			lowerCamelCase       )}
														else:
																					raise KeyError(
																					    "You should supply a configuration name selected in "
																					    "[\"boolq\", \"cb\", \"copa\", \"multirc\", \"record\", \"rte\", \"wic\", \"wsc\", \"wsc.fixed\", \"axb\", \"axg\",]"       )

