
"""simple docstring"""
from sklearn.metrics import fa_score, matthews_corrcoef

import datasets

from .record_evaluation import evaluate as evaluate_record


SCREAMING_SNAKE_CASE_					     =	'''\
@article{wang2019superglue,
  title={SuperGLUE: A Stickier Benchmark for General-Purpose Language Understanding Systems},
  author={Wang, Alex and Pruksachatkun, Yada and Nangia, Nikita and Singh, Amanpreet and Michael, Julian and Hill, Felix and Levy, Omer and Bowman, Samuel R},
  journal={arXiv preprint arXiv:1905.00537},
  year={2019}
}
'''

SCREAMING_SNAKE_CASE_					     =	'''\
SuperGLUE (https://super.gluebenchmark.com/) is a new benchmark styled after
GLUE with a new set of more difficult language understanding tasks, improved
resources, and a new public leaderboard.
'''

SCREAMING_SNAKE_CASE_					     =	'''
Compute SuperGLUE evaluation metric associated to each SuperGLUE dataset.
Args:
    predictions: list of predictions to score. Depending on the SuperGlUE subset:
        - for \'record\': list of question-answer dictionaries with the following keys:
            - \'idx\': index of the question as specified by the dataset
            - \'prediction_text\': the predicted answer text
        - for \'multirc\': list of question-answer dictionaries with the following keys:
            - \'idx\': index of the question-answer pair as specified by the dataset
            - \'prediction\': the predicted answer label
        - otherwise: list of predicted labels
    references: list of reference labels. Depending on the SuperGLUE subset:
        - for \'record\': list of question-answers dictionaries with the following keys:
            - \'idx\': index of the question as specified by the dataset
            - \'answers\': list of possible answers
        - otherwise: list of reference labels
Returns: depending on the SuperGLUE subset:
    - for \'record\':
        - \'exact_match\': Exact match between answer and gold answer
        - \'f1\': F1 score
    - for \'multirc\':
        - \'exact_match\': Exact match between answer and gold answer
        - \'f1_m\': Per-question macro-F1 score
        - \'f1_a\': Average F1 score over all answers
    - for \'axb\':
        \'matthews_correlation\': Matthew Correlation
    - for \'cb\':
        - \'accuracy\': Accuracy
        - \'f1\': F1 score
    - for all others:
        - \'accuracy\': Accuracy
Examples:

    >>> super_glue_metric = datasets.load_metric(\'super_glue\', \'copa\')  # any of ["copa", "rte", "wic", "wsc", "wsc.fixed", "boolq", "axg"]
    >>> predictions = [0, 1]
    >>> references = [0, 1]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {\'accuracy\': 1.0}

    >>> super_glue_metric = datasets.load_metric(\'super_glue\', \'cb\')
    >>> predictions = [0, 1]
    >>> references = [0, 1]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {\'accuracy\': 1.0, \'f1\': 1.0}

    >>> super_glue_metric = datasets.load_metric(\'super_glue\', \'record\')
    >>> predictions = [{\'idx\': {\'passage\': 0, \'query\': 0}, \'prediction_text\': \'answer\'}]
    >>> references = [{\'idx\': {\'passage\': 0, \'query\': 0}, \'answers\': [\'answer\', \'another_answer\']}]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {\'exact_match\': 1.0, \'f1\': 1.0}

    >>> super_glue_metric = datasets.load_metric(\'super_glue\', \'multirc\')
    >>> predictions = [{\'idx\': {\'answer\': 0, \'paragraph\': 0, \'question\': 0}, \'prediction\': 0}, {\'idx\': {\'answer\': 1, \'paragraph\': 2, \'question\': 3}, \'prediction\': 1}]
    >>> references = [0, 1]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {\'exact_match\': 1.0, \'f1_m\': 1.0, \'f1_a\': 1.0}

    >>> super_glue_metric = datasets.load_metric(\'super_glue\', \'axb\')
    >>> references = [0, 1]
    >>> predictions = [0, 1]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {\'matthews_correlation\': 1.0}
'''

def  lowercase     (_lowerCAmelCase ,    _lowerCAmelCase						):
						return float((preds == labels).mean()						)

def  lowercase     (_lowerCAmelCase ,    _lowerCAmelCase ,    _lowerCAmelCase="binary"						):
						__lowerCAmelCase     =  simple_accuracy(_lowerCAmelCase ,    _lowerCAmelCase						)
						__lowerCAmelCase     =  float(fa_score(y_true=_lowerCAmelCase ,    y_pred=_lowerCAmelCase ,    average=_lowerCAmelCase						)						)
						return {
						    "accuracy": acc,
						    "f1": fa,
						}

def  lowercase     (_lowerCAmelCase ,    _lowerCAmelCase						):
						__lowerCAmelCase     =  {}
						for id_pred, label in zip(_lowerCAmelCase ,    _lowerCAmelCase						):
												__lowerCAmelCase     =  f"""{id_pred["idx"]["paragraph"]}-{id_pred["idx"]["question"]}"""
												__lowerCAmelCase     =  id_pred["""prediction"""]
												if question_id in question_map:
																		question_map[question_id].append((pred, label)						)
												else:
																		__lowerCAmelCase     =  [(pred, label)]
						__lowerCAmelCase   ,     __lowerCAmelCase     =  [], []
						for question, preds_labels in question_map.items():
												__lowerCAmelCase   ,     __lowerCAmelCase     =  zip(*_lowerCAmelCase						)
												__lowerCAmelCase     =  fa_score(y_true=_lowerCAmelCase ,    y_pred=_lowerCAmelCase ,    average="""macro"""						)
												fas.append(_lowerCAmelCase						)
												__lowerCAmelCase     =  int(sum(pred == label for pred, label in preds_labels						) == len(_lowerCAmelCase						)						)
												ems.append(_lowerCAmelCase						)
						__lowerCAmelCase     =  float(sum(_lowerCAmelCase						) / len(_lowerCAmelCase						)						)
						__lowerCAmelCase     =  sum(_lowerCAmelCase						) / len(_lowerCAmelCase						)
						__lowerCAmelCase     =  float(fa_score(y_true=_lowerCAmelCase ,    y_pred=[id_pred["""prediction"""] for id_pred in ids_preds]						)						)
						return {"exact_match": em, "f1_m": fa_m, "f1_a": fa_a}



@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION				,				_KWARGS_DESCRIPTION )
class       lowerCAmelCase_		(				datasets.Metric ):



							'''simple docstring'''





							def    A__						( self				)       ->  Optional[int]:
													if self.config_name not in [
													    "boolq",
													    "cb",
													    "copa",
													    "multirc",
													    "record",
													    "rte",
													    "wic",
													    "wsc",
													    "wsc.fixed",
													    "axb",
													    "axg",
													]:
																			raise KeyError(
																			    """You should supply a configuration name selected in """
																			    """[\"boolq\", \"cb\", \"copa\", \"multirc\", \"record\", \"rte\", \"wic\", \"wsc\", \"wsc.fixed\", \"axb\", \"axg\",]"""				)
													return datasets.MetricInfo(
													    description=_DESCRIPTION    ,				citation=_CITATION    ,				inputs_description=_KWARGS_DESCRIPTION    ,				features=datasets.Features(self._get_feature_types()				)    ,				codebase_urls=[]    ,				reference_urls=[]    ,				format="""numpy""" if not self.config_name == """record""" and not self.config_name == """multirc""" else None    ,				)




							def    A__						( self				)       ->  List[Any]:
													if self.config_name == "record":
																			return {
																			    "predictions": {
																			        "idx": {
																			            "passage": datasets.Value("""int64"""				),
																			            "query": datasets.Value("""int64"""				),
																			        },
																			        "prediction_text": datasets.Value("""string"""				),
																			    },
																			    "references": {
																			        "idx": {
																			            "passage": datasets.Value("""int64"""				),
																			            "query": datasets.Value("""int64"""				),
																			        },
																			        "answers": datasets.Sequence(datasets.Value("""string"""				)				),
																			    },
																			}
													elif self.config_name == "multirc":
																			return {
																			    "predictions": {
																			        "idx": {
																			            "answer": datasets.Value("""int64"""				),
																			            "paragraph": datasets.Value("""int64"""				),
																			            "question": datasets.Value("""int64"""				),
																			        },
																			        "prediction": datasets.Value("""int64"""				),
																			    },
																			    "references": datasets.Value("""int64"""				),
																			}
													else:
																			return {
																			    "predictions": datasets.Value("""int64"""				),
																			    "references": datasets.Value("""int64"""				),
																			}




							def    A__						( self    ,				snake_case_    ,				snake_case_				)       ->  str:
													if self.config_name == "axb":
																			return {"matthews_correlation": matthews_corrcoef(snake_case_    ,				snake_case_				)}
													elif self.config_name == "cb":
																			return acc_and_fa(snake_case_    ,				snake_case_    ,				fa_avg="""macro"""				)
													elif self.config_name == "record":
																			__lowerCAmelCase     =  [
																			    {
																			        """qas""": [
																			            {"""id""": ref["""idx"""]["""query"""], """answers""": [{"""text""": ans} for ans in ref["""answers"""]]}
																			            for ref in references
																			        ]
																			    }
																			]
																			__lowerCAmelCase     =  {pred["""idx"""]["""query"""]: pred["""prediction_text"""] for pred in predictions}
																			return evaluate_record(snake_case_    ,				snake_case_				)[0]
													elif self.config_name == "multirc":
																			return evaluate_multirc(snake_case_    ,				snake_case_				)
													elif self.config_name in ["copa", "rte", "wic", "wsc", "wsc.fixed", "boolq", "axg"]:
																			return {"accuracy": simple_accuracy(snake_case_    ,				snake_case_				)}
													else:
																			raise KeyError(
																			    """You should supply a configuration name selected in """
																			    """[\"boolq\", \"cb\", \"copa\", \"multirc\", \"record\", \"rte\", \"wic\", \"wsc\", \"wsc.fixed\", \"axb\", \"axg\",]"""				)


"""simple docstring"""
import gc
import unittest

import torch
from transformers import CLIPTextConfig, CLIPTextModel, CLIPTextModelWithProjection, CLIPTokenizer

from diffusers import (
    AutoencoderKL,
    DDIMScheduler,
    DDPMScheduler,
    PriorTransformer,
    StableUnCLIPPipeline,
    UNetaDConditionModel,
)
from diffusers.pipelines.stable_diffusion.stable_unclip_image_normalizer import StableUnCLIPImageNormalizer
from diffusers.utils.testing_utils import enable_full_determinism, load_numpy, require_torch_gpu, slow, torch_device

from ..pipeline_params import TEXT_TO_IMAGE_BATCH_PARAMS, TEXT_TO_IMAGE_IMAGE_PARAMS, TEXT_TO_IMAGE_PARAMS
from ..test_pipelines_common import (
    PipelineKarrasSchedulerTesterMixin,
    PipelineLatentTesterMixin,
    PipelineTesterMixin,
    assert_mean_pixel_difference,
)


enable_full_determinism()
class       lowerCAmelCase_		(				A__				,				A__				,				A__				,				unittest.TestCase ):



							'''simple docstring'''





							_snake_case						      =		StableUnCLIPPipeline
							_snake_case						      =		TEXT_TO_IMAGE_PARAMS
							_snake_case						      =		TEXT_TO_IMAGE_BATCH_PARAMS
							_snake_case						      =		TEXT_TO_IMAGE_IMAGE_PARAMS
							_snake_case						      =		TEXT_TO_IMAGE_IMAGE_PARAMS

							# TODO(will) Expected attn_bias.stride(1) == 0 to be true, but got false
							_snake_case						      =		False




							def    A__						( self				)       ->  Optional[Any]:
													__lowerCAmelCase     =  32
													__lowerCAmelCase     =  embedder_hidden_size

													# prior components

													torch.manual_seed(0				)
													__lowerCAmelCase     =  CLIPTokenizer.from_pretrained("""hf-internal-testing/tiny-random-clip"""				)

													torch.manual_seed(0				)
													__lowerCAmelCase     =  CLIPTextModelWithProjection(
													    CLIPTextConfig(
													        bos_token_id=0    ,				eos_token_id=2    ,				hidden_size=snake_case_    ,				projection_dim=snake_case_    ,				intermediate_size=37    ,				layer_norm_eps=1e-0_5    ,				num_attention_heads=4    ,				num_hidden_layers=5    ,				pad_token_id=1    ,				vocab_size=1_000    ,				)				)

													torch.manual_seed(0				)
													__lowerCAmelCase     =  PriorTransformer(
													    num_attention_heads=2    ,				attention_head_dim=12    ,				embedding_dim=snake_case_    ,				num_layers=1    ,				)

													torch.manual_seed(0				)
													__lowerCAmelCase     =  DDPMScheduler(
													    variance_type="""fixed_small_log"""    ,				prediction_type="""sample"""    ,				num_train_timesteps=1_000    ,				clip_sample=snake_case_    ,				clip_sample_range=5.0    ,				beta_schedule="""squaredcos_cap_v2"""    ,				)

													# regular denoising components

													torch.manual_seed(0				)
													__lowerCAmelCase     =  StableUnCLIPImageNormalizer(embedding_dim=snake_case_				)
													__lowerCAmelCase     =  DDPMScheduler(beta_schedule="""squaredcos_cap_v2"""				)

													torch.manual_seed(0				)
													__lowerCAmelCase     =  CLIPTokenizer.from_pretrained("""hf-internal-testing/tiny-random-clip"""				)

													torch.manual_seed(0				)
													__lowerCAmelCase     =  CLIPTextModel(
													    CLIPTextConfig(
													        bos_token_id=0    ,				eos_token_id=2    ,				hidden_size=snake_case_    ,				projection_dim=32    ,				intermediate_size=37    ,				layer_norm_eps=1e-0_5    ,				num_attention_heads=4    ,				num_hidden_layers=5    ,				pad_token_id=1    ,				vocab_size=1_000    ,				)				)

													torch.manual_seed(0				)
													__lowerCAmelCase     =  UNetaDConditionModel(
													    sample_size=32    ,				in_channels=4    ,				out_channels=4    ,				down_block_types=("""CrossAttnDownBlock2D""", """DownBlock2D""")    ,				up_block_types=("""UpBlock2D""", """CrossAttnUpBlock2D""")    ,				block_out_channels=(32, 64)    ,				attention_head_dim=(2, 4)    ,				class_embed_type="""projection"""    ,				projection_class_embeddings_input_dim=embedder_projection_dim * 2    ,				cross_attention_dim=snake_case_    ,				layers_per_block=1    ,				upcast_attention=snake_case_    ,				use_linear_projection=snake_case_    ,				)

													torch.manual_seed(0				)
													__lowerCAmelCase     =  DDIMScheduler(
													    beta_schedule="""scaled_linear"""    ,				beta_start=0.00_085    ,				beta_end=0.012    ,				prediction_type="""v_prediction"""    ,				set_alpha_to_one=snake_case_    ,				steps_offset=1    ,				)

													torch.manual_seed(0				)
													__lowerCAmelCase     =  AutoencoderKL()

													__lowerCAmelCase     =  {
													    # prior components
													    """prior_tokenizer""": prior_tokenizer,
													    """prior_text_encoder""": prior_text_encoder,
													    """prior""": prior,
													    """prior_scheduler""": prior_scheduler,
													    # image noising components
													    """image_normalizer""": image_normalizer,
													    """image_noising_scheduler""": image_noising_scheduler,
													    # regular denoising components
													    """tokenizer""": tokenizer,
													    """text_encoder""": text_encoder,
													    """unet""": unet,
													    """scheduler""": scheduler,
													    """vae""": vae,
													}

													return components




							def    A__						( self    ,				snake_case_    ,				snake_case_=0				)       ->  Optional[Any]:
													if str(snake_case_				).startswith("""mps"""				):
																			__lowerCAmelCase     =  torch.manual_seed(snake_case_				)
													else:
																			__lowerCAmelCase     =  torch.Generator(device=snake_case_				).manual_seed(snake_case_				)
													__lowerCAmelCase     =  {
													    """prompt""": """A painting of a squirrel eating a burger""",
													    """generator""": generator,
													    """num_inference_steps""": 2,
													    """prior_num_inference_steps""": 2,
													    """output_type""": """numpy""",
													}
													return inputs




							def    A__						( self				)       ->  Tuple:
													__lowerCAmelCase     =  torch_device == """cpu"""

													self._test_attention_slicing_forward_pass(test_max_difference=snake_case_				)




							def    A__						( self				)       ->  Tuple:
													__lowerCAmelCase     =  torch_device in ["""cpu""", """mps"""]

													self._test_inference_batch_single_identical(test_max_difference=snake_case_				)
@slow
@require_torch_gpu
class       lowerCAmelCase_		(				unittest.TestCase ):



							'''simple docstring'''





							def    A__						( self				)       ->  Optional[int]:
													# clean up the VRAM after each test
													super().tearDown()
													gc.collect()
													torch.cuda.empty_cache()




							def    A__						( self				)       ->  Tuple:
													__lowerCAmelCase     =  load_numpy(
													    """https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/stable_unclip/stable_unclip_2_1_l_anime_turtle_fp16.npy"""				)

													__lowerCAmelCase     =  StableUnCLIPPipeline.from_pretrained("""fusing/stable-unclip-2-1-l"""    ,				torch_dtype=torch.floataa				)
													pipe.to(snake_case_				)
													pipe.set_progress_bar_config(disable=snake_case_				)
													# stable unclip will oom when integration tests are run on a V100,
													# so turn on memory savings
													pipe.enable_attention_slicing()
													pipe.enable_sequential_cpu_offload()

													__lowerCAmelCase     =  torch.Generator(device="""cpu"""				).manual_seed(0				)
													__lowerCAmelCase     =  pipe("""anime turle"""    ,				generator=snake_case_    ,				output_type="""np"""				)

													__lowerCAmelCase     =  output.images[0]

													assert image.shape == (768, 768, 3)

													assert_mean_pixel_difference(snake_case_    ,				snake_case_				)




							def    A__						( self				)       ->  Tuple:
													torch.cuda.empty_cache()
													torch.cuda.reset_max_memory_allocated()
													torch.cuda.reset_peak_memory_stats()

													__lowerCAmelCase     =  StableUnCLIPPipeline.from_pretrained("""fusing/stable-unclip-2-1-l"""    ,				torch_dtype=torch.floataa				)
													__lowerCAmelCase     =  pipe.to(snake_case_				)
													pipe.set_progress_bar_config(disable=snake_case_				)
													pipe.enable_attention_slicing()
													pipe.enable_sequential_cpu_offload()

													__lowerCAmelCase     =  pipe(
													    """anime turtle"""    ,				prior_num_inference_steps=2    ,				num_inference_steps=2    ,				output_type="""np"""    ,				)

													__lowerCAmelCase     =  torch.cuda.max_memory_allocated()
													# make sure that less than 7 GB is allocated
													assert mem_bytes < 7 * 10**9
