




'''simple docstring'''



from sklearn.metrics import fa_score, matthews_corrcoef

import datasets

from .record_evaluation import evaluate as evaluate_record


UpperCamelCase_    	=				"\\n@article{wang2019superglue,\n  title={SuperGLUE: A Stickier Benchmark for General-Purpose Language Understanding Systems},\n  author={Wang, Alex and Pruksachatkun, Yada and Nangia, Nikita and Singh, Amanpreet and Michael, Julian and Hill, Felix and Levy, Omer and Bowman, Samuel R},\n  journal={arXiv preprint arXiv:1905.00537},\n  year={2019}\n}\n"

UpperCamelCase_    	=				"\\nSuperGLUE (https://super.gluebenchmark.com/) is a new benchmark styled after\nGLUE with a new set of more difficult language understanding tasks, improved\nresources, and a new public leaderboard.\n"

UpperCamelCase_    	=				"\nCompute SuperGLUE evaluation metric associated to each SuperGLUE dataset.\nArgs:\n    predictions: list of predictions to score. Depending on the SuperGlUE subset:\n        - for 'record': list of question-answer dictionaries with the following keys:\n            - 'idx': index of the question as specified by the dataset\n            - 'prediction_text': the predicted answer text\n        - for 'multirc': list of question-answer dictionaries with the following keys:\n            - 'idx': index of the question-answer pair as specified by the dataset\n            - 'prediction': the predicted answer label\n        - otherwise: list of predicted labels\n    references: list of reference labels. Depending on the SuperGLUE subset:\n        - for 'record': list of question-answers dictionaries with the following keys:\n            - 'idx': index of the question as specified by the dataset\n            - 'answers': list of possible answers\n        - otherwise: list of reference labels\nReturns: depending on the SuperGLUE subset:\n    - for 'record':\n        - 'exact_match': Exact match between answer and gold answer\n        - 'f1': F1 score\n    - for 'multirc':\n        - 'exact_match': Exact match between answer and gold answer\n        - 'f1_m': Per-question macro-F1 score\n        - 'f1_a': Average F1 score over all answers\n    - for 'axb':\n        'matthews_correlation': Matthew Correlation\n    - for 'cb':\n        - 'accuracy': Accuracy\n        - 'f1': F1 score\n    - for all others:\n        - 'accuracy': Accuracy\nExamples:\n\n    >>> super_glue_metric = datasets.load_metric('super_glue', 'copa')  # any of [\"copa\", \"rte\", \"wic\", \"wsc\", \"wsc.fixed\", \"boolq\", \"axg\"]\n    >>> predictions = [0, 1]\n    >>> references = [0, 1]\n    >>> results = super_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {'accuracy': 1.0}\n\n    >>> super_glue_metric = datasets.load_metric('super_glue', 'cb')\n    >>> predictions = [0, 1]\n    >>> references = [0, 1]\n    >>> results = super_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {'accuracy': 1.0, 'f1': 1.0}\n\n    >>> super_glue_metric = datasets.load_metric('super_glue', 'record')\n    >>> predictions = [{'idx': {'passage': 0, 'query': 0}, 'prediction_text': 'answer'}]\n    >>> references = [{'idx': {'passage': 0, 'query': 0}, 'answers': ['answer', 'another_answer']}]\n    >>> results = super_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {'exact_match': 1.0, 'f1': 1.0}\n\n    >>> super_glue_metric = datasets.load_metric('super_glue', 'multirc')\n    >>> predictions = [{'idx': {'answer': 0, 'paragraph': 0, 'question': 0}, 'prediction': 0}, {'idx': {'answer': 1, 'paragraph': 2, 'question': 3}, 'prediction': 1}]\n    >>> references = [0, 1]\n    >>> results = super_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {'exact_match': 1.0, 'f1_m': 1.0, 'f1_a': 1.0}\n\n    >>> super_glue_metric = datasets.load_metric('super_glue', 'axb')\n    >>> references = [0, 1]\n    >>> predictions = [0, 1]\n    >>> results = super_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {'matthews_correlation': 1.0}\n"




def   lowercase__(			__UpperCamelCase:					str      ,__UpperCamelCase:					Any    ):

      """simple docstring"""






      return float((preds == labels).mean()    )




def   lowercase__(			__UpperCamelCase:					Dict      ,__UpperCamelCase:					Optional[int]      ,__UpperCamelCase:					Union[str, Any]="binary"    ):

      """simple docstring"""






      SCREAMING_SNAKE_CASE   :      Optional[Any]    					=						simple_accuracy(__UpperCamelCase      ,__UpperCamelCase    )
      SCREAMING_SNAKE_CASE   :      Any    					=						float(fa_score(y_true=__UpperCamelCase      ,y_pred=__UpperCamelCase      ,average=__UpperCamelCase    )    )
      return {
          "accuracy": acc,
          "f1": fa,
      }




def   lowercase__(			__UpperCamelCase:					List[Any]      ,__UpperCamelCase:					Tuple    ):

      """simple docstring"""






      SCREAMING_SNAKE_CASE   :      Optional[Any]    					=						{}
      for id_pred, label in zip(__UpperCamelCase      ,__UpperCamelCase    ):
            SCREAMING_SNAKE_CASE   :      str    					=						f"{id_pred['idx']['paragraph']}-{id_pred['idx']['question']}"
            SCREAMING_SNAKE_CASE   :      Any    					=						id_pred['prediction']
            if question_id in question_map:
                  question_map[question_id].append((pred, label)    )
            else:
                  SCREAMING_SNAKE_CASE   :      Optional[Any]    					=						[(pred, label)]
      SCREAMING_SNAKE_CASE					,    SCREAMING_SNAKE_CASE   :      Optional[int]    					=						[], []
      for question, preds_labels in question_map.items():
            SCREAMING_SNAKE_CASE					,    SCREAMING_SNAKE_CASE   :      str    					=						zip(*__UpperCamelCase    )
            SCREAMING_SNAKE_CASE   :      Union[str, Any]    					=						fa_score(y_true=__UpperCamelCase      ,y_pred=__UpperCamelCase      ,average='macro'    )
            fas.append(__UpperCamelCase    )
            SCREAMING_SNAKE_CASE   :      Optional[int]    					=						int(sum(pred == label for pred, label in preds_labels    ) == len(__UpperCamelCase    )    )
            ems.append(__UpperCamelCase    )
      SCREAMING_SNAKE_CASE   :      Tuple    					=						float(sum(__UpperCamelCase    ) / len(__UpperCamelCase    )    )
      SCREAMING_SNAKE_CASE   :      Optional[int]    					=						sum(__UpperCamelCase    ) / len(__UpperCamelCase    )
      SCREAMING_SNAKE_CASE   :      int    					=						float(fa_score(y_true=__UpperCamelCase      ,y_pred=[id_pred['prediction'] for id_pred in ids_preds]    )    )
      return {"exact_match": em, "f1_m": fa_m, "f1_a": fa_a}






@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION ,      _KWARGS_DESCRIPTION )
class 							_a				(			datasets.Metric ):





       '''simple docstring'''






       def        UpperCamelCase_							(      self  ):







             '''simple docstring'''







             if self.config_name not in [
                 "boolq",
                 "cb",
                 "copa",
                 "multirc",
                 "record",
                 "rte",
                 "wic",
                 "wsc",
                 "wsc.fixed",
                 "axb",
                 "axg",
             ]:
                   raise KeyError(
                       'You should supply a configuration name selected in '
                       '["boolq", "cb", "copa", "multirc", "record", "rte", "wic", "wsc", "wsc.fixed", "axb", "axg",]'  )
             return datasets.MetricInfo(
                 description=_DESCRIPTION,							citation=_CITATION,							inputs_description=_KWARGS_DESCRIPTION,							features=datasets.Features(self._get_feature_types()  ),							codebase_urls=[],							reference_urls=[],							format='numpy' if not self.config_name == 'record' and not self.config_name == 'multirc' else None,							)







       def        UpperCamelCase_							(      self  ):







             '''simple docstring'''







             if self.config_name == "record":
                   return {
                       "predictions": {
                           "idx": {
                               "passage": datasets.Value('int64'  ),
                               "query": datasets.Value('int64'  ),
                           },
                           "prediction_text": datasets.Value('string'  ),
                       },
                       "references": {
                           "idx": {
                               "passage": datasets.Value('int64'  ),
                               "query": datasets.Value('int64'  ),
                           },
                           "answers": datasets.Sequence(datasets.Value('string'  )  ),
                       },
                   }
             elif self.config_name == "multirc":
                   return {
                       "predictions": {
                           "idx": {
                               "answer": datasets.Value('int64'  ),
                               "paragraph": datasets.Value('int64'  ),
                               "question": datasets.Value('int64'  ),
                           },
                           "prediction": datasets.Value('int64'  ),
                       },
                       "references": datasets.Value('int64'  ),
                   }
             else:
                   return {
                       "predictions": datasets.Value('int64'  ),
                       "references": datasets.Value('int64'  ),
                   }







       def        UpperCamelCase_							(      self,							A,							A  ):







             '''simple docstring'''







             if self.config_name == "axb":
                   return {"matthews_correlation": matthews_corrcoef(A,							A  )}
             elif self.config_name == "cb":
                   return acc_and_fa(A,							A,							fa_avg='macro'  )
             elif self.config_name == "record":
                   SCREAMING_SNAKE_CASE   :      List[Any]    					=						[
                       {
                           'qas': [
                               {'id': ref['idx']['query'], 'answers': [{'text': ans} for ans in ref['answers']]}
                               for ref in references
                           ]
                       }
                   ]
                   SCREAMING_SNAKE_CASE   :      Any    					=						{pred['idx']['query']: pred['prediction_text'] for pred in predictions}
                   return evaluate_record(A,							A  )[0]
             elif self.config_name == "multirc":
                   return evaluate_multirc(A,							A  )
             elif self.config_name in ["copa", "rte", "wic", "wsc", "wsc.fixed", "boolq", "axg"]:
                   return {"accuracy": simple_accuracy(A,							A  )}
             else:
                   raise KeyError(
                       'You should supply a configuration name selected in '
                       '["boolq", "cb", "copa", "multirc", "record", "rte", "wic", "wsc", "wsc.fixed", "axb", "axg",]'  )










import unittest

import numpy as np
import torch
from torch import nn
from transformers import (
    CLIPImageProcessor,
    CLIPTextConfig,
    CLIPTextModelWithProjection,
    CLIPTokenizer,
    CLIPVisionConfig,
    CLIPVisionModelWithProjection,
)

from diffusers import KandinskyVaaPriorPipeline, PriorTransformer, UnCLIPScheduler
from diffusers.utils import torch_device
from diffusers.utils.testing_utils import enable_full_determinism, skip_mps

from ..test_pipelines_common import PipelineTesterMixin


enable_full_determinism()




class       snake_case__ (				lowercase_						,     unittest.TestCase):

       '''simple docstring'''
       lowerCamelCase				:					Tuple								=			KandinskyVaaPriorPipeline
       lowerCamelCase				:					Optional[Any]								=			["prompt"]
       lowerCamelCase				:					Any								=			["prompt", "negative_prompt"]
       lowerCamelCase				:					Tuple								=			[
           "num_images_per_prompt",
           "generator",
           "num_inference_steps",
           "latents",
           "negative_prompt",
           "guidance_scale",
           "output_type",
           "return_dict",
       ]
       lowerCamelCase				:					Any								=			False



       @property
       def        __lowercase    (				self     )			->   Union[str, Any]:






            '''simple docstring'''





            return 32



       @property
       def        __lowercase    (				self     )			->   Any:






            '''simple docstring'''





            return 32



       @property
       def        __lowercase    (				self     )			->   List[Any]:






            '''simple docstring'''





            return self.time_input_dim



       @property
       def        __lowercase    (				self     )			->   Optional[Any]:






            '''simple docstring'''





            return self.time_input_dim * 4



       @property
       def        __lowercase    (				self     )			->   Optional[Any]:






            '''simple docstring'''





            return 1_00



       @property
       def        __lowercase    (				self     )			->   Optional[Any]:






            '''simple docstring'''





            __snake_case      :Tuple 							=    CLIPTokenizer.from_pretrained("""hf-internal-testing/tiny-random-clip"""     )
            return tokenizer



       @property
       def        __lowercase    (				self     )			->   str:






            '''simple docstring'''





            torch.manual_seed(0     )
            __snake_case      :Dict 							=    CLIPTextConfig(
                bos_token_id=0      ,    eos_token_id=2      ,    hidden_size=self.text_embedder_hidden_size      ,    projection_dim=self.text_embedder_hidden_size      ,    intermediate_size=37      ,    layer_norm_eps=1e-05      ,    num_attention_heads=4      ,    num_hidden_layers=5      ,    pad_token_id=1      ,    vocab_size=10_00      ,    )
            return CLIPTextModelWithProjection(a__     )



       @property
       def        __lowercase    (				self     )			->   str:






            '''simple docstring'''





            torch.manual_seed(0     )

            __snake_case      :List[Any] 							=    {
                """num_attention_heads""": 2,
                """attention_head_dim""": 12,
                """embedding_dim""": self.text_embedder_hidden_size,
                """num_layers""": 1,
            }

            __snake_case      :Union[str, Any] 							=    PriorTransformer(**a__     )
            # clip_std and clip_mean is initialized to be 0 so PriorTransformer.post_process_latents will always return 0 - set clip_std to be 1 so it won't return 0
            __snake_case      :int 							=    nn.Parameter(torch.ones(model.clip_std.shape     )     )
            return model



       @property
       def        __lowercase    (				self     )			->   List[str]:






            '''simple docstring'''





            torch.manual_seed(0     )
            __snake_case      :List[Any] 							=    CLIPVisionConfig(
                hidden_size=self.text_embedder_hidden_size      ,    image_size=2_24      ,    projection_dim=self.text_embedder_hidden_size      ,    intermediate_size=37      ,    num_attention_heads=4      ,    num_channels=3      ,    num_hidden_layers=5      ,    patch_size=14      ,    )

            __snake_case      :Tuple 							=    CLIPVisionModelWithProjection(a__     )
            return model



       @property
       def        __lowercase    (				self     )			->   Optional[Any]:






            '''simple docstring'''





            __snake_case      :Optional[int] 							=    CLIPImageProcessor(
                crop_size=2_24      ,    do_center_crop=a__      ,    do_normalize=a__      ,    do_resize=a__      ,    image_mean=[0.48_14_54_66, 0.4_57_82_75, 0.40_82_10_73]      ,    image_std=[0.26_86_29_54, 0.26_13_02_58, 0.27_57_77_11]      ,    resample=3      ,    size=2_24      ,    )

            return image_processor



       def        __lowercase    (				self     )			->   Optional[Any]:






            '''simple docstring'''





            __snake_case      :Optional[Any] 							=    self.dummy_prior
            __snake_case      :Tuple 							=    self.dummy_image_encoder
            __snake_case      :List[Any] 							=    self.dummy_text_encoder
            __snake_case      :List[str] 							=    self.dummy_tokenizer
            __snake_case      :int 							=    self.dummy_image_processor

            __snake_case      :Union[str, Any] 							=    UnCLIPScheduler(
                variance_type="""fixed_small_log"""      ,    prediction_type="""sample"""      ,    num_train_timesteps=10_00      ,    clip_sample=a__      ,    clip_sample_range=10.0      ,    )

            __snake_case      :Union[str, Any] 							=    {
                """prior""": prior,
                """image_encoder""": image_encoder,
                """text_encoder""": text_encoder,
                """tokenizer""": tokenizer,
                """scheduler""": scheduler,
                """image_processor""": image_processor,
            }

            return components



       def        __lowercase    (				self      ,    a__      ,    a__=0     )			->   Optional[Any]:






            '''simple docstring'''





            if str(a__     ).startswith("""mps"""     ):
                 __snake_case      :Any 							=    torch.manual_seed(a__     )
            else:
                 __snake_case      :Any 							=    torch.Generator(device=a__     ).manual_seed(a__     )
            __snake_case      :Union[str, Any] 							=    {
                """prompt""": """horse""",
                """generator""": generator,
                """guidance_scale""": 4.0,
                """num_inference_steps""": 2,
                """output_type""": """np""",
            }
            return inputs



       def        __lowercase    (				self     )			->   List[str]:






            '''simple docstring'''





            __snake_case      :str 							=    """cpu"""

            __snake_case      :Optional[Any] 							=    self.get_dummy_components()

            __snake_case      :List[Any] 							=    self.pipeline_class(**a__     )
            __snake_case      :Union[str, Any] 							=    pipe.to(a__     )

            pipe.set_progress_bar_config(disable=a__     )

            __snake_case      :List[Any] 							=    pipe(**self.get_dummy_inputs(a__     )     )
            __snake_case      :Any 							=    output.image_embeds

            __snake_case      :Any 							=    pipe(
                **self.get_dummy_inputs(a__     )      ,    return_dict=a__      ,    )[0]

            __snake_case      :Any 							=    image[0, -10:]
            __snake_case      :Any 							=    image_from_tuple[0, -10:]

            assert image.shape == (1, 32)

            __snake_case      :List[str] 							=    np.array(
                [-0.05_32, 1.71_20, 0.36_56, -1.08_52, -0.89_46, -1.17_56, 0.43_48, 0.24_82, 0.51_46, -0.11_56]     )

            assert np.abs(image_slice.flatten() - expected_slice     ).max() < 1e-2
            assert np.abs(image_from_tuple_slice.flatten() - expected_slice     ).max() < 1e-2



       @skip_mps
       def        __lowercase    (				self     )			->   str:






            '''simple docstring'''





            __snake_case      :List[Any] 							=    torch_device == """cpu"""
            __snake_case      :List[str] 							=    True
            __snake_case      :Optional[Any] 							=    False

            self._test_inference_batch_single_identical(
                test_max_difference=a__      ,    relax_max_difference=a__      ,    test_mean_pixel_difference=a__      ,    )



       @skip_mps
       def        __lowercase    (				self     )			->   List[Any]:






            '''simple docstring'''





            __snake_case      :Any 							=    torch_device == """cpu"""
            __snake_case      :str 							=    False

            self._test_attention_slicing_forward_pass(
                test_max_difference=a__      ,    test_mean_pixel_difference=a__      ,    )


