




from sklearn.metrics import fa_score, matthews_corrcoef

import datasets

from .record_evaluation import evaluate as evaluate_record


SCREAMING_SNAKE_CASE:     List[Any]									= '''\
@article{wang2019superglue,
  title={SuperGLUE: A Stickier Benchmark for General-Purpose Language Understanding Systems},
  author={Wang, Alex and Pruksachatkun, Yada and Nangia, Nikita and Singh, Amanpreet and Michael, Julian and Hill, Felix and Levy, Omer and Bowman, Samuel R},
  journal={arXiv preprint arXiv:1905.00537},
  year={2019}
}
'''

SCREAMING_SNAKE_CASE:     Optional[Any]									= '''\
SuperGLUE (https://super.gluebenchmark.com/) is a new benchmark styled after
GLUE with a new set of more difficult language understanding tasks, improved
resources, and a new public leaderboard.
'''

SCREAMING_SNAKE_CASE:     Dict									= '''
Compute SuperGLUE evaluation metric associated to each SuperGLUE dataset.
Args:
    predictions: list of predictions to score. Depending on the SuperGlUE subset:
        - for \'record\': list of question-answer dictionaries with the following keys:
            - \'idx\': index of the question as specified by the dataset
            - \'prediction_text\': the predicted answer text
        - for \'multirc\': list of question-answer dictionaries with the following keys:
            - \'idx\': index of the question-answer pair as specified by the dataset
            - \'prediction\': the predicted answer label
        - otherwise: list of predicted labels
    references: list of reference labels. Depending on the SuperGLUE subset:
        - for \'record\': list of question-answers dictionaries with the following keys:
            - \'idx\': index of the question as specified by the dataset
            - \'answers\': list of possible answers
        - otherwise: list of reference labels
Returns: depending on the SuperGLUE subset:
    - for \'record\':
        - \'exact_match\': Exact match between answer and gold answer
        - \'f1\': F1 score
    - for \'multirc\':
        - \'exact_match\': Exact match between answer and gold answer
        - \'f1_m\': Per-question macro-F1 score
        - \'f1_a\': Average F1 score over all answers
    - for \'axb\':
        \'matthews_correlation\': Matthew Correlation
    - for \'cb\':
        - \'accuracy\': Accuracy
        - \'f1\': F1 score
    - for all others:
        - \'accuracy\': Accuracy
Examples:

    >>> super_glue_metric = datasets.load_metric(\'super_glue\', \'copa\')  # any of ["copa", "rte", "wic", "wsc", "wsc.fixed", "boolq", "axg"]
    >>> predictions = [0, 1]
    >>> references = [0, 1]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {\'accuracy\': 1.0}

    >>> super_glue_metric = datasets.load_metric(\'super_glue\', \'cb\')
    >>> predictions = [0, 1]
    >>> references = [0, 1]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {\'accuracy\': 1.0, \'f1\': 1.0}

    >>> super_glue_metric = datasets.load_metric(\'super_glue\', \'record\')
    >>> predictions = [{\'idx\': {\'passage\': 0, \'query\': 0}, \'prediction_text\': \'answer\'}]
    >>> references = [{\'idx\': {\'passage\': 0, \'query\': 0}, \'answers\': [\'answer\', \'another_answer\']}]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {\'exact_match\': 1.0, \'f1\': 1.0}

    >>> super_glue_metric = datasets.load_metric(\'super_glue\', \'multirc\')
    >>> predictions = [{\'idx\': {\'answer\': 0, \'paragraph\': 0, \'question\': 0}, \'prediction\': 0}, {\'idx\': {\'answer\': 1, \'paragraph\': 2, \'question\': 3}, \'prediction\': 1}]
    >>> references = [0, 1]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {\'exact_match\': 1.0, \'f1_m\': 1.0, \'f1_a\': 1.0}

    >>> super_glue_metric = datasets.load_metric(\'super_glue\', \'axb\')
    >>> references = [0, 1]
    >>> predictions = [0, 1]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {\'matthews_correlation\': 1.0}
'''


def 					_a							(					lowerCAmelCase    ,						lowerCAmelCase )->	Optional[Any]:
		return float((preds == labels).mean() )


def 					_a							(					lowerCAmelCase    ,						lowerCAmelCase    ,						lowerCAmelCase="binary" )->	str:
		SCREAMING_SNAKE_CASE_					      =							simple_accuracy(lowerCAmelCase    ,						lowerCAmelCase )
		SCREAMING_SNAKE_CASE_					      =							float(fa_score(y_true=lowerCAmelCase    ,						y_pred=lowerCAmelCase    ,						average=lowerCAmelCase ) )
		return {
		    "accuracy": acc,
		    "f1": fa,
		}


def 					_a							(					lowerCAmelCase    ,						lowerCAmelCase )->	Dict:
		SCREAMING_SNAKE_CASE_					      =							{}
		for id_pred, label in zip(lowerCAmelCase    ,						lowerCAmelCase ):
				SCREAMING_SNAKE_CASE_					      =							F'''{id_pred["idx"]["paragraph"]}-{id_pred["idx"]["question"]}'''
				SCREAMING_SNAKE_CASE_					      =							id_pred['prediction']
				if question_id in question_map:
						question_map[question_id].append((pred, label) )
				else:
						SCREAMING_SNAKE_CASE_					      =							[(pred, label)]
		SCREAMING_SNAKE_CASE_      ,				SCREAMING_SNAKE_CASE_					      =							[], []
		for question, preds_labels in question_map.items():
				SCREAMING_SNAKE_CASE_      ,				SCREAMING_SNAKE_CASE_					      =							zip(*lowerCAmelCase )
				SCREAMING_SNAKE_CASE_					      =							fa_score(y_true=lowerCAmelCase    ,						y_pred=lowerCAmelCase    ,						average='macro' )
				fas.append(lowerCAmelCase )
				SCREAMING_SNAKE_CASE_					      =							int(sum(pred == label for pred, label in preds_labels ) == len(lowerCAmelCase ) )
				ems.append(lowerCAmelCase )
		SCREAMING_SNAKE_CASE_					      =							float(sum(lowerCAmelCase ) / len(lowerCAmelCase ) )
		SCREAMING_SNAKE_CASE_					      =							sum(lowerCAmelCase ) / len(lowerCAmelCase )
		SCREAMING_SNAKE_CASE_					      =							float(fa_score(y_true=lowerCAmelCase    ,						y_pred=[id_pred['prediction'] for id_pred in ids_preds] ) )
		return {"exact_match": em, "f1_m": fa_m, "f1_a": fa_a}




@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION							,	_KWARGS_DESCRIPTION )
class    lowercase_		(datasets.Metric ):



						def 	__a					(			self	:   Union[str, Any]  ):

								"""simple docstring"""

								if self.config_name not in [
								    "boolq",
								    "cb",
								    "copa",
								    "multirc",
								    "record",
								    "rte",
								    "wic",
								    "wsc",
								    "wsc.fixed",
								    "axb",
								    "axg",
								]:
										raise KeyError(
										    'You should supply a configuration name selected in '
										    '["boolq", "cb", "copa", "multirc", "record", "rte", "wic", "wsc", "wsc.fixed", "axb", "axg",]'  )
								return datasets.MetricInfo(
								    description=_DESCRIPTION					,							citation=_CITATION					,							inputs_description=_KWARGS_DESCRIPTION					,							features=datasets.Features(self._get_feature_types()  )					,							codebase_urls=[]					,							reference_urls=[]					,							format='numpy' if not self.config_name == 'record' and not self.config_name == 'multirc' else None					,							)



						def 	__a					(			self	:   List[Any]  ):

								"""simple docstring"""

								if self.config_name == "record":
										return {
										    "predictions": {
										        "idx": {
										            "passage": datasets.Value('int64'  ),
										            "query": datasets.Value('int64'  ),
										        },
										        "prediction_text": datasets.Value('string'  ),
										    },
										    "references": {
										        "idx": {
										            "passage": datasets.Value('int64'  ),
										            "query": datasets.Value('int64'  ),
										        },
										        "answers": datasets.Sequence(datasets.Value('string'  )  ),
										    },
										}
								elif self.config_name == "multirc":
										return {
										    "predictions": {
										        "idx": {
										            "answer": datasets.Value('int64'  ),
										            "paragraph": datasets.Value('int64'  ),
										            "question": datasets.Value('int64'  ),
										        },
										        "prediction": datasets.Value('int64'  ),
										    },
										    "references": datasets.Value('int64'  ),
										}
								else:
										return {
										    "predictions": datasets.Value('int64'  ),
										    "references": datasets.Value('int64'  ),
										}






						def 	__a					(			self	:   str					,							snake_case__	:   List[Any]					,							snake_case__	:   Union[str, Any]  ):

								"""simple docstring"""

								if self.config_name == "axb":
										return {"matthews_correlation": matthews_corrcoef(snake_case__					,							snake_case__  )}
								elif self.config_name == "cb":
										return acc_and_fa(snake_case__					,							snake_case__					,							fa_avg='macro'  )
								elif self.config_name == "record":
										SCREAMING_SNAKE_CASE_					      =							[
										    {
										        'qas': [
										            {'id': ref['idx']['query'], 'answers': [{'text': ans} for ans in ref['answers']]}
										            for ref in references
										        ]
										    }
										]
										SCREAMING_SNAKE_CASE_					      =							{pred['idx']['query']: pred['prediction_text'] for pred in predictions}
										return evaluate_record(snake_case__					,							snake_case__  )[0]
								elif self.config_name == "multirc":
										return evaluate_multirc(snake_case__					,							snake_case__  )
								elif self.config_name in ["copa", "rte", "wic", "wsc", "wsc.fixed", "boolq", "axg"]:
										return {"accuracy": simple_accuracy(snake_case__					,							snake_case__  )}
								else:
										raise KeyError(
										    'You should supply a configuration name selected in '
										    '["boolq", "cb", "copa", "multirc", "record", "rte", "wic", "wsc", "wsc.fixed", "axb", "axg",]'  )





import unittest

from transformers import MODEL_FOR_ZERO_SHOT_OBJECT_DETECTION_MAPPING, is_vision_available, pipeline
from transformers.testing_utils import (
    is_pipeline_test,
    nested_simplify,
    require_tf,
    require_torch,
    require_vision,
    slow,
)

from .test_pipelines_common import ANY


if is_vision_available():
						from PIL import Image
else:



						class    lowercase_		:



												@staticmethod
												def 	__a					(			*snake_case__	:   List[Any]					,							**snake_case__	:   List[Any]  ):

														"""simple docstring"""

														pass



@is_pipeline_test
@require_vision
@require_torch
class    lowercase_		(unittest.TestCase ):
						lowerCAmelCase__			      =MODEL_FOR_ZERO_SHOT_OBJECT_DETECTION_MAPPING



						def 	__a					(			self	:   Any					,							snake_case__	:   List[str]					,							snake_case__	:   int					,							snake_case__	:   Optional[int]  ):

								"""simple docstring"""

								SCREAMING_SNAKE_CASE_					      =							pipeline(
								    'zero-shot-object-detection'					,							model='hf-internal-testing/tiny-random-owlvit-object-detection'  )

								SCREAMING_SNAKE_CASE_					      =							[
								    {
								        'image': './tests/fixtures/tests_samples/COCO/000000039769.png',
								        'candidate_labels': ['cat', 'remote', 'couch'],
								    }
								]
								return object_detector, examples



						def 	__a					(			self	:   List[Any]					,							snake_case__	:   Union[str, Any]					,							snake_case__	:   Union[str, Any]  ):

								"""simple docstring"""

								SCREAMING_SNAKE_CASE_					      =							object_detector(examples[0]					,							threshold=0.0  )

								SCREAMING_SNAKE_CASE_					      =							len(snake_case__  )
								self.assertGreater(snake_case__					,							0  )
								self.assertEqual(
								    snake_case__					,							[
								        {
								            'score': ANY(snake_case__  ),
								            'label': ANY(snake_case__  ),
								            'box': {'xmin': ANY(snake_case__  ), 'ymin': ANY(snake_case__  ), 'xmax': ANY(snake_case__  ), 'ymax': ANY(snake_case__  )},
								        }
								        for i in range(snake_case__  )
								    ]					,							)



						@require_tf
						@unittest.skip('Zero Shot Object Detection not implemented in TF'  )
						def 	__a					(			self	:   Union[str, Any]  ):

								"""simple docstring"""

								pass



						@require_torch
						def 	__a					(			self	:   Tuple  ):

								"""simple docstring"""

								SCREAMING_SNAKE_CASE_					      =							pipeline(
								    'zero-shot-object-detection'					,							model='hf-internal-testing/tiny-random-owlvit-object-detection'  )

								SCREAMING_SNAKE_CASE_					      =							object_detector(
								    './tests/fixtures/tests_samples/COCO/000000039769.png'					,							candidate_labels=['cat', 'remote', 'couch']					,							threshold=0.64					,							)

								self.assertEqual(
								    nested_simplify(snake_case__					,							decimals=4  )					,							[
								        {'score': 0.72_35, 'label': 'cat', 'box': {'xmin': 2_04, 'ymin': 1_67, 'xmax': 2_32, 'ymax': 1_90}},
								        {'score': 0.72_18, 'label': 'remote', 'box': {'xmin': 2_04, 'ymin': 1_67, 'xmax': 2_32, 'ymax': 1_90}},
								        {'score': 0.71_84, 'label': 'couch', 'box': {'xmin': 2_04, 'ymin': 1_67, 'xmax': 2_32, 'ymax': 1_90}},
								        {'score': 0.67_48, 'label': 'remote', 'box': {'xmin': 5_71, 'ymin': 83, 'xmax': 5_98, 'ymax': 1_03}},
								        {'score': 0.66_56, 'label': 'cat', 'box': {'xmin': 5_71, 'ymin': 83, 'xmax': 5_98, 'ymax': 1_03}},
								        {'score': 0.66_14, 'label': 'couch', 'box': {'xmin': 5_71, 'ymin': 83, 'xmax': 5_98, 'ymax': 1_03}},
								        {'score': 0.64_56, 'label': 'remote', 'box': {'xmin': 4_94, 'ymin': 1_05, 'xmax': 5_21, 'ymax': 1_27}},
								        {'score': 0.6_42, 'label': 'remote', 'box': {'xmin': 67, 'ymin': 2_74, 'xmax': 93, 'ymax': 2_97}},
								        {'score': 0.64_19, 'label': 'cat', 'box': {'xmin': 4_94, 'ymin': 1_05, 'xmax': 5_21, 'ymax': 1_27}},
								    ]					,							)

								SCREAMING_SNAKE_CASE_					      =							object_detector(
								    [
								        {
								            'image': './tests/fixtures/tests_samples/COCO/000000039769.png',
								            'candidate_labels': ['cat', 'remote', 'couch'],
								        }
								    ]					,							threshold=0.64					,							)

								self.assertEqual(
								    nested_simplify(snake_case__					,							decimals=4  )					,							[
								        [
								            {'score': 0.72_35, 'label': 'cat', 'box': {'xmin': 2_04, 'ymin': 1_67, 'xmax': 2_32, 'ymax': 1_90}},
								            {'score': 0.72_18, 'label': 'remote', 'box': {'xmin': 2_04, 'ymin': 1_67, 'xmax': 2_32, 'ymax': 1_90}},
								            {'score': 0.71_84, 'label': 'couch', 'box': {'xmin': 2_04, 'ymin': 1_67, 'xmax': 2_32, 'ymax': 1_90}},
								            {'score': 0.67_48, 'label': 'remote', 'box': {'xmin': 5_71, 'ymin': 83, 'xmax': 5_98, 'ymax': 1_03}},
								            {'score': 0.66_56, 'label': 'cat', 'box': {'xmin': 5_71, 'ymin': 83, 'xmax': 5_98, 'ymax': 1_03}},
								            {'score': 0.66_14, 'label': 'couch', 'box': {'xmin': 5_71, 'ymin': 83, 'xmax': 5_98, 'ymax': 1_03}},
								            {'score': 0.64_56, 'label': 'remote', 'box': {'xmin': 4_94, 'ymin': 1_05, 'xmax': 5_21, 'ymax': 1_27}},
								            {'score': 0.6_42, 'label': 'remote', 'box': {'xmin': 67, 'ymin': 2_74, 'xmax': 93, 'ymax': 2_97}},
								            {'score': 0.64_19, 'label': 'cat', 'box': {'xmin': 4_94, 'ymin': 1_05, 'xmax': 5_21, 'ymax': 1_27}},
								        ]
								    ]					,							)



						@require_torch
						@slow
						def 	__a					(			self	:   Optional[Any]  ):

								"""simple docstring"""

								SCREAMING_SNAKE_CASE_					      =							pipeline('zero-shot-object-detection'  )

								SCREAMING_SNAKE_CASE_					      =							object_detector(
								    'http://images.cocodataset.org/val2017/000000039769.jpg'					,							candidate_labels=['cat', 'remote', 'couch']					,							)
								self.assertEqual(
								    nested_simplify(snake_case__					,							decimals=4  )					,							[
								        {'score': 0.28_68, 'label': 'cat', 'box': {'xmin': 3_24, 'ymin': 20, 'xmax': 6_40, 'ymax': 3_73}},
								        {'score': 0.2_77, 'label': 'remote', 'box': {'xmin': 40, 'ymin': 72, 'xmax': 1_77, 'ymax': 1_15}},
								        {'score': 0.25_37, 'label': 'cat', 'box': {'xmin': 1, 'ymin': 55, 'xmax': 3_15, 'ymax': 4_72}},
								        {'score': 0.14_74, 'label': 'remote', 'box': {'xmin': 3_35, 'ymin': 74, 'xmax': 3_71, 'ymax': 1_87}},
								        {'score': 0.12_08, 'label': 'couch', 'box': {'xmin': 4, 'ymin': 0, 'xmax': 6_42, 'ymax': 4_76}},
								    ]					,							)

								SCREAMING_SNAKE_CASE_					      =							object_detector(
								    [
								        {
								            'image': 'http://images.cocodataset.org/val2017/000000039769.jpg',
								            'candidate_labels': ['cat', 'remote', 'couch'],
								        },
								        {
								            'image': 'http://images.cocodataset.org/val2017/000000039769.jpg',
								            'candidate_labels': ['cat', 'remote', 'couch'],
								        },
								    ]					,							)
								self.assertEqual(
								    nested_simplify(snake_case__					,							decimals=4  )					,							[
								        [
								            {'score': 0.28_68, 'label': 'cat', 'box': {'xmin': 3_24, 'ymin': 20, 'xmax': 6_40, 'ymax': 3_73}},
								            {'score': 0.2_77, 'label': 'remote', 'box': {'xmin': 40, 'ymin': 72, 'xmax': 1_77, 'ymax': 1_15}},
								            {'score': 0.25_37, 'label': 'cat', 'box': {'xmin': 1, 'ymin': 55, 'xmax': 3_15, 'ymax': 4_72}},
								            {'score': 0.14_74, 'label': 'remote', 'box': {'xmin': 3_35, 'ymin': 74, 'xmax': 3_71, 'ymax': 1_87}},
								            {'score': 0.12_08, 'label': 'couch', 'box': {'xmin': 4, 'ymin': 0, 'xmax': 6_42, 'ymax': 4_76}},
								        ],
								        [
								            {'score': 0.28_68, 'label': 'cat', 'box': {'xmin': 3_24, 'ymin': 20, 'xmax': 6_40, 'ymax': 3_73}},
								            {'score': 0.2_77, 'label': 'remote', 'box': {'xmin': 40, 'ymin': 72, 'xmax': 1_77, 'ymax': 1_15}},
								            {'score': 0.25_37, 'label': 'cat', 'box': {'xmin': 1, 'ymin': 55, 'xmax': 3_15, 'ymax': 4_72}},
								            {'score': 0.14_74, 'label': 'remote', 'box': {'xmin': 3_35, 'ymin': 74, 'xmax': 3_71, 'ymax': 1_87}},
								            {'score': 0.12_08, 'label': 'couch', 'box': {'xmin': 4, 'ymin': 0, 'xmax': 6_42, 'ymax': 4_76}},
								        ],
								    ]					,							)



						@require_tf
						@unittest.skip('Zero Shot Object Detection not implemented in TF'  )
						def 	__a					(			self	:   Union[str, Any]  ):

								"""simple docstring"""

								pass



						@require_torch
						@slow
						def 	__a					(			self	:   Union[str, Any]  ):

								"""simple docstring"""

								SCREAMING_SNAKE_CASE_					      =							0.2
								SCREAMING_SNAKE_CASE_					      =							pipeline('zero-shot-object-detection'  )

								SCREAMING_SNAKE_CASE_					      =							object_detector(
								    'http://images.cocodataset.org/val2017/000000039769.jpg'					,							candidate_labels=['cat', 'remote', 'couch']					,							threshold=snake_case__					,							)
								self.assertEqual(
								    nested_simplify(snake_case__					,							decimals=4  )					,							[
								        {'score': 0.28_68, 'label': 'cat', 'box': {'xmin': 3_24, 'ymin': 20, 'xmax': 6_40, 'ymax': 3_73}},
								        {'score': 0.2_77, 'label': 'remote', 'box': {'xmin': 40, 'ymin': 72, 'xmax': 1_77, 'ymax': 1_15}},
								        {'score': 0.25_37, 'label': 'cat', 'box': {'xmin': 1, 'ymin': 55, 'xmax': 3_15, 'ymax': 4_72}},
								    ]					,							)






						@require_torch
						@slow
						def 	__a					(			self	:   Tuple  ):

								"""simple docstring"""

								SCREAMING_SNAKE_CASE_					      =							2
								SCREAMING_SNAKE_CASE_					      =							pipeline('zero-shot-object-detection'  )

								SCREAMING_SNAKE_CASE_					      =							object_detector(
								    'http://images.cocodataset.org/val2017/000000039769.jpg'					,							candidate_labels=['cat', 'remote', 'couch']					,							top_k=snake_case__					,							)
								self.assertEqual(
								    nested_simplify(snake_case__					,							decimals=4  )					,							[
								        {'score': 0.28_68, 'label': 'cat', 'box': {'xmin': 3_24, 'ymin': 20, 'xmax': 6_40, 'ymax': 3_73}},
								        {'score': 0.2_77, 'label': 'remote', 'box': {'xmin': 40, 'ymin': 72, 'xmax': 1_77, 'ymax': 1_15}},
								    ]					,							)