



"""simple docstring"""




import numpy as np
from scipy.spatial.distance import cdist
from sklearn.metrics import fa_score

import datasets


_lowerCAmelCase							:   Optional[Any]				=				"\\n    @inproceedings{kakwani2020indicnlpsuite,\n    title={{IndicNLPSuite: Monolingual Corpora, Evaluation Benchmarks and Pre-trained Multilingual Language Models for Indian Languages}},\n    author={Divyanshu Kakwani and Anoop Kunchukuttan and Satish Golla and Gokul N.C. and Avik Bhattacharyya and Mitesh M. Khapra and Pratyush Kumar},\n    year={2020},\n    booktitle={Findings of EMNLP},\n}\n"

_lowerCAmelCase							:   List[Any]				=				"\\n    IndicGLUE is a natural language understanding benchmark for Indian languages. It contains a wide\n    variety of tasks and covers 11 major Indian languages - as, bn, gu, hi, kn, ml, mr, or, pa, ta, te.\n"

_lowerCAmelCase							:   Any				=				"\nCompute IndicGLUE evaluation metric associated to each IndicGLUE dataset.\nArgs:\n    predictions: list of predictions to score (as int64),\n        except for 'cvit-mkb-clsr' where each prediction is a vector (of float32).\n    references: list of ground truth labels corresponding to the predictions (as int64),\n        except for 'cvit-mkb-clsr' where each reference is a vector (of float32).\nReturns: depending on the IndicGLUE subset, one or several of:\n    \"accuracy\": Accuracy\n    \"f1\": F1 score\n    \"precision\": Precision@10\nExamples:\n\n    >>> indic_glue_metric = datasets.load_metric('indic_glue', 'wnli')  # 'wnli' or any of [\"copa\", \"sna\", \"csqa\", \"wstp\", \"inltkh\", \"bbca\", \"iitp-mr\", \"iitp-pr\", \"actsa-sc\", \"md\"]\n    >>> references = [0, 1]\n    >>> predictions = [0, 1]\n    >>> results = indic_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {'accuracy': 1.0}\n\n    >>> indic_glue_metric = datasets.load_metric('indic_glue', 'wiki-ner')\n    >>> references = [0, 1]\n    >>> predictions = [0, 1]\n    >>> results = indic_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {'accuracy': 1.0, 'f1': 1.0}\n\n    >>> indic_glue_metric = datasets.load_metric('indic_glue', 'cvit-mkb-clsr')\n    >>> references = [[0.5, 0.5, 0.5], [0.1, 0.2, 0.3]]\n    >>> predictions = [[0.5, 0.5, 0.5], [0.1, 0.2, 0.3]]\n    >>> results = indic_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {'precision@10': 1.0}\n\n"

def 		__snake_case		( SCREAMING_SNAKE_CASE__      :		str		,    SCREAMING_SNAKE_CASE__      :		Optional[Any]							)    ->  Union[str, Any]:





     '''simple docstring'''

     return float((preds == labels).mean()							)

def 		__snake_case		( SCREAMING_SNAKE_CASE__      :		List[Any]		,    SCREAMING_SNAKE_CASE__      :		str							)    ->  Optional[Any]:





     '''simple docstring'''

     _UpperCAmelCase     :      Union[str, Any]														=       simple_accuracy(SCREAMING_SNAKE_CASE__		,    SCREAMING_SNAKE_CASE__							)
     _UpperCAmelCase     :      List[str]														=       float(fa_score(y_true=SCREAMING_SNAKE_CASE__		,    y_pred=SCREAMING_SNAKE_CASE__							)							)
     return {
         "accuracy": acc,
         "f1": fa,
     }

def 		__snake_case		( SCREAMING_SNAKE_CASE__      :		Tuple		,    SCREAMING_SNAKE_CASE__      :		Optional[int]							)    ->  Optional[Any]:





     '''simple docstring'''

     _UpperCAmelCase     :      Union[str, Any]														=       np.array(SCREAMING_SNAKE_CASE__							)
     _UpperCAmelCase     :      List[str]														=       np.array(SCREAMING_SNAKE_CASE__							)
     _UpperCAmelCase     :      Optional[Any]														=       en_sentvecs.shape[0]

     # mean centering
     _UpperCAmelCase     :      List[Any]														=       en_sentvecs - np.mean(SCREAMING_SNAKE_CASE__		,    axis=0							)
     _UpperCAmelCase     :      Optional[int]														=       in_sentvecs - np.mean(SCREAMING_SNAKE_CASE__		,    axis=0							)

     _UpperCAmelCase     :      Any														=       cdist(SCREAMING_SNAKE_CASE__		,    SCREAMING_SNAKE_CASE__		,    "cosine"							)
     _UpperCAmelCase     :      Optional[Any]														=       np.array(range(SCREAMING_SNAKE_CASE__							)							)
     _UpperCAmelCase     :      Tuple														=       sim.argsort(axis=1							)[:, :10]
     _UpperCAmelCase     :      Optional[Any]														=       np.any(preds == actual[:, None]		,    axis=1							)
     return float(matches.mean()							)






@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION			,     _KWARGS_DESCRIPTION		)
class       UpperCAmelCase_    (							datasets.Metric		):



      def 		snake_case_   (	self	:      int		):
           if self.config_name not in [
               "wnli",
               "copa",
               "sna",
               "csqa",
               "wstp",
               "inltkh",
               "bbca",
               "cvit-mkb-clsr",
               "iitp-mr",
               "iitp-pr",
               "actsa-sc",
               "md",
               "wiki-ner",
           ]:
                raise KeyError(
                    "You should supply a configuration name selected in "
                    "[\"wnli\", \"copa\", \"sna\", \"csqa\", \"wstp\", \"inltkh\", \"bbca\", "
                    "\"cvit-mkb-clsr\", \"iitp-mr\", \"iitp-pr\", \"actsa-sc\", \"md\", "
                    "\"wiki-ner\"]"		)
           return datasets.MetricInfo(
               description=_DESCRIPTION	,      citation=_CITATION	,      inputs_description=_KWARGS_DESCRIPTION	,      features=datasets.Features(
                   {
                       "predictions": datasets.Value("int64"		)
                       if self.config_name != "cvit-mkb-clsr"
                       else datasets.Sequence(datasets.Value("float32"		)		),
                       "references": datasets.Value("int64"		)
                       if self.config_name != "cvit-mkb-clsr"
                       else datasets.Sequence(datasets.Value("float32"		)		),
                   }		)	,      codebase_urls=[]	,      reference_urls=[]	,      format="numpy" if self.config_name != "cvit-mkb-clsr" else None	,      )



      def 		snake_case_   (	self	:      Any	,      A	:      int	,      A	:      int		):
           if self.config_name == "cvit-mkb-clsr":
                return {"precision@10": precision_at_aa(A	,      A		)}
           elif self.config_name in ["wiki-ner"]:
                return acc_and_fa(A	,      A		)
           elif self.config_name in [
               "wnli",
               "copa",
               "sna",
               "csqa",
               "wstp",
               "inltkh",
               "bbca",
               "iitp-mr",
               "iitp-pr",
               "actsa-sc",
               "md",
           ]:
                return {"accuracy": simple_accuracy(A	,      A		)}
           else:
                raise KeyError(
                    "You should supply a configuration name selected in "
                    "[\"wnli\", \"copa\", \"sna\", \"csqa\", \"wstp\", \"inltkh\", \"bbca\", "
                    "\"cvit-mkb-clsr\", \"iitp-mr\", \"iitp-pr\", \"actsa-sc\", \"md\", "
                    "\"wiki-ner\"]"		)





"""simple docstring"""




import unittest

from transformers import is_vision_available
from transformers.pipelines import pipeline
from transformers.testing_utils import (
    is_pipeline_test,
    nested_simplify,
    require_tf,
    require_torch,
    require_vision,
    slow,
)

from .test_pipelines_common import ANY


if is_vision_available():
 from PIL import Image
else:






 class       UpperCAmelCase_    :



       @staticmethod
       def 		snake_case_   (	*A	:      Union[str, Any]	,      **A	:      Dict		):
            pass






@is_pipeline_test
@require_vision
class       UpperCAmelCase_    (							unittest.TestCase		):



      @require_torch
      def 		snake_case_   (	self	:      Union[str, Any]		):
           _UpperCAmelCase     :      Union[str, Any]														=       pipeline(
               model="hf-internal-testing/tiny-random-clip-zero-shot-image-classification"	,      )
           _UpperCAmelCase     :      Union[str, Any]														=       Image.open("./tests/fixtures/tests_samples/COCO/000000039769.png"		)
           _UpperCAmelCase     :      Tuple														=       image_classifier(A	,      candidate_labels=["a", "b", "c"]		)

           # The floating scores are so close, we enter floating error approximation and the order is not guaranteed across
           # python and torch versions.
           self.assertIn(
               nested_simplify(A		)	,      [
                   [{"score": 0.333, "label": "a"}, {"score": 0.333, "label": "b"}, {"score": 0.333, "label": "c"}],
                   [{"score": 0.333, "label": "a"}, {"score": 0.333, "label": "c"}, {"score": 0.333, "label": "b"}],
               ]	,      )

           _UpperCAmelCase     :      str														=       image_classifier([image] * 5	,      candidate_labels=["A", "B", "C"]	,      batch_size=2		)
           self.assertEqual(
               nested_simplify(A		)	,      [
                   [
                       {"score": 0.333, "label": ANY(A		)},
                       {"score": 0.333, "label": ANY(A		)},
                       {"score": 0.333, "label": ANY(A		)},
                   ],
                   [
                       {"score": 0.333, "label": ANY(A		)},
                       {"score": 0.333, "label": ANY(A		)},
                       {"score": 0.333, "label": ANY(A		)},
                   ],
                   [
                       {"score": 0.333, "label": ANY(A		)},
                       {"score": 0.333, "label": ANY(A		)},
                       {"score": 0.333, "label": ANY(A		)},
                   ],
                   [
                       {"score": 0.333, "label": ANY(A		)},
                       {"score": 0.333, "label": ANY(A		)},
                       {"score": 0.333, "label": ANY(A		)},
                   ],
                   [
                       {"score": 0.333, "label": ANY(A		)},
                       {"score": 0.333, "label": ANY(A		)},
                       {"score": 0.333, "label": ANY(A		)},
                   ],
               ]	,      )



      @require_tf
      def 		snake_case_   (	self	:      Any		):
           _UpperCAmelCase     :      List[str]														=       pipeline(
               model="hf-internal-testing/tiny-random-clip-zero-shot-image-classification"	,      framework="tf"		)
           _UpperCAmelCase     :      int														=       Image.open("./tests/fixtures/tests_samples/COCO/000000039769.png"		)
           _UpperCAmelCase     :      Union[str, Any]														=       image_classifier(A	,      candidate_labels=["a", "b", "c"]		)

           self.assertEqual(
               nested_simplify(A		)	,      [{"score": 0.333, "label": "a"}, {"score": 0.333, "label": "b"}, {"score": 0.333, "label": "c"}]	,      )

           _UpperCAmelCase     :      Optional[Any]														=       image_classifier([image] * 5	,      candidate_labels=["A", "B", "C"]	,      batch_size=2		)
           self.assertEqual(
               nested_simplify(A		)	,      [
                   [
                       {"score": 0.333, "label": ANY(A		)},
                       {"score": 0.333, "label": ANY(A		)},
                       {"score": 0.333, "label": ANY(A		)},
                   ],
                   [
                       {"score": 0.333, "label": ANY(A		)},
                       {"score": 0.333, "label": ANY(A		)},
                       {"score": 0.333, "label": ANY(A		)},
                   ],
                   [
                       {"score": 0.333, "label": ANY(A		)},
                       {"score": 0.333, "label": ANY(A		)},
                       {"score": 0.333, "label": ANY(A		)},
                   ],
                   [
                       {"score": 0.333, "label": ANY(A		)},
                       {"score": 0.333, "label": ANY(A		)},
                       {"score": 0.333, "label": ANY(A		)},
                   ],
                   [
                       {"score": 0.333, "label": ANY(A		)},
                       {"score": 0.333, "label": ANY(A		)},
                       {"score": 0.333, "label": ANY(A		)},
                   ],
               ]	,      )



      @slow
      @require_torch
      def 		snake_case_   (	self	:      Any		):
           _UpperCAmelCase     :      Optional[Any]														=       pipeline(
               task="zero-shot-image-classification"	,      model="openai/clip-vit-base-patch32"	,      )
           # This is an image of 2 cats with remotes and no planes
           _UpperCAmelCase     :      Dict														=       Image.open("./tests/fixtures/tests_samples/COCO/000000039769.png"		)
           _UpperCAmelCase     :      List[Any]														=       image_classifier(A	,      candidate_labels=["cat", "plane", "remote"]		)

           self.assertEqual(
               nested_simplify(A		)	,      [
                   {"score": 0.511, "label": "remote"},
                   {"score": 0.485, "label": "cat"},
                   {"score": 0.004, "label": "plane"},
               ]	,      )

           _UpperCAmelCase     :      Tuple														=       image_classifier([image] * 5	,      candidate_labels=["cat", "plane", "remote"]	,      batch_size=2		)
           self.assertEqual(
               nested_simplify(A		)	,      [
                   [
                       {"score": 0.511, "label": "remote"},
                       {"score": 0.485, "label": "cat"},
                       {"score": 0.004, "label": "plane"},
                   ],
               ]
               * 5	,      )



      @slow
      @require_tf
      def 		snake_case_   (	self	:      Any		):
           _UpperCAmelCase     :      List[Any]														=       pipeline(
               task="zero-shot-image-classification"	,      model="openai/clip-vit-base-patch32"	,      framework="tf"		)
           # This is an image of 2 cats with remotes and no planes
           _UpperCAmelCase     :      Union[str, Any]														=       Image.open("./tests/fixtures/tests_samples/COCO/000000039769.png"		)
           _UpperCAmelCase     :      Union[str, Any]														=       image_classifier(A	,      candidate_labels=["cat", "plane", "remote"]		)
           self.assertEqual(
               nested_simplify(A		)	,      [
                   {"score": 0.511, "label": "remote"},
                   {"score": 0.485, "label": "cat"},
                   {"score": 0.004, "label": "plane"},
               ]	,      )

           _UpperCAmelCase     :      Union[str, Any]														=       image_classifier([image] * 5	,      candidate_labels=["cat", "plane", "remote"]	,      batch_size=2		)
           self.assertEqual(
               nested_simplify(A		)	,      [
                   [
                       {"score": 0.511, "label": "remote"},
                       {"score": 0.485, "label": "cat"},
                       {"score": 0.004, "label": "plane"},
                   ],
               ]
               * 5	,      )
