





import numpy as np
from scipy.spatial.distance import cdist
from sklearn.metrics import fa_score

import datasets


UpperCamelCase__      ='\\n    @inproceedings{kakwani2020indicnlpsuite,\n    title={{IndicNLPSuite: Monolingual Corpora, Evaluation Benchmarks and Pre-trained Multilingual Language Models for Indian Languages}},\n    author={Divyanshu Kakwani and Anoop Kunchukuttan and Satish Golla and Gokul N.C. and Avik Bhattacharyya and Mitesh M. Khapra and Pratyush Kumar},\n    year={2020},\n    booktitle={Findings of EMNLP},\n}\n'

UpperCamelCase__      ='\\n    IndicGLUE is a natural language understanding benchmark for Indian languages. It contains a wide\n    variety of tasks and covers 11 major Indian languages - as, bn, gu, hi, kn, ml, mr, or, pa, ta, te.\n'

UpperCamelCase__      ='\nCompute IndicGLUE evaluation metric associated to each IndicGLUE dataset.\nArgs:\n    predictions: list of predictions to score (as int64),\n        except for \'cvit-mkb-clsr\' where each prediction is a vector (of float32).\n    references: list of ground truth labels corresponding to the predictions (as int64),\n        except for \'cvit-mkb-clsr\' where each reference is a vector (of float32).\nReturns: depending on the IndicGLUE subset, one or several of:\n    "accuracy": Accuracy\n    "f1": F1 score\n    "precision": Precision@10\nExamples:\n\n    >>> indic_glue_metric = datasets.load_metric(\'indic_glue\', \'wnli\')  # \'wnli\' or any of ["copa", "sna", "csqa", "wstp", "inltkh", "bbca", "iitp-mr", "iitp-pr", "actsa-sc", "md"]\n    >>> references = [0, 1]\n    >>> predictions = [0, 1]\n    >>> results = indic_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {\'accuracy\': 1.0}\n\n    >>> indic_glue_metric = datasets.load_metric(\'indic_glue\', \'wiki-ner\')\n    >>> references = [0, 1]\n    >>> predictions = [0, 1]\n    >>> results = indic_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {\'accuracy\': 1.0, \'f1\': 1.0}\n\n    >>> indic_glue_metric = datasets.load_metric(\'indic_glue\', \'cvit-mkb-clsr\')\n    >>> references = [[0.5, 0.5, 0.5], [0.1, 0.2, 0.3]]\n    >>> predictions = [[0.5, 0.5, 0.5], [0.1, 0.2, 0.3]]\n    >>> results = indic_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {\'precision@10\': 1.0}\n\n'






def     lowerCamelCase__			(__lowerCamelCase,       __lowerCamelCase	):
     return float((preds == labels).mean()	)






def     lowerCamelCase__			(__lowerCamelCase,       __lowerCamelCase	):
     _SCREAMING_SNAKE_CASE       :	Dict						    =							simple_accuracy(__lowerCamelCase,       __lowerCamelCase	)
     _SCREAMING_SNAKE_CASE       :	List[str]						    =							float(fa_score(y_true=__lowerCamelCase,       y_pred=__lowerCamelCase	)	)
     return {
         "accuracy": acc,
         "f1": fa,
     }






def     lowerCamelCase__			(__lowerCamelCase,       __lowerCamelCase	):
     _SCREAMING_SNAKE_CASE       :	Any						    =							np.array(__lowerCamelCase	)
     _SCREAMING_SNAKE_CASE       :	Tuple						    =							np.array(__lowerCamelCase	)
     _SCREAMING_SNAKE_CASE       :	Optional[Any]						    =							en_sentvecs.shape[0]

     # mean centering
     _SCREAMING_SNAKE_CASE       :	str						    =							en_sentvecs - np.mean(__lowerCamelCase,       axis=0	)
     _SCREAMING_SNAKE_CASE       :	Optional[Any]						    =							in_sentvecs - np.mean(__lowerCamelCase,       axis=0	)

     _SCREAMING_SNAKE_CASE       :	Optional[Any]						    =							cdist(__lowerCamelCase,       __lowerCamelCase,       "cosine"	)
     _SCREAMING_SNAKE_CASE       :	Optional[int]						    =							np.array(range(__lowerCamelCase	)	)
     _SCREAMING_SNAKE_CASE       :	int						    =							sim.argsort(axis=1	)[:, :10]
     _SCREAMING_SNAKE_CASE       :	List[Any]						    =							np.any(preds == actual[:, None],       axis=1	)
     return float(matches.mean()	)





@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION  ,     _KWARGS_DESCRIPTION    )
class 			lowerCAmelCase__( datasets.Metric    ):






       '''simple docstring'''


       def     UpperCamelCase_			(      self							)			->     int:
            if self.config_name not in [
                "wnli",
                "copa",
                "sna",
                "csqa",
                "wstp",
                "inltkh",
                "bbca",
                "cvit-mkb-clsr",
                "iitp-mr",
                "iitp-pr",
                "actsa-sc",
                "md",
                "wiki-ner",
            ]:
                 raise KeyError(
                     "You should supply a configuration name selected in "
                     "[\"wnli\", \"copa\", \"sna\", \"csqa\", \"wstp\", \"inltkh\", \"bbca\", "
                     "\"cvit-mkb-clsr\", \"iitp-mr\", \"iitp-pr\", \"actsa-sc\", \"md\", "
                     "\"wiki-ner\"]"							)
            return datasets.MetricInfo(
                description=_DESCRIPTION    ,						citation=_CITATION    ,						inputs_description=_KWARGS_DESCRIPTION    ,						features=datasets.Features(
                    {
                        "predictions": datasets.Value("int64"							)
                        if self.config_name != "cvit-mkb-clsr"
                        else datasets.Sequence(datasets.Value("float32"							)							),
                        "references": datasets.Value("int64"							)
                        if self.config_name != "cvit-mkb-clsr"
                        else datasets.Sequence(datasets.Value("float32"							)							),
                    }							)    ,						codebase_urls=[]    ,						reference_urls=[]    ,						format="numpy" if self.config_name != "cvit-mkb-clsr" else None    ,						)







       def     UpperCamelCase_			(      self    ,						__lowerCamelCase    ,						__lowerCamelCase							)			->     Any:
            if self.config_name == "cvit-mkb-clsr":
                 return {"precision@10": precision_at_aa(__lowerCamelCase    ,						__lowerCamelCase							)}
            elif self.config_name in ["wiki-ner"]:
                 return acc_and_fa(__lowerCamelCase    ,						__lowerCamelCase							)
            elif self.config_name in [
                "wnli",
                "copa",
                "sna",
                "csqa",
                "wstp",
                "inltkh",
                "bbca",
                "iitp-mr",
                "iitp-pr",
                "actsa-sc",
                "md",
            ]:
                 return {"accuracy": simple_accuracy(__lowerCamelCase    ,						__lowerCamelCase							)}
            else:
                 raise KeyError(
                     "You should supply a configuration name selected in "
                     "[\"wnli\", \"copa\", \"sna\", \"csqa\", \"wstp\", \"inltkh\", \"bbca\", "
                     "\"cvit-mkb-clsr\", \"iitp-mr\", \"iitp-pr\", \"actsa-sc\", \"md\", "
                     "\"wiki-ner\"]"							)






import inspect
import unittest

from transformers import DecisionTransformerConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
       import torch

       from transformers import DecisionTransformerModel
       from transformers.models.decision_transformer.modeling_decision_transformer import (
           DECISION_TRANSFORMER_PRETRAINED_MODEL_ARCHIVE_LIST,
       )





class 			lowerCAmelCase__:






       '''simple docstring'''


       def __init__(      self    ,						__lowerCamelCase    ,						__lowerCamelCase=1_3    ,						__lowerCamelCase=7    ,						__lowerCamelCase=6    ,						__lowerCamelCase=1_7    ,						__lowerCamelCase=2_3    ,						__lowerCamelCase=1_1    ,						__lowerCamelCase=True    ,						)			->     Optional[int]:
            _SCREAMING_SNAKE_CASE       :	List[Any]						    =							parent
            _SCREAMING_SNAKE_CASE       :	Optional[Any]						    =							batch_size
            _SCREAMING_SNAKE_CASE       :	int						    =							seq_length
            _SCREAMING_SNAKE_CASE       :	Optional[Any]						    =							act_dim
            _SCREAMING_SNAKE_CASE       :	Optional[int]						    =							state_dim
            _SCREAMING_SNAKE_CASE       :	Optional[int]						    =							hidden_size
            _SCREAMING_SNAKE_CASE       :	Any						    =							max_length
            _SCREAMING_SNAKE_CASE       :	Optional[int]						    =							is_training




       def     UpperCamelCase_			(      self							)			->     List[Any]:
            _SCREAMING_SNAKE_CASE       :	List[str]						    =							floats_tensor((self.batch_size, self.seq_length, self.state_dim)							)
            _SCREAMING_SNAKE_CASE       :	Union[str, Any]						    =							floats_tensor((self.batch_size, self.seq_length, self.act_dim)							)
            _SCREAMING_SNAKE_CASE       :	Union[str, Any]						    =							floats_tensor((self.batch_size, self.seq_length, 1)							)
            _SCREAMING_SNAKE_CASE       :	str						    =							floats_tensor((self.batch_size, self.seq_length, 1)							)
            _SCREAMING_SNAKE_CASE       :	str						    =							ids_tensor((self.batch_size, self.seq_length)    ,						vocab_size=1_0_0_0							)
            _SCREAMING_SNAKE_CASE       :	List[Any]						    =							random_attention_mask((self.batch_size, self.seq_length)							)

            _SCREAMING_SNAKE_CASE       :	List[str]						    =							self.get_config()

            return (
                config,
                states,
                actions,
                rewards,
                returns_to_go,
                timesteps,
                attention_mask,
            )




       def     UpperCamelCase_			(      self							)			->     List[Any]:
            return DecisionTransformerConfig(
                batch_size=self.batch_size    ,						seq_length=self.seq_length    ,						act_dim=self.act_dim    ,						state_dim=self.state_dim    ,						hidden_size=self.hidden_size    ,						max_length=self.max_length    ,						)




       def     UpperCamelCase_			(      self    ,						__lowerCamelCase    ,						__lowerCamelCase    ,						__lowerCamelCase    ,						__lowerCamelCase    ,						__lowerCamelCase    ,						__lowerCamelCase    ,						__lowerCamelCase    ,						)			->     Any:
            _SCREAMING_SNAKE_CASE       :	List[Any]						    =							DecisionTransformerModel(config=__lowerCamelCase							)
            model.to(__lowerCamelCase							)
            model.eval()
            _SCREAMING_SNAKE_CASE       :	List[str]						    =							model(__lowerCamelCase    ,						__lowerCamelCase    ,						__lowerCamelCase    ,						__lowerCamelCase    ,						__lowerCamelCase    ,						__lowerCamelCase							)

            self.parent.assertEqual(result.state_preds.shape    ,						states.shape							)
            self.parent.assertEqual(result.action_preds.shape    ,						actions.shape							)
            self.parent.assertEqual(result.return_preds.shape    ,						returns_to_go.shape							)
            self.parent.assertEqual(
                result.last_hidden_state.shape    ,						(self.batch_size, self.seq_length * 3, self.hidden_size)							)  # seq length *3 as there are 3 modelities: states, returns and actions







       def     UpperCamelCase_			(      self							)			->     Dict:
            _SCREAMING_SNAKE_CASE       :	int						    =							self.prepare_config_and_inputs()
            (
                (
                _SCREAMING_SNAKE_CASE
            )    ,			(
                _SCREAMING_SNAKE_CASE
            )    ,			(
                _SCREAMING_SNAKE_CASE
            )    ,			(
                _SCREAMING_SNAKE_CASE
            )    ,			(
                _SCREAMING_SNAKE_CASE
            )    ,			(
                _SCREAMING_SNAKE_CASE
            )    ,			(
                _SCREAMING_SNAKE_CASE
            )    ,			
            )       :	Optional[Any]						    =							config_and_inputs
            _SCREAMING_SNAKE_CASE       :	Any						    =							{
                "states": states,
                "actions": actions,
                "rewards": rewards,
                "returns_to_go": returns_to_go,
                "timesteps": timesteps,
                "attention_mask": attention_mask,
            }
            return config, inputs_dict





@require_torch
class 			lowerCAmelCase__( __lowercase  ,     __lowercase  ,     __lowercase  ,     unittest.TestCase    ):






       '''simple docstring'''


       __snake_case 			=    (DecisionTransformerModel,) if is_torch_available() else ()
       __snake_case 			=    ()
       __snake_case 			=    {'feature-extraction': DecisionTransformerModel} if is_torch_available() else {}

       # Ignoring of a failing test from GenerationTesterMixin, as the model does not use inputs_ids
       __snake_case 			=    False

       # Ignoring of a failing tests from ModelTesterMixin, as the model does not implement these features
       __snake_case 			=    False
       __snake_case 			=    False
       __snake_case 			=    False
       __snake_case 			=    False
       __snake_case 			=    False
       __snake_case 			=    False
       __snake_case 			=    False
       __snake_case 			=    False
       __snake_case 			=    False




       def     UpperCamelCase_			(      self							)			->     Union[str, Any]:
            _SCREAMING_SNAKE_CASE       :	int						    =							DecisionTransformerModelTester(self							)
            _SCREAMING_SNAKE_CASE       :	Optional[Any]						    =							ConfigTester(self    ,						config_class=__lowerCamelCase    ,						hidden_size=3_7							)




       def     UpperCamelCase_			(      self							)			->     Optional[int]:
            self.config_tester.run_common_tests()




       def     UpperCamelCase_			(      self							)			->     List[Any]:
            _SCREAMING_SNAKE_CASE       :	str						    =							self.model_tester.prepare_config_and_inputs()
            self.model_tester.create_and_check_model(*__lowerCamelCase							)




       @slow
       def     UpperCamelCase_			(      self							)			->     Dict:
            for model_name in DECISION_TRANSFORMER_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
                 _SCREAMING_SNAKE_CASE       :	str						    =							DecisionTransformerModel.from_pretrained(__lowerCamelCase							)
                 self.assertIsNotNone(__lowerCamelCase							)







       def     UpperCamelCase_			(      self							)			->     Optional[int]:
            _SCREAMING_SNAKE_CASE    ,			_SCREAMING_SNAKE_CASE       :	Dict						    =							self.model_tester.prepare_config_and_inputs_for_common()

            for model_class in self.all_model_classes:
                 _SCREAMING_SNAKE_CASE       :	Tuple						    =							model_class(__lowerCamelCase							)
                 _SCREAMING_SNAKE_CASE       :	List[str]						    =							inspect.signature(model.forward							)
                 # signature.parameters is an OrderedDict => so arg_names order is deterministic
                 _SCREAMING_SNAKE_CASE       :	int						    =							[*signature.parameters.keys()]

                 _SCREAMING_SNAKE_CASE       :	Tuple						    =							[
                     "states",
                     "actions",
                     "rewards",
                     "returns_to_go",
                     "timesteps",
                     "attention_mask",
                 ]

                 self.assertListEqual(arg_names[: len(__lowerCamelCase							)]    ,						__lowerCamelCase							)





@require_torch
class 			lowerCAmelCase__( unittest.TestCase    ):






       '''simple docstring'''


       @slow
       def     UpperCamelCase_			(      self							)			->     Union[str, Any]:
            _SCREAMING_SNAKE_CASE       :	str						    =							2  # number of steps of autoregressive prediction we will perform
            _SCREAMING_SNAKE_CASE       :	Tuple						    =							1_0  # defined by the RL environment, may be normalized
            _SCREAMING_SNAKE_CASE       :	int						    =							DecisionTransformerModel.from_pretrained("edbeeching/decision-transformer-gym-hopper-expert"							)
            _SCREAMING_SNAKE_CASE       :	Union[str, Any]						    =							model.to(__lowerCamelCase							)
            _SCREAMING_SNAKE_CASE       :	Optional[Any]						    =							model.config
            torch.manual_seed(0							)
            _SCREAMING_SNAKE_CASE       :	Optional[Any]						    =							torch.randn(1    ,						1    ,						config.state_dim							).to(device=__lowerCamelCase    ,						dtype=torch.floataa							)  # env.reset()

            _SCREAMING_SNAKE_CASE       :	List[str]						    =							torch.tensor(
                [[0.24_2793, -0.2869_3074, 0.874_2613], [0.6781_5274, -0.0810_1085, -0.1295_2147]]    ,						device=__lowerCamelCase							)

            _SCREAMING_SNAKE_CASE       :	List[Any]						    =							torch.tensor(__lowerCamelCase    ,						device=__lowerCamelCase    ,						dtype=torch.floataa							).reshape(1    ,						1    ,						1							)
            _SCREAMING_SNAKE_CASE       :	List[str]						    =							state
            _SCREAMING_SNAKE_CASE       :	Union[str, Any]						    =							torch.zeros(1    ,						0    ,						config.act_dim    ,						device=__lowerCamelCase    ,						dtype=torch.floataa							)
            _SCREAMING_SNAKE_CASE       :	Dict						    =							torch.zeros(1    ,						0    ,						device=__lowerCamelCase    ,						dtype=torch.floataa							)
            _SCREAMING_SNAKE_CASE       :	Union[str, Any]						    =							torch.tensor(0    ,						device=__lowerCamelCase    ,						dtype=torch.long							).reshape(1    ,						1							)

            for step in range(__lowerCamelCase							):
                 _SCREAMING_SNAKE_CASE       :	int						    =							torch.cat([actions, torch.zeros(1    ,						1    ,						config.act_dim    ,						device=__lowerCamelCase							)]    ,						dim=1							)
                 _SCREAMING_SNAKE_CASE       :	Any						    =							torch.cat([rewards, torch.zeros(1    ,						1    ,						device=__lowerCamelCase							)]    ,						dim=1							)

                 _SCREAMING_SNAKE_CASE       :	Union[str, Any]						    =							torch.ones(1    ,						states.shape[1]							).to(dtype=torch.long    ,						device=states.device							)

                 with torch.no_grad():
                      _SCREAMING_SNAKE_CASE    ,			_SCREAMING_SNAKE_CASE    ,			_SCREAMING_SNAKE_CASE       :	Optional[Any]						    =							model(
                          states=__lowerCamelCase    ,						actions=__lowerCamelCase    ,						rewards=__lowerCamelCase    ,						returns_to_go=__lowerCamelCase    ,						timesteps=__lowerCamelCase    ,						attention_mask=__lowerCamelCase    ,						return_dict=__lowerCamelCase    ,						)

                 self.assertEqual(action_pred.shape    ,						actions.shape							)
                 self.assertTrue(torch.allclose(action_pred[0, -1]    ,						expected_outputs[step]    ,						atol=1E-4							)							)
                 _SCREAMING_SNAKE_CASE    ,			_SCREAMING_SNAKE_CASE    ,			_SCREAMING_SNAKE_CASE    ,			_SCREAMING_SNAKE_CASE       :	Tuple						    =							(  # env.step(action)
                     torch.randn(1    ,						1    ,						config.state_dim							).to(device=__lowerCamelCase    ,						dtype=torch.floataa							),
                     1.0,
                     False,
                     {},
                 )

                 _SCREAMING_SNAKE_CASE       :	Optional[Any]						    =							action_pred[0, -1]
                 _SCREAMING_SNAKE_CASE       :	Optional[Any]						    =							torch.cat([states, state]    ,						dim=1							)
                 _SCREAMING_SNAKE_CASE       :	List[str]						    =							returns_to_go[0, -1] - reward
                 _SCREAMING_SNAKE_CASE       :	str						    =							torch.cat([returns_to_go, pred_return.reshape(1    ,						1    ,						1							)]    ,						dim=1							)
                 _SCREAMING_SNAKE_CASE       :	Tuple						    =							torch.cat(
                     [timesteps, torch.ones((1, 1)    ,						device=__lowerCamelCase    ,						dtype=torch.long							) * (step + 1)]    ,						dim=1							)