

"""simple docstring"""




import numpy as np
from scipy.spatial.distance import cdist
from sklearn.metrics import fa_score

import datasets


_lowercase   :	str												=						'\\n    @inproceedings{kakwani2020indicnlpsuite,\n    title={{IndicNLPSuite: Monolingual Corpora, Evaluation Benchmarks and Pre-trained Multilingual Language Models for Indian Languages}},\n    author={Divyanshu Kakwani and Anoop Kunchukuttan and Satish Golla and Gokul N.C. and Avik Bhattacharyya and Mitesh M. Khapra and Pratyush Kumar},\n    year={2020},\n    booktitle={Findings of EMNLP},\n}\n'

_lowercase   :	Optional[int]												=						'\\n    IndicGLUE is a natural language understanding benchmark for Indian languages. It contains a wide\n    variety of tasks and covers 11 major Indian languages - as, bn, gu, hi, kn, ml, mr, or, pa, ta, te.\n'

_lowercase   :	str												=						'\nCompute IndicGLUE evaluation metric associated to each IndicGLUE dataset.\nArgs:\n    predictions: list of predictions to score (as int64),\n        except for \'cvit-mkb-clsr\' where each prediction is a vector (of float32).\n    references: list of ground truth labels corresponding to the predictions (as int64),\n        except for \'cvit-mkb-clsr\' where each reference is a vector (of float32).\nReturns: depending on the IndicGLUE subset, one or several of:\n    "accuracy": Accuracy\n    "f1": F1 score\n    "precision": Precision@10\nExamples:\n\n    >>> indic_glue_metric = datasets.load_metric(\'indic_glue\', \'wnli\')  # \'wnli\' or any of ["copa", "sna", "csqa", "wstp", "inltkh", "bbca", "iitp-mr", "iitp-pr", "actsa-sc", "md"]\n    >>> references = [0, 1]\n    >>> predictions = [0, 1]\n    >>> results = indic_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {\'accuracy\': 1.0}\n\n    >>> indic_glue_metric = datasets.load_metric(\'indic_glue\', \'wiki-ner\')\n    >>> references = [0, 1]\n    >>> predictions = [0, 1]\n    >>> results = indic_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {\'accuracy\': 1.0, \'f1\': 1.0}\n\n    >>> indic_glue_metric = datasets.load_metric(\'indic_glue\', \'cvit-mkb-clsr\')\n    >>> references = [[0.5, 0.5, 0.5], [0.1, 0.2, 0.3]]\n    >>> predictions = [[0.5, 0.5, 0.5], [0.1, 0.2, 0.3]]\n    >>> results = indic_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {\'precision@10\': 1.0}\n\n'





def       lowercase__						(       snake_case_   :Optional[Any]			,		snake_case_   :Dict    ):
	return float((preds == labels).mean()    )





def       lowercase__						(       snake_case_   :Union[str, Any]			,		snake_case_   :str    ):
	__UpperCAmelCase      		= simple_accuracy(snake_case_			,		snake_case_    )
	__UpperCAmelCase      		= float(fa_score(y_true=snake_case_			,		y_pred=snake_case_    )    )
	return {
	    "accuracy": acc,
	    "f1": fa,
	}





def       lowercase__						(       snake_case_   :str			,		snake_case_   :List[Any]    ):
	__UpperCAmelCase      		= np.array(snake_case_    )
	__UpperCAmelCase      		= np.array(snake_case_    )
	__UpperCAmelCase      		= en_sentvecs.shape[0]

	# mean centering
	__UpperCAmelCase      		= en_sentvecs - np.mean(snake_case_			,		axis=0    )
	__UpperCAmelCase      		= in_sentvecs - np.mean(snake_case_			,		axis=0    )

	__UpperCAmelCase      		= cdist(snake_case_			,		snake_case_			,		'''cosine'''    )
	__UpperCAmelCase      		= np.array(range(snake_case_    )    )
	__UpperCAmelCase      		= sim.argsort(axis=1    )[:, :10]
	__UpperCAmelCase      		= np.any(preds == actual[:, None]			,		axis=1    )
	return float(matches.mean()    )




@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION		,    _KWARGS_DESCRIPTION       )
class      _UpperCAmelCase   (							datasets.Metric       ):



	def        a  (  self     :       Union[str, Any]						):
		if self.config_name not in [
		    "wnli",
		    "copa",
		    "sna",
		    "csqa",
		    "wstp",
		    "inltkh",
		    "bbca",
		    "cvit-mkb-clsr",
		    "iitp-mr",
		    "iitp-pr",
		    "actsa-sc",
		    "md",
		    "wiki-ner",
		]:
			raise KeyError(
			    '''You should supply a configuration name selected in '''
			    '''["wnli", "copa", "sna", "csqa", "wstp", "inltkh", "bbca", '''
			    '''"cvit-mkb-clsr", "iitp-mr", "iitp-pr", "actsa-sc", "md", '''
			    '''"wiki-ner"]'''						)
		return datasets.MetricInfo(
		    description=_DESCRIPTION			,    citation=_CITATION			,    inputs_description=_KWARGS_DESCRIPTION			,    features=datasets.Features(
		        {
		            '''predictions''': datasets.Value('''int64'''						)
		            if self.config_name != '''cvit-mkb-clsr'''
		            else datasets.Sequence(datasets.Value('''float32'''						)						),
		            '''references''': datasets.Value('''int64'''						)
		            if self.config_name != '''cvit-mkb-clsr'''
		            else datasets.Sequence(datasets.Value('''float32'''						)						),
		        }						)			,    codebase_urls=[]			,    reference_urls=[]			,    format='''numpy''' if self.config_name != '''cvit-mkb-clsr''' else None			,    )



	def        a  (  self     :       Any			,    _lowercase     :       List[str]			,    _lowercase     :       int						):
		if self.config_name == "cvit-mkb-clsr":
			return {"precision@10": precision_at_aa(_lowercase			,    _lowercase						)}
		elif self.config_name in ["wiki-ner"]:
			return acc_and_fa(_lowercase			,    _lowercase						)
		elif self.config_name in [
		    "wnli",
		    "copa",
		    "sna",
		    "csqa",
		    "wstp",
		    "inltkh",
		    "bbca",
		    "iitp-mr",
		    "iitp-pr",
		    "actsa-sc",
		    "md",
		]:
			return {"accuracy": simple_accuracy(_lowercase			,    _lowercase						)}
		else:
			raise KeyError(
			    '''You should supply a configuration name selected in '''
			    '''["wnli", "copa", "sna", "csqa", "wstp", "inltkh", "bbca", '''
			    '''"cvit-mkb-clsr", "iitp-mr", "iitp-pr", "actsa-sc", "md", '''
			    '''"wiki-ner"]'''						)







"""simple docstring"""





import unittest
from pathlib import Path
from tempfile import TemporaryDirectory

from transformers import AutoConfig, TFGPTaLMHeadModel, is_keras_nlp_available, is_tf_available
from transformers.models.gpta.tokenization_gpta import GPTaTokenizer
from transformers.testing_utils import require_keras_nlp, require_tf, slow


if is_tf_available():
  import tensorflow as tf

if is_keras_nlp_available():
  from transformers.models.gpta import TFGPTaTokenizer


__A											=						["""gpt2"""]
__A											=						"""gpt2"""

if is_tf_available():






  class    _lowerCAmelCase  (	tf.Module			):
         """simple docstring"""




         def __init__(       self       ,					__UpperCAmelCase ):



           '''simple docstring'''

           super().__init__()
           lowerCAmelCase__     :List[str]											=				tokenizer
           lowerCAmelCase__     :Optional[int]											=				AutoConfig.from_pretrained(__UpperCAmelCase )
           lowerCAmelCase__     :Optional[Any]											=				TFGPTaLMHeadModel.from_config(__UpperCAmelCase )






         @tf.function(input_signature=(tf.TensorSpec((None,)       ,					tf.string       ,					name='text' ),) )
         def 	snake_case		(       self       ,					__UpperCAmelCase ):



           '''simple docstring'''

           lowerCAmelCase__     :str											=				self.tokenizer(__UpperCAmelCase )
           lowerCAmelCase__     :int											=				tokenized['input_ids'].to_tensor()

           lowerCAmelCase__     :Optional[Any]											=				tf.cast(input_ids_dense > 0       ,					tf.intaa )
           # input_mask = tf.reshape(input_mask, [-1, MAX_SEQ_LEN])

           lowerCAmelCase__     :int											=				self.model(input_ids=__UpperCAmelCase       ,					attention_mask=__UpperCAmelCase )['logits']

           return outputs






@require_tf
@require_keras_nlp
class    _lowerCAmelCase  (	unittest.TestCase			):
       """simple docstring"""




       def 	snake_case		(       self ):



         '''simple docstring'''

         super().setUp()

         lowerCAmelCase__     :List[str]											=				[GPTaTokenizer.from_pretrained(__UpperCAmelCase ) for checkpoint in (TOKENIZER_CHECKPOINTS)]
         lowerCAmelCase__     :Union[str, Any]											=				[TFGPTaTokenizer.from_pretrained(__UpperCAmelCase ) for checkpoint in TOKENIZER_CHECKPOINTS]
         assert len(self.tokenizers ) == len(self.tf_tokenizers )

         lowerCAmelCase__     :int											=				[
             'This is a straightforward English test sentence.',
             'This one has some weird characters\rto\nsee\r\nif  those\u00E9break things.',
             'Now we\'re going to add some Chinese: 一 二 三 一二三',
             'And some much more rare Chinese: 齉 堃 齉堃',
             'Je vais aussi écrire en français pour tester les accents',
             'Classical Irish also has some unusual characters, so in they go: Gaelaċ, ꝼ',
         ]
         lowerCAmelCase__     :str											=				list(zip(self.test_sentences       ,					self.test_sentences[::-1] ) )






       def 	snake_case		(       self ):



         '''simple docstring'''

         for tokenizer, tf_tokenizer in zip(self.tokenizers       ,					self.tf_tokenizers ):
           for test_inputs in self.test_sentences:
             lowerCAmelCase__     :int											=				tokenizer([test_inputs]       ,					return_tensors='tf' )
             lowerCAmelCase__     :Optional[int]											=				tf_tokenizer([test_inputs] )

             for key in python_outputs.keys():
               # convert them to numpy to avoid messing with ragged tensors
               lowerCAmelCase__     :Optional[int]											=				python_outputs[key].numpy()
               lowerCAmelCase__     :List[str]											=				tf_outputs[key].numpy()

               self.assertTrue(tf.reduce_all(python_outputs_values.shape == tf_outputs_values.shape ) )
               self.assertTrue(tf.reduce_all(tf.cast(__UpperCAmelCase       ,					tf.intaa ) == tf_outputs_values ) )






       @slow
       def 	snake_case		(       self ):



         '''simple docstring'''

         for tf_tokenizer in self.tf_tokenizers:
           lowerCAmelCase__     :Optional[int]											=				tf.function(__UpperCAmelCase )
           for test_inputs in self.test_sentences:
             lowerCAmelCase__     :Any											=				tf.constant(__UpperCAmelCase )
             lowerCAmelCase__     :int											=				compiled_tokenizer(__UpperCAmelCase )
             lowerCAmelCase__     :Any											=				tf_tokenizer(__UpperCAmelCase )

             for key in eager_outputs.keys():
               self.assertTrue(tf.reduce_all(eager_outputs[key] == compiled_outputs[key] ) )






       @slow
       def 	snake_case		(       self ):



         '''simple docstring'''

         for tf_tokenizer in self.tf_tokenizers:
           lowerCAmelCase__     :List[str]											=				ModelToSave(tokenizer=__UpperCAmelCase )
           lowerCAmelCase__     :Union[str, Any]											=				tf.convert_to_tensor([self.test_sentences[0]] )
           lowerCAmelCase__     :List[str]											=				model.serving(__UpperCAmelCase )  # Build model with some sample inputs
           with TemporaryDirectory() as tempdir:
             lowerCAmelCase__     :Union[str, Any]											=				Path(__UpperCAmelCase ) / 'saved.model'
             tf.saved_model.save(__UpperCAmelCase       ,					__UpperCAmelCase       ,					signatures={'serving_default': model.serving} )
             lowerCAmelCase__     :str											=				tf.saved_model.load(__UpperCAmelCase )
           lowerCAmelCase__     :Dict											=				loaded_model.signatures['serving_default'](__UpperCAmelCase )['output_0']
           # We may see small differences because the loaded model is compiled, so we need an epsilon for the test
           self.assertTrue(tf.reduce_all(out == loaded_output ) )






       @slow
       def 	snake_case		(       self ):



         '''simple docstring'''

         for tf_tokenizer in self.tf_tokenizers:
           lowerCAmelCase__     :List[str]											=				tf.convert_to_tensor([self.test_sentences[0]] )
           lowerCAmelCase__     :List[str]											=				tf_tokenizer(__UpperCAmelCase )  # Build model with some sample inputs

           lowerCAmelCase__     :Union[str, Any]											=				tf_tokenizer.get_config()
           lowerCAmelCase__     :Tuple											=				TFGPTaTokenizer.from_config(__UpperCAmelCase )
           lowerCAmelCase__     :Tuple											=				model_from_config(__UpperCAmelCase )

           for key in from_config_output.keys():
             self.assertTrue(tf.reduce_all(from_config_output[key] == out[key] ) )






       @slow
       def 	snake_case		(       self ):



         '''simple docstring'''

         for tf_tokenizer in self.tf_tokenizers:
           # for the test to run
           lowerCAmelCase__     :int											=				1_2_3_1_2_3

           for max_length in [3, 5, 1_0_2_4]:
             lowerCAmelCase__     :Tuple											=				tf.convert_to_tensor([self.test_sentences[0]] )
             lowerCAmelCase__     :List[str]											=				tf_tokenizer(__UpperCAmelCase       ,					max_length=__UpperCAmelCase )

             lowerCAmelCase__     :Union[str, Any]											=				out['input_ids'].numpy().shape[1]

             assert out_length == max_length
