
import os
import tempfile
import unittest

from transformers import NezhaConfig, is_torch_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_torch, require_torch_gpu, slow, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
 import torch

 from transformers import (
     MODEL_FOR_PRETRAINING_MAPPING,
     NezhaForMaskedLM,
     NezhaForMultipleChoice,
     NezhaForNextSentencePrediction,
     NezhaForPreTraining,
     NezhaForQuestionAnswering,
     NezhaForSequenceClassification,
     NezhaForTokenClassification,
     NezhaModel,
 )
 from transformers.models.nezha.modeling_nezha import NEZHA_PRETRAINED_MODEL_ARCHIVE_LIST


class 			lowercase_		:


       def __init__(    self						,       __A						,       __A=13						,       __A=7						,       __A=True						,       __A=True						,       __A=True						,       __A=True						,       __A=99						,       __A=32						,       __A=5						,       __A=4						,       __A=37						,       __A="gelu"						,       __A=0.1						,       __A=0.1						,       __A=128						,       __A=32						,       __A=16						,       __A=2						,       __A=0.02						,       __A=3						,       __A=4						,       __A=None						,       )						->    List[str]:
              SCREAMING_SNAKE_CASE_							:				str		=parent
              SCREAMING_SNAKE_CASE_							:				List[Any]		=batch_size
              SCREAMING_SNAKE_CASE_							:				int		=seq_length
              SCREAMING_SNAKE_CASE_							:				List[Any]		=is_training
              SCREAMING_SNAKE_CASE_							:				Any		=use_input_mask
              SCREAMING_SNAKE_CASE_							:				str		=use_token_type_ids
              SCREAMING_SNAKE_CASE_							:				int		=use_labels
              SCREAMING_SNAKE_CASE_							:				List[str]		=vocab_size
              SCREAMING_SNAKE_CASE_							:				Tuple		=hidden_size
              SCREAMING_SNAKE_CASE_							:				Optional[int]		=num_hidden_layers
              SCREAMING_SNAKE_CASE_							:				Optional[Any]		=num_attention_heads
              SCREAMING_SNAKE_CASE_							:				int		=intermediate_size
              SCREAMING_SNAKE_CASE_							:				List[str]		=hidden_act
              SCREAMING_SNAKE_CASE_							:				List[Any]		=hidden_dropout_prob
              SCREAMING_SNAKE_CASE_							:				Any		=attention_probs_dropout_prob
              SCREAMING_SNAKE_CASE_							:				Any		=max_position_embeddings
              SCREAMING_SNAKE_CASE_							:				Dict		=type_vocab_size
              SCREAMING_SNAKE_CASE_							:				List[str]		=type_sequence_label_size
              SCREAMING_SNAKE_CASE_							:				Optional[int]		=initializer_range
              SCREAMING_SNAKE_CASE_							:				Tuple		=num_labels
              SCREAMING_SNAKE_CASE_							:				Optional[int]		=num_choices
              SCREAMING_SNAKE_CASE_							:				Optional[int]		=scope


       def 	_snake_case				(    self				)						->    List[str]:
              SCREAMING_SNAKE_CASE_							:				Tuple		=ids_tensor([self.batch_size, self.seq_length]						,       self.vocab_size				)

              SCREAMING_SNAKE_CASE_							:				Optional[Any]		=None
              if self.use_input_mask:
                     SCREAMING_SNAKE_CASE_							:				List[Any]		=random_attention_mask([self.batch_size, self.seq_length]				)

              SCREAMING_SNAKE_CASE_							:				Optional[int]		=None
              if self.use_token_type_ids:
                     SCREAMING_SNAKE_CASE_							:				Optional[Any]		=ids_tensor([self.batch_size, self.seq_length]						,       self.type_vocab_size				)

              SCREAMING_SNAKE_CASE_							:				Tuple		=None
              SCREAMING_SNAKE_CASE_							:				List[str]		=None
              SCREAMING_SNAKE_CASE_							:				int		=None
              if self.use_labels:
                     SCREAMING_SNAKE_CASE_							:				Optional[Any]		=ids_tensor([self.batch_size]						,       self.type_sequence_label_size				)
                     SCREAMING_SNAKE_CASE_							:				Optional[int]		=ids_tensor([self.batch_size, self.seq_length]						,       self.num_labels				)
                     SCREAMING_SNAKE_CASE_							:				List[Any]		=ids_tensor([self.batch_size]						,       self.num_choices				)

              SCREAMING_SNAKE_CASE_							:				str		=self.get_config()

              return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels


       def 	_snake_case				(    self				)						->    List[Any]:
              return NezhaConfig(
                  vocab_size=self.vocab_size						,       hidden_size=self.hidden_size						,       num_hidden_layers=self.num_hidden_layers						,       num_attention_heads=self.num_attention_heads						,       intermediate_size=self.intermediate_size						,       hidden_act=self.hidden_act						,       hidden_dropout_prob=self.hidden_dropout_prob						,       attention_probs_dropout_prob=self.attention_probs_dropout_prob						,       max_position_embeddings=self.max_position_embeddings						,       type_vocab_size=self.type_vocab_size						,       is_decoder=__A						,       initializer_range=self.initializer_range						,       )


       def 	_snake_case				(    self				)						->    Dict:
              (
                  (
                  SCREAMING_SNAKE_CASE_
              )	,     (
                  SCREAMING_SNAKE_CASE_
              )	,     (
                  SCREAMING_SNAKE_CASE_
              )	,     (
                  SCREAMING_SNAKE_CASE_
              )	,     (
                  SCREAMING_SNAKE_CASE_
              )	,     (
                  SCREAMING_SNAKE_CASE_
              )	,     (
                  SCREAMING_SNAKE_CASE_
              )	,     
              )							:				Any		=self.prepare_config_and_inputs()

              SCREAMING_SNAKE_CASE_							:				List[Any]		=True
              SCREAMING_SNAKE_CASE_							:				Optional[Any]		=floats_tensor([self.batch_size, self.seq_length, self.hidden_size]				)
              SCREAMING_SNAKE_CASE_							:				Optional[Any]		=ids_tensor([self.batch_size, self.seq_length]						,       vocab_size=2				)

              return (
                  config,
                  input_ids,
                  token_type_ids,
                  input_mask,
                  sequence_labels,
                  token_labels,
                  choice_labels,
                  encoder_hidden_states,
                  encoder_attention_mask,
              )


       def 	_snake_case				(    self						,       __A						,       __A						,       __A						,       __A						,       __A						,       __A						,       __A				)						->    Optional[int]:
              SCREAMING_SNAKE_CASE_							:				Optional[Any]		=NezhaModel(config=__A				)
              model.to(__A				)
              model.eval()
              SCREAMING_SNAKE_CASE_							:				Optional[Any]		=model(__A						,       attention_mask=__A						,       token_type_ids=__A				)
              SCREAMING_SNAKE_CASE_							:				Dict		=model(__A						,       token_type_ids=__A				)
              SCREAMING_SNAKE_CASE_							:				Optional[int]		=model(__A				)
              self.parent.assertEqual(result.last_hidden_state.shape						,       (self.batch_size, self.seq_length, self.hidden_size)				)
              self.parent.assertEqual(result.pooler_output.shape						,       (self.batch_size, self.hidden_size)				)


       def 	_snake_case				(    self						,       __A						,       __A						,       __A						,       __A						,       __A						,       __A						,       __A						,       __A						,       __A						,       )						->    Optional[Any]:
              SCREAMING_SNAKE_CASE_							:				List[str]		=True
              SCREAMING_SNAKE_CASE_							:				int		=NezhaModel(__A				)
              model.to(__A				)
              model.eval()
              SCREAMING_SNAKE_CASE_							:				Dict		=model(
                  __A						,       attention_mask=__A						,       token_type_ids=__A						,       encoder_hidden_states=__A						,       encoder_attention_mask=__A						,       )
              SCREAMING_SNAKE_CASE_							:				int		=model(
                  __A						,       attention_mask=__A						,       token_type_ids=__A						,       encoder_hidden_states=__A						,       )
              SCREAMING_SNAKE_CASE_							:				Tuple		=model(__A						,       attention_mask=__A						,       token_type_ids=__A				)
              self.parent.assertEqual(result.last_hidden_state.shape						,       (self.batch_size, self.seq_length, self.hidden_size)				)
              self.parent.assertEqual(result.pooler_output.shape						,       (self.batch_size, self.hidden_size)				)


       def 	_snake_case				(    self						,       __A						,       __A						,       __A						,       __A						,       __A						,       __A						,       __A				)						->    Dict:
              SCREAMING_SNAKE_CASE_							:				List[str]		=NezhaForMaskedLM(config=__A				)
              model.to(__A				)
              model.eval()
              SCREAMING_SNAKE_CASE_							:				str		=model(__A						,       attention_mask=__A						,       token_type_ids=__A						,       labels=__A				)
              self.parent.assertEqual(result.logits.shape						,       (self.batch_size, self.seq_length, self.vocab_size)				)


       def 	_snake_case				(    self						,       __A						,       __A						,       __A						,       __A						,       __A						,       __A						,       __A				)						->    Tuple:
              SCREAMING_SNAKE_CASE_							:				Union[str, Any]		=NezhaForNextSentencePrediction(config=__A				)
              model.to(__A				)
              model.eval()
              SCREAMING_SNAKE_CASE_							:				List[str]		=model(
                  __A						,       attention_mask=__A						,       token_type_ids=__A						,       labels=__A						,       )
              self.parent.assertEqual(result.logits.shape						,       (self.batch_size, 2)				)


       def 	_snake_case				(    self						,       __A						,       __A						,       __A						,       __A						,       __A						,       __A						,       __A				)						->    Union[str, Any]:
              SCREAMING_SNAKE_CASE_							:				Tuple		=NezhaForPreTraining(config=__A				)
              model.to(__A				)
              model.eval()
              SCREAMING_SNAKE_CASE_							:				Dict		=model(
                  __A						,       attention_mask=__A						,       token_type_ids=__A						,       labels=__A						,       next_sentence_label=__A						,       )
              self.parent.assertEqual(result.prediction_logits.shape						,       (self.batch_size, self.seq_length, self.vocab_size)				)
              self.parent.assertEqual(result.seq_relationship_logits.shape						,       (self.batch_size, 2)				)


       def 	_snake_case				(    self						,       __A						,       __A						,       __A						,       __A						,       __A						,       __A						,       __A				)						->    Union[str, Any]:
              SCREAMING_SNAKE_CASE_							:				Optional[Any]		=NezhaForQuestionAnswering(config=__A				)
              model.to(__A				)
              model.eval()
              SCREAMING_SNAKE_CASE_							:				Dict		=model(
                  __A						,       attention_mask=__A						,       token_type_ids=__A						,       start_positions=__A						,       end_positions=__A						,       )
              self.parent.assertEqual(result.start_logits.shape						,       (self.batch_size, self.seq_length)				)
              self.parent.assertEqual(result.end_logits.shape						,       (self.batch_size, self.seq_length)				)


       def 	_snake_case				(    self						,       __A						,       __A						,       __A						,       __A						,       __A						,       __A						,       __A				)						->    Tuple:
              SCREAMING_SNAKE_CASE_							:				Optional[int]		=self.num_labels
              SCREAMING_SNAKE_CASE_							:				Tuple		=NezhaForSequenceClassification(__A				)
              model.to(__A				)
              model.eval()
              SCREAMING_SNAKE_CASE_							:				Union[str, Any]		=model(__A						,       attention_mask=__A						,       token_type_ids=__A						,       labels=__A				)
              self.parent.assertEqual(result.logits.shape						,       (self.batch_size, self.num_labels)				)


       def 	_snake_case				(    self						,       __A						,       __A						,       __A						,       __A						,       __A						,       __A						,       __A				)						->    int:
              SCREAMING_SNAKE_CASE_							:				Tuple		=self.num_labels
              SCREAMING_SNAKE_CASE_							:				Optional[int]		=NezhaForTokenClassification(config=__A				)
              model.to(__A				)
              model.eval()
              SCREAMING_SNAKE_CASE_							:				int		=model(__A						,       attention_mask=__A						,       token_type_ids=__A						,       labels=__A				)
              self.parent.assertEqual(result.logits.shape						,       (self.batch_size, self.seq_length, self.num_labels)				)


       def 	_snake_case				(    self						,       __A						,       __A						,       __A						,       __A						,       __A						,       __A						,       __A				)						->    List[Any]:
              SCREAMING_SNAKE_CASE_							:				List[Any]		=self.num_choices
              SCREAMING_SNAKE_CASE_							:				int		=NezhaForMultipleChoice(config=__A				)
              model.to(__A				)
              model.eval()
              SCREAMING_SNAKE_CASE_							:				Dict		=input_ids.unsqueeze(1				).expand(-1						,       self.num_choices						,       -1				).contiguous()
              SCREAMING_SNAKE_CASE_							:				int		=token_type_ids.unsqueeze(1				).expand(-1						,       self.num_choices						,       -1				).contiguous()
              SCREAMING_SNAKE_CASE_							:				Optional[Any]		=input_mask.unsqueeze(1				).expand(-1						,       self.num_choices						,       -1				).contiguous()
              SCREAMING_SNAKE_CASE_							:				Union[str, Any]		=model(
                  __A						,       attention_mask=__A						,       token_type_ids=__A						,       labels=__A						,       )
              self.parent.assertEqual(result.logits.shape						,       (self.batch_size, self.num_choices)				)





       def 	_snake_case				(    self				)						->    Optional[Any]:
              SCREAMING_SNAKE_CASE_							:				List[str]		=self.prepare_config_and_inputs()
              (
                  (
                  SCREAMING_SNAKE_CASE_
              )	,     (
                  SCREAMING_SNAKE_CASE_
              )	,     (
                  SCREAMING_SNAKE_CASE_
              )	,     (
                  SCREAMING_SNAKE_CASE_
              )	,     (
                  SCREAMING_SNAKE_CASE_
              )	,     (
                  SCREAMING_SNAKE_CASE_
              )	,     (
                  SCREAMING_SNAKE_CASE_
              )	,     
              )							:				Any		=config_and_inputs
              SCREAMING_SNAKE_CASE_							:				Optional[int]		={'''input_ids''': input_ids, '''token_type_ids''': token_type_ids, '''attention_mask''': input_mask}
              return config, inputs_dict






@require_torch
class 			lowercase_		(      A  ,      A  ,      A  ,      unittest.TestCase       ):
       __lowerCamelCase				       =						(
           (
               NezhaModel,
               NezhaForMaskedLM,
               NezhaForMultipleChoice,
               NezhaForNextSentencePrediction,
               NezhaForPreTraining,
               NezhaForQuestionAnswering,
               NezhaForSequenceClassification,
               NezhaForTokenClassification,
           )
           if is_torch_available()
           else ()
       )
       __lowerCamelCase				       =						(
           {
               "feature-extraction": NezhaModel,
               "fill-mask": NezhaForMaskedLM,
               "question-answering": NezhaForQuestionAnswering,
               "text-classification": NezhaForSequenceClassification,
               "token-classification": NezhaForTokenClassification,
               "zero-shot": NezhaForSequenceClassification,
           }
           if is_torch_available()
           else {}
       )
       __lowerCamelCase				       =						True


       def 	_snake_case				(    self						,       __A						,       __A						,       __A=False				)						->    str:
              SCREAMING_SNAKE_CASE_							:				Dict		=super()._prepare_for_class(__A						,       __A						,       return_labels=__A				)

              if return_labels:
                     if model_class in get_values(__A				):
                            SCREAMING_SNAKE_CASE_							:				Optional[int]		=torch.zeros(
                                (self.model_tester.batch_size, self.model_tester.seq_length)						,       dtype=torch.long						,       device=__A				)
                            SCREAMING_SNAKE_CASE_							:				Optional[Any]		=torch.zeros(
                                self.model_tester.batch_size						,       dtype=torch.long						,       device=__A				)
              return inputs_dict


       def 	_snake_case				(    self				)						->    int:
              SCREAMING_SNAKE_CASE_							:				Union[str, Any]		=NezhaModelTester(self				)
              SCREAMING_SNAKE_CASE_							:				Optional[Any]		=ConfigTester(self						,       config_class=__A						,       hidden_size=37				)


       def 	_snake_case				(    self				)						->    Optional[Any]:
              self.config_tester.run_common_tests()


       def 	_snake_case				(    self				)						->    Dict:
              SCREAMING_SNAKE_CASE_							:				Tuple		=self.model_tester.prepare_config_and_inputs()
              self.model_tester.create_and_check_model(*__A				)


       def 	_snake_case				(    self				)						->    int:
              SCREAMING_SNAKE_CASE_							:				Optional[int]		=self.model_tester.prepare_config_and_inputs_for_decoder()
              self.model_tester.create_and_check_model_as_decoder(*__A				)


       def 	_snake_case				(    self				)						->    Tuple:
              # This regression test was failing with PyTorch < 1.3
              (
                  (
                  SCREAMING_SNAKE_CASE_
              )	,     (
                  SCREAMING_SNAKE_CASE_
              )	,     (
                  SCREAMING_SNAKE_CASE_
              )	,     (
                  SCREAMING_SNAKE_CASE_
              )	,     (
                  SCREAMING_SNAKE_CASE_
              )	,     (
                  SCREAMING_SNAKE_CASE_
              )	,     (
                  SCREAMING_SNAKE_CASE_
              )	,     (
                  SCREAMING_SNAKE_CASE_
              )	,     (
                  SCREAMING_SNAKE_CASE_
              )	,     
              )							:				int		=self.model_tester.prepare_config_and_inputs_for_decoder()

              SCREAMING_SNAKE_CASE_							:				Union[str, Any]		=None

              self.model_tester.create_and_check_model_as_decoder(
                  __A						,       __A						,       __A						,       __A						,       __A						,       __A						,       __A						,       __A						,       __A						,       )


       def 	_snake_case				(    self				)						->    List[str]:
              SCREAMING_SNAKE_CASE_							:				Any		=self.model_tester.prepare_config_and_inputs()
              self.model_tester.create_and_check_for_masked_lm(*__A				)


       def 	_snake_case				(    self				)						->    Optional[Any]:
              SCREAMING_SNAKE_CASE_							:				List[Any]		=self.model_tester.prepare_config_and_inputs()
              self.model_tester.create_and_check_for_multiple_choice(*__A				)


       def 	_snake_case				(    self				)						->    str:
              SCREAMING_SNAKE_CASE_							:				Any		=self.model_tester.prepare_config_and_inputs()
              self.model_tester.create_and_check_for_next_sequence_prediction(*__A				)


       def 	_snake_case				(    self				)						->    List[str]:
              SCREAMING_SNAKE_CASE_							:				Optional[Any]		=self.model_tester.prepare_config_and_inputs()
              self.model_tester.create_and_check_for_pretraining(*__A				)


       def 	_snake_case				(    self				)						->    Optional[int]:
              SCREAMING_SNAKE_CASE_							:				Any		=self.model_tester.prepare_config_and_inputs()
              self.model_tester.create_and_check_for_question_answering(*__A				)


       def 	_snake_case				(    self				)						->    str:
              SCREAMING_SNAKE_CASE_							:				Optional[int]		=self.model_tester.prepare_config_and_inputs()
              self.model_tester.create_and_check_for_sequence_classification(*__A				)


       def 	_snake_case				(    self				)						->    Tuple:
              SCREAMING_SNAKE_CASE_							:				Optional[int]		=self.model_tester.prepare_config_and_inputs()
              self.model_tester.create_and_check_for_token_classification(*__A				)


       @slow
       def 	_snake_case				(    self				)						->    List[str]:
              for model_name in NEZHA_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
                     SCREAMING_SNAKE_CASE_							:				Dict		=NezhaModel.from_pretrained(__A				)
                     self.assertIsNotNone(__A				)





       @slow
       @require_torch_gpu
       def 	_snake_case				(    self				)						->    List[Any]:
              SCREAMING_SNAKE_CASE_	,     SCREAMING_SNAKE_CASE_							:				List[str]		=self.model_tester.prepare_config_and_inputs_for_common()
              for model_class in self.all_model_classes:
                     # NezhaForMultipleChoice behaves incorrectly in JIT environments.
                     if model_class == NezhaForMultipleChoice:
                            return

                     SCREAMING_SNAKE_CASE_							:				Tuple		=True
                     SCREAMING_SNAKE_CASE_							:				Union[str, Any]		=model_class(config=__A				)

                     SCREAMING_SNAKE_CASE_							:				str		=self._prepare_for_class(__A						,       __A				)
                     SCREAMING_SNAKE_CASE_							:				Optional[int]		=torch.jit.trace(
                         __A						,       (inputs_dict['''input_ids'''].to('''cpu'''				), inputs_dict['''attention_mask'''].to('''cpu'''				))				)

                     with tempfile.TemporaryDirectory() as tmp:
                            torch.jit.save(__A						,       os.path.join(__A						,       '''bert.pt'''				)				)
                            SCREAMING_SNAKE_CASE_							:				str		=torch.jit.load(os.path.join(__A						,       '''bert.pt'''				)						,       map_location=__A				)
                            loaded(inputs_dict['''input_ids'''].to(__A				)						,       inputs_dict['''attention_mask'''].to(__A				)				)






@require_torch
class 			lowercase_		(      unittest.TestCase       ):


       @slow
       def 	_snake_case				(    self				)						->    List[str]:
              SCREAMING_SNAKE_CASE_							:				Dict		=NezhaModel.from_pretrained('''sijunhe/nezha-cn-base'''				)
              SCREAMING_SNAKE_CASE_							:				Optional[Any]		=torch.tensor([[0, 1, 2, 3, 4, 5]]				)
              SCREAMING_SNAKE_CASE_							:				Union[str, Any]		=torch.tensor([[0, 1, 1, 1, 1, 1]]				)
              with torch.no_grad():
                     SCREAMING_SNAKE_CASE_							:				Dict		=model(__A						,       attention_mask=__A				)[0]
              SCREAMING_SNAKE_CASE_							:				Dict		=torch.Size((1, 6, 768)				)
              self.assertEqual(output.shape						,       __A				)
              SCREAMING_SNAKE_CASE_							:				List[Any]		=torch.tensor([[[0.0_685, 0.2_441, 0.1_102], [0.0_600, 0.1_906, 0.1_349], [0.0_221, 0.0_819, 0.0_586]]]				)

              self.assertTrue(torch.allclose(output[:, 1:4, 1:4]						,       __A						,       atol=1e-4				)				)





       @slow
       def 	_snake_case				(    self				)						->    Any:
              SCREAMING_SNAKE_CASE_							:				Union[str, Any]		=NezhaForMaskedLM.from_pretrained('''sijunhe/nezha-cn-base'''				)
              SCREAMING_SNAKE_CASE_							:				Dict		=torch.tensor([[0, 1, 2, 3, 4, 5]]				)
              SCREAMING_SNAKE_CASE_							:				Tuple		=torch.tensor([[1, 1, 1, 1, 1, 1]]				)
              with torch.no_grad():
                     SCREAMING_SNAKE_CASE_							:				Any		=model(__A						,       attention_mask=__A				)[0]
              SCREAMING_SNAKE_CASE_							:				List[Any]		=torch.Size((1, 6, 21_128)				)
              self.assertEqual(output.shape						,       __A				)
              SCREAMING_SNAKE_CASE_							:				int		=torch.tensor(
                  [[-2.7_939, -1.7_902, -2.2_189], [-2.8_585, -1.8_908, -2.3_723], [-2.6_499, -1.7_750, -2.2_558]]				)

              self.assertTrue(torch.allclose(output[:, 1:4, 1:4]						,       __A						,       atol=1e-4				)				)


from scipy.stats import pearsonr, spearmanr
from sklearn.metrics import fa_score, matthews_corrcoef

import datasets


_lowercase											=    """\
@inproceedings{wang2019glue,
  title={{GLUE}: A Multi-Task Benchmark and Analysis Platform for Natural Language Understanding},
  author={Wang, Alex and Singh, Amanpreet and Michael, Julian and Hill, Felix and Levy, Omer and Bowman, Samuel R.},
  note={In the Proceedings of ICLR.},
  year={2019}
}
"""

_lowercase											=    """\
GLUE, the General Language Understanding Evaluation benchmark
(https://gluebenchmark.com/) is a collection of resources for training,
evaluating, and analyzing natural language understanding systems.
"""

_lowercase											=    """
Compute GLUE evaluation metric associated to each GLUE dataset.
Args:
    predictions: list of predictions to score.
        Each translation should be tokenized into a list of tokens.
    references: list of lists of references for each translation.
        Each reference should be tokenized into a list of tokens.
Returns: depending on the GLUE subset, one or several of:
    \"accuracy\": Accuracy
    \"f1\": F1 score
    \"pearson\": Pearson Correlation
    \"spearmanr\": Spearman Correlation
    \"matthews_correlation\": Matthew Correlation
Examples:

    >>> glue_metric = datasets.load_metric('glue', 'sst2')  # 'sst2' or any of [\"mnli\", \"mnli_mismatched\", \"mnli_matched\", \"qnli\", \"rte\", \"wnli\", \"hans\"]
    >>> references = [0, 1]
    >>> predictions = [0, 1]
    >>> results = glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {'accuracy': 1.0}

    >>> glue_metric = datasets.load_metric('glue', 'mrpc')  # 'mrpc' or 'qqp'
    >>> references = [0, 1]
    >>> predictions = [0, 1]
    >>> results = glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {'accuracy': 1.0, 'f1': 1.0}

    >>> glue_metric = datasets.load_metric('glue', 'stsb')
    >>> references = [0., 1., 2., 3., 4., 5.]
    >>> predictions = [0., 1., 2., 3., 4., 5.]
    >>> results = glue_metric.compute(predictions=predictions, references=references)
    >>> print({\"pearson\": round(results[\"pearson\"], 2), \"spearmanr\": round(results[\"spearmanr\"], 2)})
    {'pearson': 1.0, 'spearmanr': 1.0}

    >>> glue_metric = datasets.load_metric('glue', 'cola')
    >>> references = [0, 1]
    >>> predictions = [0, 1]
    >>> results = glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {'matthews_correlation': 1.0}
"""
def 							SCREAMING_SNAKE_CASE_	(					UpperCAmelCase_	:     List[Any]			,							UpperCAmelCase_	:     Union[str, Any]				)			->						int:
       return float((preds == labels).mean()				)
def 							SCREAMING_SNAKE_CASE_	(					UpperCAmelCase_	:     int			,							UpperCAmelCase_	:     str				)			->						Union[str, Any]:
       SCREAMING_SNAKE_CASE_							:				Tuple		=simple_accuracy(UpperCAmelCase_			,							UpperCAmelCase_				)
       SCREAMING_SNAKE_CASE_							:				Union[str, Any]		=float(fa_score(y_true=UpperCAmelCase_			,							y_pred=UpperCAmelCase_				)				)
       return {
           "accuracy": acc,
           "f1": fa,
       }
def 							SCREAMING_SNAKE_CASE_	(					UpperCAmelCase_	:     Optional[int]			,							UpperCAmelCase_	:     Dict				)			->						List[str]:
       SCREAMING_SNAKE_CASE_							:				str		=float(pearsonr(UpperCAmelCase_			,							UpperCAmelCase_				)[0]				)
       SCREAMING_SNAKE_CASE_							:				List[str]		=float(spearmanr(UpperCAmelCase_			,							UpperCAmelCase_				)[0]				)
       return {
           "pearson": pearson_corr,
           "spearmanr": spearman_corr,
       }


@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION  ,      _KWARGS_DESCRIPTION       )
class 			lowercase_		(      datasets.Metric       ):


       def 	_snake_case				(    self				)						->    str:
              if self.config_name not in [
                  "sst2",
                  "mnli",
                  "mnli_mismatched",
                  "mnli_matched",
                  "cola",
                  "stsb",
                  "mrpc",
                  "qqp",
                  "qnli",
                  "rte",
                  "wnli",
                  "hans",
              ]:
                     raise KeyError(
                         '''You should supply a configuration name selected in '''
                         '''["sst2", "mnli", "mnli_mismatched", "mnli_matched", '''
                         '''"cola", "stsb", "mrpc", "qqp", "qnli", "rte", "wnli", "hans"]'''				)
              return datasets.MetricInfo(
                  description=_DESCRIPTION						,       citation=_CITATION						,       inputs_description=_KWARGS_DESCRIPTION						,       features=datasets.Features(
                      {
                          '''predictions''': datasets.Value('''int64''' if self.config_name != '''stsb''' else '''float32'''				),
                          '''references''': datasets.Value('''int64''' if self.config_name != '''stsb''' else '''float32'''				),
                      }				)						,       codebase_urls=[]						,       reference_urls=[]						,       format='''numpy'''						,       )





       def 	_snake_case				(    self						,       __A						,       __A				)						->    List[Any]:
              if self.config_name == "cola":
                     return {"matthews_correlation": matthews_corrcoef(__A						,       __A				)}
              elif self.config_name == "stsb":
                     return pearson_and_spearman(__A						,       __A				)
              elif self.config_name in ["mrpc", "qqp"]:
                     return acc_and_fa(__A						,       __A				)
              elif self.config_name in ["sst2", "mnli", "mnli_mismatched", "mnli_matched", "qnli", "rte", "wnli", "hans"]:
                     return {"accuracy": simple_accuracy(__A						,       __A				)}
              else:
                     raise KeyError(
                         '''You should supply a configuration name selected in '''
                         '''["sst2", "mnli", "mnli_mismatched", "mnli_matched", '''
                         '''"cola", "stsb", "mrpc", "qqp", "qnli", "rte", "wnli", "hans"]'''				)
