

"""simple docstring"""




import os
import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from huggingface_hub.file_download import http_get
from requests.exceptions import HTTPError

from transformers import (
    AlbertTokenizer,
    AutoTokenizer,
    BertTokenizer,
    BertTokenizerFast,
    GPTaTokenizerFast,
    is_tokenizers_available,
)
from transformers.testing_utils import TOKEN, USER, is_staging_test, require_tokenizers
from transformers.tokenization_utils import Trie


sys.path.append(str(Path(__file__).parent.parent / 'utils'))

from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
      from test_module.custom_tokenization_fast import CustomTokenizerFast







class   __lowercase      (     unittest.TestCase       ):

  def  __lowercase							(	self		:      Any     ):






   '''simple docstring'''
   # A mock response for an HTTP head request to emulate server down
   UpperCAmelCase__       :	Optional[int]        =       mock.Mock()
   UpperCAmelCase__       :	List[Any]        =       500
   UpperCAmelCase__       :	int        =       {}
   UpperCAmelCase__       :	List[str]        =       HTTPError
   UpperCAmelCase__       :	str        =       {}

   # Download this model to make sure it's in the cache.
   UpperCAmelCase__       :	int        =       BertTokenizer.from_pretrained("""hf-internal-testing/tiny-random-bert"""     )

   # Under the mock environment we get a 500 error when trying to reach the tokenizer.
   with mock.patch("""requests.Session.request"""       ,return_value=A     ) as mock_head:
    UpperCAmelCase__       :	Any        =       BertTokenizer.from_pretrained("""hf-internal-testing/tiny-random-bert"""     )
    # This check we did call the fake head request
    mock_head.assert_called()

  @require_tokenizers
  def  __lowercase							(	self		:      int     ):






   '''simple docstring'''
   # A mock response for an HTTP head request to emulate server down
   UpperCAmelCase__       :	List[str]        =       mock.Mock()
   UpperCAmelCase__       :	List[str]        =       500
   UpperCAmelCase__       :	Optional[int]        =       {}
   UpperCAmelCase__       :	Any        =       HTTPError
   UpperCAmelCase__       :	Any        =       {}

   # Download this model to make sure it's in the cache.
   UpperCAmelCase__       :	List[Any]        =       GPTaTokenizerFast.from_pretrained("""gpt2"""     )

   # Under the mock environment we get a 500 error when trying to reach the tokenizer.
   with mock.patch("""requests.Session.request"""       ,return_value=A     ) as mock_head:
    UpperCAmelCase__       :	List[Any]        =       GPTaTokenizerFast.from_pretrained("""gpt2"""     )
    # This check we did call the fake head request
    mock_head.assert_called()

  def  __lowercase							(	self		:      List[str]     ):






   '''simple docstring'''
   # This test is for deprecated behavior and can be removed in v5
   try:
    UpperCAmelCase__       :	List[str]        =       tempfile.mktemp()
    with open(A       ,"""wb"""     ) as f:
     http_get("""https://huggingface.co/albert-base-v1/resolve/main/spiece.model"""       ,A     )

    UpperCAmelCase__       :	Any        =       AlbertTokenizer.from_pretrained(A     )
   finally:
    os.remove(A     )

   # Supporting this legacy load introduced a weird bug where the tokenizer would load local files if they are in
   # the current folder and have the right name.
   if os.path.isfile("""tokenizer.json"""     ):
    # We skip the test if the user has a `tokenizer.json` in this folder to avoid deleting it.
    return
   try:
    with open("""tokenizer.json"""       ,"""wb"""     ) as f:
     http_get("""https://huggingface.co/hf-internal-testing/tiny-random-bert/blob/main/tokenizer.json"""       ,A     )
    UpperCAmelCase__       :	List[str]        =       AutoTokenizer.from_pretrained("""hf-internal-testing/tiny-random-gpt2"""     )
    # The tiny random BERT has a vocab size of 1024, tiny gpt2 as a vocab size of 1000
    self.assertEqual(tokenizer.vocab_size       ,1_000     )
    # Tokenizer should depend on the remote checkpoint, not the local tokenizer.json file.

   finally:
    os.remove("""tokenizer.json"""     )




  def  __lowercase							(	self		:      List[str]     ):






   '''simple docstring'''
   # This test is for deprecated behavior and can be removed in v5
   UpperCAmelCase__       :	Tuple        =       AlbertTokenizer.from_pretrained("""https://huggingface.co/albert-base-v1/resolve/main/spiece.model"""     )







@is_staging_test
class   __lowercase      (     unittest.TestCase       ):
  snake_case_												=							["""[UNK]""", """[CLS]""", """[SEP]""", """[PAD]""", """[MASK]""", """bla""", """blou"""]

  @classmethod
  def  __lowercase							(	cls		:      Optional[Any]     ):






   '''simple docstring'''
   UpperCAmelCase__       :	int        =       TOKEN
   HfFolder.save_token(A     )

  @classmethod
  def  __lowercase							(	cls		:      List[Any]     ):






   '''simple docstring'''
   try:
    delete_repo(token=cls._token       ,repo_id="""test-tokenizer"""     )
   except HTTPError:
    pass

   try:
    delete_repo(token=cls._token       ,repo_id="""valid_org/test-tokenizer-org"""     )
   except HTTPError:
    pass

   try:
    delete_repo(token=cls._token       ,repo_id="""test-dynamic-tokenizer"""     )
   except HTTPError:
    pass

  def  __lowercase							(	self		:      int     ):






   '''simple docstring'''
   with tempfile.TemporaryDirectory() as tmp_dir:
    UpperCAmelCase__       :	List[Any]        =       os.path.join(A       ,"""vocab.txt"""     )
    with open(A       ,"""w"""       ,encoding="""utf-8"""     ) as vocab_writer:
     vocab_writer.write("""""".join([x + """\n""" for x in self.vocab_tokens]     )     )
    UpperCAmelCase__       :	List[Any]        =       BertTokenizer(A     )

   tokenizer.push_to_hub("""test-tokenizer"""       ,use_auth_token=self._token     )
   UpperCAmelCase__       :	List[Any]        =       BertTokenizer.from_pretrained(f"{USER}/test-tokenizer"     )
   self.assertDictEqual(new_tokenizer.vocab       ,tokenizer.vocab     )

   # Reset repo
   delete_repo(token=self._token       ,repo_id="""test-tokenizer"""     )

   # Push to hub via save_pretrained
   with tempfile.TemporaryDirectory() as tmp_dir:
    tokenizer.save_pretrained(A       ,repo_id="""test-tokenizer"""       ,push_to_hub=A       ,use_auth_token=self._token     )

   UpperCAmelCase__       :	str        =       BertTokenizer.from_pretrained(f"{USER}/test-tokenizer"     )
   self.assertDictEqual(new_tokenizer.vocab       ,tokenizer.vocab     )

  def  __lowercase							(	self		:      List[str]     ):






   '''simple docstring'''
   with tempfile.TemporaryDirectory() as tmp_dir:
    UpperCAmelCase__       :	Dict        =       os.path.join(A       ,"""vocab.txt"""     )
    with open(A       ,"""w"""       ,encoding="""utf-8"""     ) as vocab_writer:
     vocab_writer.write("""""".join([x + """\n""" for x in self.vocab_tokens]     )     )
    UpperCAmelCase__       :	Union[str, Any]        =       BertTokenizer(A     )

   tokenizer.push_to_hub("""valid_org/test-tokenizer-org"""       ,use_auth_token=self._token     )
   UpperCAmelCase__       :	Dict        =       BertTokenizer.from_pretrained("""valid_org/test-tokenizer-org"""     )
   self.assertDictEqual(new_tokenizer.vocab       ,tokenizer.vocab     )

   # Reset repo
   delete_repo(token=self._token       ,repo_id="""valid_org/test-tokenizer-org"""     )

   # Push to hub via save_pretrained
   with tempfile.TemporaryDirectory() as tmp_dir:
    tokenizer.save_pretrained(
        A       ,repo_id="""valid_org/test-tokenizer-org"""       ,push_to_hub=A       ,use_auth_token=self._token     )

   UpperCAmelCase__       :	List[str]        =       BertTokenizer.from_pretrained("""valid_org/test-tokenizer-org"""     )
   self.assertDictEqual(new_tokenizer.vocab       ,tokenizer.vocab     )




  @require_tokenizers
  def  __lowercase							(	self		:      str     ):






   '''simple docstring'''
   CustomTokenizer.register_for_auto_class()
   with tempfile.TemporaryDirectory() as tmp_dir:
    UpperCAmelCase__       :	Optional[Any]        =       os.path.join(A       ,"""vocab.txt"""     )
    with open(A       ,"""w"""       ,encoding="""utf-8"""     ) as vocab_writer:
     vocab_writer.write("""""".join([x + """\n""" for x in self.vocab_tokens]     )     )
    UpperCAmelCase__       :	Tuple        =       CustomTokenizer(A     )

   # No fast custom tokenizer
   tokenizer.push_to_hub("""test-dynamic-tokenizer"""       ,use_auth_token=self._token     )

   UpperCAmelCase__       :	Dict        =       AutoTokenizer.from_pretrained(f"{USER}/test-dynamic-tokenizer"       ,trust_remote_code=A     )
   # Can't make an isinstance check because the new_model.config is from the CustomTokenizer class of a dynamic module
   self.assertEqual(tokenizer.__class__.__name__       ,"""CustomTokenizer"""     )

   # Fast and slow custom tokenizer
   CustomTokenizerFast.register_for_auto_class()
   with tempfile.TemporaryDirectory() as tmp_dir:
    UpperCAmelCase__       :	List[str]        =       os.path.join(A       ,"""vocab.txt"""     )
    with open(A       ,"""w"""       ,encoding="""utf-8"""     ) as vocab_writer:
     vocab_writer.write("""""".join([x + """\n""" for x in self.vocab_tokens]     )     )

    UpperCAmelCase__       :	Dict        =       BertTokenizerFast.from_pretrained(A     )
    bert_tokenizer.save_pretrained(A     )
    UpperCAmelCase__       :	Dict        =       CustomTokenizerFast.from_pretrained(A     )

   tokenizer.push_to_hub("""test-dynamic-tokenizer"""       ,use_auth_token=self._token     )

   UpperCAmelCase__       :	Any        =       AutoTokenizer.from_pretrained(f"{USER}/test-dynamic-tokenizer"       ,trust_remote_code=A     )
   # Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
   self.assertEqual(tokenizer.__class__.__name__       ,"""CustomTokenizerFast"""     )
   UpperCAmelCase__       :	str        =       AutoTokenizer.from_pretrained(
       f"{USER}/test-dynamic-tokenizer"       ,use_fast=A       ,trust_remote_code=A     )
   # Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
   self.assertEqual(tokenizer.__class__.__name__       ,"""CustomTokenizer"""     )







class   __lowercase      (     unittest.TestCase       ):

  def  __lowercase							(	self		:      int     ):






   '''simple docstring'''
   UpperCAmelCase__       :	Tuple        =       Trie()
   trie.add("""Hello 友達"""     )
   self.assertEqual(trie.data       ,{"""H""": {"""e""": {"""l""": {"""l""": {"""o""": {""" """: {"""友""": {"""達""": {"""""": 1}}}}}}}}}     )
   trie.add("""Hello"""     )
   trie.data
   self.assertEqual(trie.data       ,{"""H""": {"""e""": {"""l""": {"""l""": {"""o""": {"""""": 1, """ """: {"""友""": {"""達""": {"""""": 1}}}}}}}}}     )

  def  __lowercase							(	self		:      Any     ):






   '''simple docstring'''
   UpperCAmelCase__       :	Optional[int]        =       Trie()
   self.assertEqual(trie.split("""[CLS] This is a extra_id_100"""     )       ,["""[CLS] This is a extra_id_100"""]     )
   trie.add("""[CLS]"""     )
   trie.add("""extra_id_1"""     )
   trie.add("""extra_id_100"""     )
   self.assertEqual(trie.split("""[CLS] This is a extra_id_100"""     )       ,["""[CLS]""", """ This is a """, """extra_id_100"""]     )

  def  __lowercase							(	self		:      List[Any]     ):






   '''simple docstring'''
   UpperCAmelCase__       :	int        =       Trie()
   trie.add("""A"""     )
   self.assertEqual(trie.split("""ABC"""     )       ,["""A""", """BC"""]     )
   self.assertEqual(trie.split("""BCA"""     )       ,["""BC""", """A"""]     )

  def  __lowercase							(	self		:      Optional[Any]     ):






   '''simple docstring'''
   UpperCAmelCase__       :	Tuple        =       Trie()
   trie.add("""TOKEN]"""     )
   trie.add("""[SPECIAL_TOKEN]"""     )
   self.assertEqual(trie.split("""This is something [SPECIAL_TOKEN]"""     )       ,["""This is something """, """[SPECIAL_TOKEN]"""]     )

  def  __lowercase							(	self		:      Dict     ):






   '''simple docstring'''
   UpperCAmelCase__       :	Dict        =       Trie()
   trie.add("""A"""     )
   trie.add("""P"""     )
   trie.add("""[SPECIAL_TOKEN]"""     )
   self.assertEqual(trie.split("""This is something [SPECIAL_TOKEN]"""     )       ,["""This is something """, """[SPECIAL_TOKEN]"""]     )

  def  __lowercase							(	self		:      Dict     ):






   '''simple docstring'''
   UpperCAmelCase__       :	Optional[int]        =       Trie()
   trie.add("""AB"""     )
   trie.add("""B"""     )
   trie.add("""C"""     )
   self.assertEqual(trie.split("""ABC"""     )       ,["""AB""", """C"""]     )

  def  __lowercase							(	self		:      str     ):






   '''simple docstring'''
   UpperCAmelCase__       :	int        =       Trie()
   trie.add("""ABC"""     )
   trie.add("""B"""     )
   trie.add("""CD"""     )
   self.assertEqual(trie.split("""ABCD"""     )       ,["""ABC""", """D"""]     )




  def  __lowercase							(	self		:      Union[str, Any]     ):






   '''simple docstring'''
   # Even if the offsets are wrong, we necessarily output correct string
   # parts.
   UpperCAmelCase__       :	Dict        =       Trie()
   UpperCAmelCase__       :	Optional[int]        =       trie.cut_text("""ABC"""       ,[0, 0, 2, 1, 2, 3]     )
   self.assertEqual(A       ,["""AB""", """C"""]     )







from scipy.stats import pearsonr, spearmanr
from sklearn.metrics import fa_score, matthews_corrcoef

import datasets


_lowercase											=    """\
@inproceedings{wang2019glue,
  title={{GLUE}: A Multi-Task Benchmark and Analysis Platform for Natural Language Understanding},
  author={Wang, Alex and Singh, Amanpreet and Michael, Julian and Hill, Felix and Levy, Omer and Bowman, Samuel R.},
  note={In the Proceedings of ICLR.},
  year={2019}
}
"""

_lowercase											=    """\
GLUE, the General Language Understanding Evaluation benchmark
(https://gluebenchmark.com/) is a collection of resources for training,
evaluating, and analyzing natural language understanding systems.
"""

_lowercase											=    """
Compute GLUE evaluation metric associated to each GLUE dataset.
Args:
    predictions: list of predictions to score.
        Each translation should be tokenized into a list of tokens.
    references: list of lists of references for each translation.
        Each reference should be tokenized into a list of tokens.
Returns: depending on the GLUE subset, one or several of:
    \"accuracy\": Accuracy
    \"f1\": F1 score
    \"pearson\": Pearson Correlation
    \"spearmanr\": Spearman Correlation
    \"matthews_correlation\": Matthew Correlation
Examples:

    >>> glue_metric = datasets.load_metric('glue', 'sst2')  # 'sst2' or any of [\"mnli\", \"mnli_mismatched\", \"mnli_matched\", \"qnli\", \"rte\", \"wnli\", \"hans\"]
    >>> references = [0, 1]
    >>> predictions = [0, 1]
    >>> results = glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {'accuracy': 1.0}

    >>> glue_metric = datasets.load_metric('glue', 'mrpc')  # 'mrpc' or 'qqp'
    >>> references = [0, 1]
    >>> predictions = [0, 1]
    >>> results = glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {'accuracy': 1.0, 'f1': 1.0}

    >>> glue_metric = datasets.load_metric('glue', 'stsb')
    >>> references = [0., 1., 2., 3., 4., 5.]
    >>> predictions = [0., 1., 2., 3., 4., 5.]
    >>> results = glue_metric.compute(predictions=predictions, references=references)
    >>> print({\"pearson\": round(results[\"pearson\"], 2), \"spearmanr\": round(results[\"spearmanr\"], 2)})
    {'pearson': 1.0, 'spearmanr': 1.0}

    >>> glue_metric = datasets.load_metric('glue', 'cola')
    >>> references = [0, 1]
    >>> predictions = [0, 1]
    >>> results = glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {'matthews_correlation': 1.0}
"""
def 							SCREAMING_SNAKE_CASE_	(					UpperCAmelCase_	:     List[Any]			,							UpperCAmelCase_	:     Union[str, Any]				)			->						int:
       return float((preds == labels).mean()				)
def 							SCREAMING_SNAKE_CASE_	(					UpperCAmelCase_	:     int			,							UpperCAmelCase_	:     str				)			->						Union[str, Any]:
       SCREAMING_SNAKE_CASE_							:				Tuple		=simple_accuracy(UpperCAmelCase_			,							UpperCAmelCase_				)
       SCREAMING_SNAKE_CASE_							:				Union[str, Any]		=float(fa_score(y_true=UpperCAmelCase_			,							y_pred=UpperCAmelCase_				)				)
       return {
           "accuracy": acc,
           "f1": fa,
       }
def 							SCREAMING_SNAKE_CASE_	(					UpperCAmelCase_	:     Optional[int]			,							UpperCAmelCase_	:     Dict				)			->						List[str]:
       SCREAMING_SNAKE_CASE_							:				str		=float(pearsonr(UpperCAmelCase_			,							UpperCAmelCase_				)[0]				)
       SCREAMING_SNAKE_CASE_							:				List[str]		=float(spearmanr(UpperCAmelCase_			,							UpperCAmelCase_				)[0]				)
       return {
           "pearson": pearson_corr,
           "spearmanr": spearman_corr,
       }


@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION  ,      _KWARGS_DESCRIPTION       )
class 			lowercase_		(      datasets.Metric       ):


       def 	_snake_case				(    self				)						->    str:
              if self.config_name not in [
                  "sst2",
                  "mnli",
                  "mnli_mismatched",
                  "mnli_matched",
                  "cola",
                  "stsb",
                  "mrpc",
                  "qqp",
                  "qnli",
                  "rte",
                  "wnli",
                  "hans",
              ]:
                     raise KeyError(
                         '''You should supply a configuration name selected in '''
                         '''["sst2", "mnli", "mnli_mismatched", "mnli_matched", '''
                         '''"cola", "stsb", "mrpc", "qqp", "qnli", "rte", "wnli", "hans"]'''				)
              return datasets.MetricInfo(
                  description=_DESCRIPTION						,       citation=_CITATION						,       inputs_description=_KWARGS_DESCRIPTION						,       features=datasets.Features(
                      {
                          '''predictions''': datasets.Value('''int64''' if self.config_name != '''stsb''' else '''float32'''				),
                          '''references''': datasets.Value('''int64''' if self.config_name != '''stsb''' else '''float32'''				),
                      }				)						,       codebase_urls=[]						,       reference_urls=[]						,       format='''numpy'''						,       )





       def 	_snake_case				(    self						,       __A						,       __A				)						->    List[Any]:
              if self.config_name == "cola":
                     return {"matthews_correlation": matthews_corrcoef(__A						,       __A				)}
              elif self.config_name == "stsb":
                     return pearson_and_spearman(__A						,       __A				)
              elif self.config_name in ["mrpc", "qqp"]:
                     return acc_and_fa(__A						,       __A				)
              elif self.config_name in ["sst2", "mnli", "mnli_mismatched", "mnli_matched", "qnli", "rte", "wnli", "hans"]:
                     return {"accuracy": simple_accuracy(__A						,       __A				)}
              else:
                     raise KeyError(
                         '''You should supply a configuration name selected in '''
                         '''["sst2", "mnli", "mnli_mismatched", "mnli_matched", '''
                         '''"cola", "stsb", "mrpc", "qqp", "qnli", "rte", "wnli", "hans"]'''				)
