




'''simple docstring'''






import json
import os
import unittest

from transformers import OpenAIGPTTokenizer, OpenAIGPTTokenizerFast
from transformers.models.openai.tokenization_openai import VOCAB_FILES_NAMES
from transformers.testing_utils import require_ftfy, require_spacy, require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin




@require_tokenizers
class  a     (     _SCREAMING_SNAKE_CASE  ,				unittest.TestCase       ):
 _lowerCAmelCase 					=	OpenAIGPTTokenizer
 _lowerCAmelCase 					=	OpenAIGPTTokenizerFast
 _lowerCAmelCase 					=	True
 _lowerCAmelCase 					=	False
 def 							__UpperCAmelCase		( self						)					->							Tuple:
      super().setUp()

      # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
      _a			      =							[
          'l',
          'o',
          'w',
          'e',
          'r',
          's',
          't',
          'i',
          'd',
          'n',
          'w</w>',
          'r</w>',
          't</w>',
          'lo',
          'low',
          'er</w>',
          'low</w>',
          'lowest</w>',
          'newer</w>',
          'wider</w>',
          '<unk>',
      ]
      _a			      =							dict(zip(__magic_name__      ,				range(len(__magic_name__						)						)						)						)
      _a			      =							['#version: 0.2', 'l o', 'lo w', 'e r</w>', '']

      _a			      =							os.path.join(self.tmpdirname      ,				VOCAB_FILES_NAMES['vocab_file']						)
      _a			      =							os.path.join(self.tmpdirname      ,				VOCAB_FILES_NAMES['merges_file']						)
      with open(self.vocab_file      ,				'w'						) as fp:
           fp.write(json.dumps(__magic_name__						)						)
      with open(self.merges_file      ,				'w'						) as fp:
           fp.write('\n'.join(__magic_name__						)						)
 def 							__UpperCAmelCase		( self      ,				__magic_name__						)					->							Dict:
      return "lower newer", "lower newer"
 def 							__UpperCAmelCase		( self						)					->							List[str]:
      _a			      =							OpenAIGPTTokenizer(self.vocab_file      ,				self.merges_file						)

      _a			      =							'lower'
      _a			      =							['low', 'er</w>']
      _a			      =							tokenizer.tokenize(__magic_name__						)
      self.assertListEqual(__magic_name__      ,				__magic_name__						)

      _a			      =							tokens + ['<unk>']
      _a			      =							[14, 15, 20]
      self.assertListEqual(tokenizer.convert_tokens_to_ids(__magic_name__						)      ,				__magic_name__						)
 def 							__UpperCAmelCase		( self      ,				__magic_name__=15						)					->							Dict:
      for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
           with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'						):
                _a			      =							self.rust_tokenizer_class.from_pretrained(__magic_name__      ,				**__magic_name__						)

                # Simple input
                _a			      =							'This is a simple input'
                _a			      =							['This is a simple input 1', 'This is a simple input 2']
                _a			      =							('This is a simple input', 'This is a pair')
                _a			      =							[
                    ('This is a simple input 1', 'This is a simple input 2'),
                    ('This is a simple pair 1', 'This is a simple pair 2'),
                ]

                # Simple input tests
                self.assertRaises(__magic_name__      ,				tokenizer_r.encode      ,				__magic_name__      ,				max_length=__magic_name__      ,				padding='max_length'						)

                # Simple input
                self.assertRaises(__magic_name__      ,				tokenizer_r.encode_plus      ,				__magic_name__      ,				max_length=__magic_name__      ,				padding='max_length'						)

                # Simple input
                self.assertRaises(
                    __magic_name__      ,				tokenizer_r.batch_encode_plus      ,				__magic_name__      ,				max_length=__magic_name__      ,				padding='max_length'      ,				)

                # Pair input
                self.assertRaises(__magic_name__      ,				tokenizer_r.encode      ,				__magic_name__      ,				max_length=__magic_name__      ,				padding='max_length'						)

                # Pair input
                self.assertRaises(__magic_name__      ,				tokenizer_r.encode_plus      ,				__magic_name__      ,				max_length=__magic_name__      ,				padding='max_length'						)

                # Pair input
                self.assertRaises(
                    __magic_name__      ,				tokenizer_r.batch_encode_plus      ,				__magic_name__      ,				max_length=__magic_name__      ,				padding='max_length'      ,				)




 def 							__UpperCAmelCase		( self						)					->							Any:
      pass





@require_ftfy
@require_spacy
@require_tokenizers
class  a     (     _SCREAMING_SNAKE_CASE       ):
 pass





'''simple docstring'''






import datasets


a_					:    List[Any]          =    "\\n@InProceedings{conneau2018xnli,\n  author = \"Conneau, Alexis\n                 and Rinott, Ruty\n                 and Lample, Guillaume\n                 and Williams, Adina\n                 and Bowman, Samuel R.\n                 and Schwenk, Holger\n                 and Stoyanov, Veselin\",\n  title = \"XNLI: Evaluating Cross-lingual Sentence Representations\",\n  booktitle = \"Proceedings of the 2018 Conference on Empirical Methods\n               in Natural Language Processing\",\n  year = \"2018\",\n  publisher = \"Association for Computational Linguistics\",\n  location = \"Brussels, Belgium\",\n}\n"

a_					:    Optional[Any]          =    "\\nXNLI is a subset of a few thousand examples from MNLI which has been translated\ninto a 14 different languages (some low-ish resource). As with MNLI, the goal is\nto predict textual entailment (does sentence A imply/contradict/neither sentence\nB) and is a classification task (given two sentences, predict one of three\nlabels).\n"

a_					:    Optional[Any]          =    "\nComputes XNLI score which is just simple accuracy.\nArgs:\n    predictions: Predicted labels.\n    references: Ground truth labels.\nReturns:\n    'accuracy': accuracy\nExamples:\n\n    >>> predictions = [0, 1]\n    >>> references = [0, 1]\n    >>> xnli_metric = datasets.load_metric(\"xnli\")\n    >>> results = xnli_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {'accuracy': 1.0}\n"






def 	_A				(lowerCAmelCase__  :List[Any]				,						lowerCAmelCase__  :Tuple		)   ->  Dict:




     '''simple docstring'''



     return (preds == labels).mean()






@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION  ,				_KWARGS_DESCRIPTION       )
class  a     (     datasets.Metric       ):
 def 							__UpperCAmelCase		( self						)					->							Dict:
      return datasets.MetricInfo(
          description=_DESCRIPTION      ,				citation=_CITATION      ,				inputs_description=_KWARGS_DESCRIPTION      ,				features=datasets.Features(
              {
                  'predictions': datasets.Value('int64' if self.config_name != 'sts-b' else 'float32'						),
                  'references': datasets.Value('int64' if self.config_name != 'sts-b' else 'float32'						),
              }						)      ,				codebase_urls=[]      ,				reference_urls=[]      ,				format='numpy'      ,				)




 def 							__UpperCAmelCase		( self      ,				__magic_name__      ,				__magic_name__						)					->							Dict:
      return {"accuracy": simple_accuracy(__magic_name__      ,				__magic_name__						)}

