






import json
import os
import unittest

from transformers import AutoTokenizer, GPTaTokenizer, GPTaTokenizerFast
from transformers.models.gpta.tokenization_gpta import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin






@require_tokenizers
class 					SCREAMING_SNAKE_CASE_  (       lowercase__  ,						unittest.TestCase ):
       """simple docstring"""


       __lowerCAmelCase    :		Optional[Any]					=GPTaTokenizer
       __lowerCAmelCase    :		Optional[int]					=GPTaTokenizerFast
       __lowerCAmelCase    :		Optional[Any]					=True
       __lowerCAmelCase    :		Any					={'''add_prefix_space''': True}
       __lowerCAmelCase    :		str					=False





       def 				UpperCamelCase__						(				self	:Tuple):



          """simple docstring"""

          super().setUp()

          # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
          _lowercase              =[
              '''l''',
              '''o''',
              '''w''',
              '''e''',
              '''r''',
              '''s''',
              '''t''',
              '''i''',
              '''d''',
              '''n''',
              '''\u0120''',
              '''\u0120l''',
              '''\u0120n''',
              '''\u0120lo''',
              '''\u0120low''',
              '''er''',
              '''\u0120lowest''',
              '''\u0120newer''',
              '''\u0120wider''',
              '''<unk>''',
              '''<|endoftext|>''',
          ]
          _lowercase              =dict(zip(UpperCAmelCase__,      range(len(UpperCAmelCase__))))
          _lowercase              =['''#version: 0.2''', '''\u0120 l''', '''\u0120l o''', '''\u0120lo w''', '''e r''', '''''']
          _lowercase              ={'''unk_token''': '''<unk>'''}

          _lowercase              =os.path.join(self.tmpdirname,      VOCAB_FILES_NAMES['vocab_file'])
          _lowercase              =os.path.join(self.tmpdirname,      VOCAB_FILES_NAMES['merges_file'])
          with open(self.vocab_file,      'w',      encoding='utf-8') as fp:
             fp.write(json.dumps(UpperCAmelCase__) + '\n')
          with open(self.merges_file,      'w',      encoding='utf-8') as fp:
             fp.write('\n'.join(UpperCAmelCase__))





       def 				UpperCamelCase__						(				self	:str,      **snake_case	:Dict):



          """simple docstring"""

          kwargs.update(self.special_tokens_map)
          return GPTaTokenizer.from_pretrained(self.tmpdirname,      **UpperCAmelCase__)





       def 				UpperCamelCase__						(				self	:str,      **snake_case	:List[Any]):



          """simple docstring"""

          kwargs.update(self.special_tokens_map)
          return GPTaTokenizerFast.from_pretrained(self.tmpdirname,      **UpperCAmelCase__)





       def 				UpperCamelCase__						(				self	:Any,      snake_case	:Tuple):



          """simple docstring"""

          _lowercase              ='''lower newer'''
          _lowercase              ='''lower newer'''
          return input_text, output_text





       def 				UpperCamelCase__						(				self	:Dict):



          """simple docstring"""

          _lowercase              =GPTaTokenizer(self.vocab_file,      self.merges_file,      **self.special_tokens_map)
          _lowercase              ='''lower newer'''
          _lowercase              =['''\u0120low''', '''er''', '''\u0120''', '''n''', '''e''', '''w''', '''er''']
          _lowercase              =tokenizer.tokenize(UpperCAmelCase__,      add_prefix_space=UpperCAmelCase__)
          self.assertListEqual(UpperCAmelCase__,      UpperCAmelCase__)

          _lowercase              =tokens + [tokenizer.unk_token]
          _lowercase              =[14, 15, 10, 9, 3, 2, 15, 19]
          self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCAmelCase__),      UpperCAmelCase__)





       def 				UpperCamelCase__						(				self	:Union[str, Any]):



          """simple docstring"""

          if not self.test_rust_tokenizer:
             return

          _lowercase              =self.get_tokenizer()
          _lowercase              =self.get_rust_tokenizer(add_prefix_space=UpperCAmelCase__)

          _lowercase              ='''lower newer'''

          # Testing tokenization
          _lowercase              =tokenizer.tokenize(UpperCAmelCase__,      add_prefix_space=UpperCAmelCase__)
          _lowercase              =rust_tokenizer.tokenize(UpperCAmelCase__)
          self.assertListEqual(UpperCAmelCase__,      UpperCAmelCase__)

          # Testing conversion to ids without special tokens
          _lowercase              =tokenizer.encode(UpperCAmelCase__,      add_special_tokens=UpperCAmelCase__,      add_prefix_space=UpperCAmelCase__)
          _lowercase              =rust_tokenizer.encode(UpperCAmelCase__,      add_special_tokens=UpperCAmelCase__)
          self.assertListEqual(UpperCAmelCase__,      UpperCAmelCase__)

          # Testing conversion to ids with special tokens
          _lowercase              =self.get_rust_tokenizer(add_prefix_space=UpperCAmelCase__)
          _lowercase              =tokenizer.encode(UpperCAmelCase__,      add_prefix_space=UpperCAmelCase__)
          _lowercase              =rust_tokenizer.encode(UpperCAmelCase__)
          self.assertListEqual(UpperCAmelCase__,      UpperCAmelCase__)

          # Testing the unknown token
          _lowercase              =tokens + [rust_tokenizer.unk_token]
          _lowercase              =[14, 15, 10, 9, 3, 2, 15, 19]
          self.assertListEqual(rust_tokenizer.convert_tokens_to_ids(UpperCAmelCase__),      UpperCAmelCase__)





       def 				UpperCamelCase__						(				self	:Tuple,      *snake_case	:List[str],      **snake_case	:int):



          """simple docstring"""

          pass





       def 				UpperCamelCase__						(				self	:Any,      snake_case	:List[Any]=15):



          """simple docstring"""

          for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
             with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''):
                _lowercase              =self.rust_tokenizer_class.from_pretrained(UpperCAmelCase__,      **UpperCAmelCase__)

                # Simple input
                _lowercase              ='''This is a simple input'''
                _lowercase              =['''This is a simple input 1''', '''This is a simple input 2''']
                _lowercase              =('''This is a simple input''', '''This is a pair''')
                _lowercase              =[
                    ('''This is a simple input 1''', '''This is a simple input 2'''),
                    ('''This is a simple pair 1''', '''This is a simple pair 2'''),
                ]

                # Simple input tests
                self.assertRaises(UpperCAmelCase__,      tokenizer_r.encode,      UpperCAmelCase__,      max_length=UpperCAmelCase__,      padding='max_length')

                # Simple input
                self.assertRaises(UpperCAmelCase__,      tokenizer_r.encode_plus,      UpperCAmelCase__,      max_length=UpperCAmelCase__,      padding='max_length')

                # Simple input
                self.assertRaises(
                    UpperCAmelCase__,      tokenizer_r.batch_encode_plus,      UpperCAmelCase__,      max_length=UpperCAmelCase__,      padding='max_length',      )

                # Pair input
                self.assertRaises(UpperCAmelCase__,      tokenizer_r.encode,      UpperCAmelCase__,      max_length=UpperCAmelCase__,      padding='max_length')

                # Pair input
                self.assertRaises(UpperCAmelCase__,      tokenizer_r.encode_plus,      UpperCAmelCase__,      max_length=UpperCAmelCase__,      padding='max_length')

                # Pair input
                self.assertRaises(
                    UpperCAmelCase__,      tokenizer_r.batch_encode_plus,      UpperCAmelCase__,      max_length=UpperCAmelCase__,      padding='max_length',      )





       def 				UpperCamelCase__						(				self	:str):



          """simple docstring"""

          _lowercase              =GPTaTokenizer.from_pretrained(self.tmpdirname,      pad_token='<pad>')

          # Simple input
          _lowercase              ='''This is a simple input'''
          _lowercase              =['''This is a simple input looooooooong''', '''This is a simple input''']
          _lowercase              =('''This is a simple input''', '''This is a pair''')
          _lowercase              =[
              ('''This is a simple input loooooong''', '''This is a simple input'''),
              ('''This is a simple pair loooooong''', '''This is a simple pair'''),
          ]

          _lowercase              =tokenizer.pad_token_id

          _lowercase              =tokenizer(UpperCAmelCase__,      padding='max_length',      max_length=30,      return_tensors='np')
          _lowercase              =tokenizer(UpperCAmelCase__,      padding=UpperCAmelCase__,      truncate=UpperCAmelCase__,      return_tensors='np')
          _lowercase              =tokenizer(*UpperCAmelCase__,      padding='max_length',      max_length=60,      return_tensors='np')
          _lowercase              =tokenizer(UpperCAmelCase__,      padding=UpperCAmelCase__,      truncate=UpperCAmelCase__,      return_tensors='np')

          # s
          # test single string max_length padding
          self.assertEqual(out_s['input_ids'].shape[-1],      30)
          self.assertTrue(pad_token_id in out_s['input_ids'])
          self.assertTrue(0 in out_s['attention_mask'])

          # s2
          # test automatic padding
          self.assertEqual(out_sa['input_ids'].shape[-1],      33)
          # long slice doesn't have padding
          self.assertFalse(pad_token_id in out_sa['input_ids'][0])
          self.assertFalse(0 in out_sa['attention_mask'][0])
          # short slice does have padding
          self.assertTrue(pad_token_id in out_sa['input_ids'][1])
          self.assertTrue(0 in out_sa['attention_mask'][1])

          # p
          # test single pair max_length padding
          self.assertEqual(out_p['input_ids'].shape[-1],      60)
          self.assertTrue(pad_token_id in out_p['input_ids'])
          self.assertTrue(0 in out_p['attention_mask'])

          # p2
          # test automatic padding pair
          self.assertEqual(out_pa['input_ids'].shape[-1],      52)
          # long slice pair doesn't have padding
          self.assertFalse(pad_token_id in out_pa['input_ids'][0])
          self.assertFalse(0 in out_pa['attention_mask'][0])
          # short slice pair does have padding
          self.assertTrue(pad_token_id in out_pa['input_ids'][1])
          self.assertTrue(0 in out_pa['attention_mask'][1])





       def 				UpperCamelCase__						(				self	:Union[str, Any]):



          """simple docstring"""

          _lowercase              ='''$$$'''
          _lowercase              =GPTaTokenizer.from_pretrained(self.tmpdirname,      bos_token=UpperCAmelCase__,      add_bos_token=UpperCAmelCase__)

          _lowercase              ='''This is a simple input'''
          _lowercase              =['''This is a simple input 1''', '''This is a simple input 2''']

          _lowercase              =tokenizer.bos_token_id

          _lowercase              =tokenizer(UpperCAmelCase__)
          _lowercase              =tokenizer(UpperCAmelCase__)

          self.assertEqual(out_s.input_ids[0],      UpperCAmelCase__)
          self.assertTrue(all(o[0] == bos_token_id for o in out_sa.input_ids))

          _lowercase              =tokenizer.decode(out_s.input_ids)
          _lowercase              =tokenizer.batch_decode(out_sa.input_ids)

          self.assertEqual(decode_s.split()[0],      UpperCAmelCase__)
          self.assertTrue(all(d.split()[0] == bos_token for d in decode_sa))





       def 				UpperCamelCase__						(				self	:Dict):



          """simple docstring"""

          pass





       def 				UpperCamelCase__						(				self	:Optional[Any]):



          """simple docstring"""

          _lowercase              =[self.get_tokenizer(do_lower_case=UpperCAmelCase__,      add_bos_token=UpperCAmelCase__)]
          for tokenizer in tokenizers:
             with self.subTest(f'''{tokenizer.__class__.__name__}'''):
                _lowercase              ='''Encode this.'''
                _lowercase              ='''This one too please.'''
                _lowercase              =tokenizer.encode(UpperCAmelCase__,      add_special_tokens=UpperCAmelCase__)
                encoded_sequence += tokenizer.encode(UpperCAmelCase__,      add_special_tokens=UpperCAmelCase__)
                _lowercase              =tokenizer.encode_plus(
                    UpperCAmelCase__,      UpperCAmelCase__,      add_special_tokens=UpperCAmelCase__,      return_special_tokens_mask=UpperCAmelCase__,      )
                _lowercase              =encoded_sequence_dict['''input_ids''']
                _lowercase              =encoded_sequence_dict['''special_tokens_mask''']
                self.assertEqual(len(UpperCAmelCase__),      len(UpperCAmelCase__))

                _lowercase              =[
                    (x if not special_tokens_mask[i] else None) for i, x in enumerate(UpperCAmelCase__)
                ]
                _lowercase              =[x for x in filtered_sequence if x is not None]
                self.assertEqual(UpperCAmelCase__,      UpperCAmelCase__)






@require_tokenizers
class 					SCREAMING_SNAKE_CASE_  (       unittest.TestCase ):
       """simple docstring"""


       def 				UpperCamelCase__						(				self	:Dict):



          """simple docstring"""

          _lowercase              =AutoTokenizer.from_pretrained('facebook/opt-350m',      from_slow=UpperCAmelCase__)
          _lowercase              ='''A photo of a cat'''

          _lowercase              =tokenizer.encode(
              UpperCAmelCase__,      )
          self.assertEqual(UpperCAmelCase__,      [2, 250, 1345, 9, 10, 4758])
          tokenizer.save_pretrained('test_opt')

          _lowercase              =AutoTokenizer.from_pretrained('./test_opt')
          _lowercase              =tokenizer.encode(
              UpperCAmelCase__,      )
          self.assertEqual(UpperCAmelCase__,      [2, 250, 1345, 9, 10, 4758])





       def 				UpperCamelCase__						(				self	:List[str]):



          """simple docstring"""

          _lowercase              =AutoTokenizer.from_pretrained('facebook/opt-350m',      use_slow=UpperCAmelCase__)
          _lowercase              ='''A photo of a cat'''

          _lowercase              =tokenizer.encode(
              UpperCAmelCase__,      )
          # Same as above
          self.assertEqual(UpperCAmelCase__,      [2, 250, 1345, 9, 10, 4758])





       @unittest.skip('This test is failing because of a bug in the fast tokenizer')
       def 				UpperCamelCase__						(				self	:Optional[Any]):



          """simple docstring"""

          _lowercase              =AutoTokenizer.from_pretrained('facebook/opt-350m',      from_slow=UpperCAmelCase__)

          _lowercase              ='''bos'''
          _lowercase              =tokenizer.get_vocab()['''bos''']

          _lowercase              ='''A photo of a cat'''
          _lowercase              =tokenizer.encode(
              UpperCAmelCase__,      )
          # We changed the bos token
          self.assertEqual(UpperCAmelCase__,      [3_1957, 250, 1345, 9, 10, 4758])
          tokenizer.save_pretrained('./tok')
          _lowercase              =AutoTokenizer.from_pretrained('./tok')
          self.assertTrue(tokenizer.is_fast)
          _lowercase              =tokenizer.encode(
              UpperCAmelCase__,      )
          self.assertEqual(UpperCAmelCase__,      [3_1957, 250, 1345, 9, 10, 4758])





'''simple docstring'''





import datasets


UpperCamelCase_    						=       """\
@InProceedings{conneau2018xnli,
  author = \"Conneau, Alexis
                 and Rinott, Ruty
                 and Lample, Guillaume
                 and Williams, Adina
                 and Bowman, Samuel R.
                 and Schwenk, Holger
                 and Stoyanov, Veselin\",
  title = \"XNLI: Evaluating Cross-lingual Sentence Representations\",
  booktitle = \"Proceedings of the 2018 Conference on Empirical Methods
               in Natural Language Processing\",
  year = \"2018\",
  publisher = \"Association for Computational Linguistics\",
  location = \"Brussels, Belgium\",
}
"""

UpperCamelCase_    						=       """\
XNLI is a subset of a few thousand examples from MNLI which has been translated
into a 14 different languages (some low-ish resource). As with MNLI, the goal is
to predict textual entailment (does sentence A imply/contradict/neither sentence
B) and is a classification task (given two sentences, predict one of three
labels).
"""

UpperCamelCase_    						=       """
Computes XNLI score which is just simple accuracy.
Args:
    predictions: Predicted labels.
    references: Ground truth labels.
Returns:
    'accuracy': accuracy
Examples:

    >>> predictions = [0, 1]
    >>> references = [0, 1]
    >>> xnli_metric = datasets.load_metric(\"xnli\")
    >>> results = xnli_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {'accuracy': 1.0}
"""






def 				_lowerCAmelCase			(   __magic_name__						:     Dict   , __magic_name__						:     Union[str, Any]						)				->  Union[str, Any]:
						return (preds == labels).mean()







@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION						,							_KWARGS_DESCRIPTION      )
class 		__SCREAMING_SNAKE_CASE							(			datasets.Metric      ):







				def  lowerCamelCase_	(       self     :		str   ):
										'''simple docstring'''
										return datasets.MetricInfo(
										    description=_DESCRIPTION		,					citation=_CITATION		,					inputs_description=_KWARGS_DESCRIPTION		,					features=datasets.Features(
										        {
										            '''predictions''': datasets.Value('''int64''' if self.config_name != '''sts-b''' else '''float32'''   ),
										            '''references''': datasets.Value('''int64''' if self.config_name != '''sts-b''' else '''float32'''   ),
										        }   )		,					codebase_urls=[]		,					reference_urls=[]		,					format='''numpy'''		,					)







				def  lowerCamelCase_	(       self     :		int		,					UpperCAmelCase__     :		Dict		,					UpperCAmelCase__     :		Optional[Any]   ):
										'''simple docstring'''
										return {"accuracy": simple_accuracy(UpperCAmelCase__		,					UpperCAmelCase__   )}





