




"""simple docstring"""







import json
import os
import unittest

from transformers import AutoTokenizer, GPTaTokenizer, GPTaTokenizerFast
from transformers.models.gpta.tokenization_gpta import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin

@require_tokenizers
class 				snake_case_			(					lowerCamelCase_	, unittest.TestCase			):







   """simple docstring"""




   A_				      =       GPTaTokenizer
   A_				      =       GPTaTokenizerFast
   A_				      =       True
   A_				      =       {'''add_prefix_space''': True}
   A_				      =       False







   def   UpperCAmelCase__			(		self)				->   List[Any]:
       super().setUp()

       # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
       UpperCamelCase              =	[
           '''l''',
           '''o''',
           '''w''',
           '''e''',
           '''r''',
           '''s''',
           '''t''',
           '''i''',
           '''d''',
           '''n''',
           '''\u0120''',
           '''\u0120l''',
           '''\u0120n''',
           '''\u0120lo''',
           '''\u0120low''',
           '''er''',
           '''\u0120lowest''',
           '''\u0120newer''',
           '''\u0120wider''',
           '''<unk>''',
           '''<|endoftext|>''',
       ]
       UpperCamelCase              =	dict(zip(lowerCamelCase_       ,					range(len(lowerCamelCase_))))
       UpperCamelCase              =	['''#version: 0.2''', '''\u0120 l''', '''\u0120l o''', '''\u0120lo w''', '''e r''', '''''']
       UpperCamelCase              =	{'''unk_token''': '''<unk>'''}

       UpperCamelCase              =	os.path.join(self.tmpdirname       ,					VOCAB_FILES_NAMES['''vocab_file'''])
       UpperCamelCase              =	os.path.join(self.tmpdirname       ,					VOCAB_FILES_NAMES['''merges_file'''])
       with open(self.vocab_file       ,					'''w'''       ,					encoding='''utf-8''') as fp:
           fp.write(json.dumps(lowerCamelCase_) + '''\n''')
       with open(self.merges_file       ,					'''w'''       ,					encoding='''utf-8''') as fp:
           fp.write('''\n'''.join(lowerCamelCase_))







   def   UpperCAmelCase__			(		self       ,					**lowerCamelCase_)				->   Tuple:
       kwargs.update(self.special_tokens_map)
       return GPTaTokenizer.from_pretrained(self.tmpdirname       ,					**lowerCamelCase_)







   def   UpperCAmelCase__			(		self       ,					**lowerCamelCase_)				->   Tuple:
       kwargs.update(self.special_tokens_map)
       return GPTaTokenizerFast.from_pretrained(self.tmpdirname       ,					**lowerCamelCase_)







   def   UpperCAmelCase__			(		self       ,					lowerCamelCase_)				->   Optional[Any]:
       UpperCamelCase              =	'''lower newer'''
       UpperCamelCase              =	'''lower newer'''
       return input_text, output_text







   def   UpperCAmelCase__			(		self)				->   Optional[Any]:
       UpperCamelCase              =	GPTaTokenizer(self.vocab_file       ,					self.merges_file       ,					**self.special_tokens_map)
       UpperCamelCase              =	'''lower newer'''
       UpperCamelCase              =	['''\u0120low''', '''er''', '''\u0120''', '''n''', '''e''', '''w''', '''er''']
       UpperCamelCase              =	tokenizer.tokenize(lowerCamelCase_       ,					add_prefix_space=lowerCamelCase_)
       self.assertListEqual(lowerCamelCase_       ,					lowerCamelCase_)

       UpperCamelCase              =	tokens + [tokenizer.unk_token]
       UpperCamelCase              =	[1_4, 1_5, 1_0, 9, 3, 2, 1_5, 1_9]
       self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCamelCase_)       ,					lowerCamelCase_)







   def   UpperCAmelCase__			(		self)				->   List[Any]:
       if not self.test_rust_tokenizer:
           return

       UpperCamelCase              =	self.get_tokenizer()
       UpperCamelCase              =	self.get_rust_tokenizer(add_prefix_space=lowerCamelCase_)

       UpperCamelCase              =	'''lower newer'''

       # Testing tokenization
       UpperCamelCase              =	tokenizer.tokenize(lowerCamelCase_       ,					add_prefix_space=lowerCamelCase_)
       UpperCamelCase              =	rust_tokenizer.tokenize(lowerCamelCase_)
       self.assertListEqual(lowerCamelCase_       ,					lowerCamelCase_)

       # Testing conversion to ids without special tokens
       UpperCamelCase              =	tokenizer.encode(lowerCamelCase_       ,					add_special_tokens=lowerCamelCase_       ,					add_prefix_space=lowerCamelCase_)
       UpperCamelCase              =	rust_tokenizer.encode(lowerCamelCase_       ,					add_special_tokens=lowerCamelCase_)
       self.assertListEqual(lowerCamelCase_       ,					lowerCamelCase_)

       # Testing conversion to ids with special tokens
       UpperCamelCase              =	self.get_rust_tokenizer(add_prefix_space=lowerCamelCase_)
       UpperCamelCase              =	tokenizer.encode(lowerCamelCase_       ,					add_prefix_space=lowerCamelCase_)
       UpperCamelCase              =	rust_tokenizer.encode(lowerCamelCase_)
       self.assertListEqual(lowerCamelCase_       ,					lowerCamelCase_)

       # Testing the unknown token
       UpperCamelCase              =	tokens + [rust_tokenizer.unk_token]
       UpperCamelCase              =	[1_4, 1_5, 1_0, 9, 3, 2, 1_5, 1_9]
       self.assertListEqual(rust_tokenizer.convert_tokens_to_ids(lowerCamelCase_)       ,					lowerCamelCase_)







   def   UpperCAmelCase__			(		self       ,					*lowerCamelCase_       ,					**lowerCamelCase_)				->   int:
       # It's very difficult to mix/test pretokenization with byte-level
       # And get both GPT2 and Roberta to work at the same time (mostly an issue of adding a space before the string)
       pass







   def   UpperCAmelCase__			(		self       ,					lowerCamelCase_=1_5)				->   List[str]:
       for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
           with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'):
               UpperCamelCase              =	self.rust_tokenizer_class.from_pretrained(lowerCamelCase_       ,					**lowerCamelCase_)

               # Simple input
               UpperCamelCase              =	'''This is a simple input'''
               UpperCamelCase              =	['''This is a simple input 1''', '''This is a simple input 2''']
               UpperCamelCase              =	('''This is a simple input''', '''This is a pair''')
               UpperCamelCase              =	[
                   ('''This is a simple input 1''', '''This is a simple input 2'''),
                   ('''This is a simple pair 1''', '''This is a simple pair 2'''),
               ]

               # Simple input tests
               self.assertRaises(lowerCamelCase_       ,					tokenizer_r.encode       ,					lowerCamelCase_       ,					max_length=lowerCamelCase_       ,					padding='''max_length''')

               # Simple input
               self.assertRaises(lowerCamelCase_       ,					tokenizer_r.encode_plus       ,					lowerCamelCase_       ,					max_length=lowerCamelCase_       ,					padding='''max_length''')

               # Simple input
               self.assertRaises(
                   lowerCamelCase_       ,					tokenizer_r.batch_encode_plus       ,					lowerCamelCase_       ,					max_length=lowerCamelCase_       ,					padding='''max_length'''       ,					)

               # Pair input
               self.assertRaises(lowerCamelCase_       ,					tokenizer_r.encode       ,					lowerCamelCase_       ,					max_length=lowerCamelCase_       ,					padding='''max_length''')

               # Pair input
               self.assertRaises(lowerCamelCase_       ,					tokenizer_r.encode_plus       ,					lowerCamelCase_       ,					max_length=lowerCamelCase_       ,					padding='''max_length''')

               # Pair input
               self.assertRaises(
                   lowerCamelCase_       ,					tokenizer_r.batch_encode_plus       ,					lowerCamelCase_       ,					max_length=lowerCamelCase_       ,					padding='''max_length'''       ,					)







   def   UpperCAmelCase__			(		self)				->   Union[str, Any]:
       UpperCamelCase              =	GPTaTokenizer.from_pretrained(self.tmpdirname       ,					pad_token='''<pad>''')

       # Simple input
       UpperCamelCase              =	'''This is a simple input'''
       UpperCamelCase              =	['''This is a simple input looooooooong''', '''This is a simple input''']
       UpperCamelCase              =	('''This is a simple input''', '''This is a pair''')
       UpperCamelCase              =	[
           ('''This is a simple input loooooong''', '''This is a simple input'''),
           ('''This is a simple pair loooooong''', '''This is a simple pair'''),
       ]

       UpperCamelCase              =	tokenizer.pad_token_id

       UpperCamelCase              =	tokenizer(lowerCamelCase_       ,					padding='''max_length'''       ,					max_length=3_0       ,					return_tensors='''np''')
       UpperCamelCase              =	tokenizer(lowerCamelCase_       ,					padding=lowerCamelCase_       ,					truncate=lowerCamelCase_       ,					return_tensors='''np''')
       UpperCamelCase              =	tokenizer(*lowerCamelCase_       ,					padding='''max_length'''       ,					max_length=6_0       ,					return_tensors='''np''')
       UpperCamelCase              =	tokenizer(lowerCamelCase_       ,					padding=lowerCamelCase_       ,					truncate=lowerCamelCase_       ,					return_tensors='''np''')

       # s
       # test single string max_length padding
       self.assertEqual(out_s['''input_ids'''].shape[-1]       ,					3_0)
       self.assertTrue(pad_token_id in out_s['''input_ids'''])
       self.assertTrue(0 in out_s['''attention_mask'''])

       # s2
       # test automatic padding
       self.assertEqual(out_sa['''input_ids'''].shape[-1]       ,					3_3)
       # long slice doesn't have padding
       self.assertFalse(pad_token_id in out_sa['''input_ids'''][0])
       self.assertFalse(0 in out_sa['''attention_mask'''][0])
       # short slice does have padding
       self.assertTrue(pad_token_id in out_sa['''input_ids'''][1])
       self.assertTrue(0 in out_sa['''attention_mask'''][1])

       # p
       # test single pair max_length padding
       self.assertEqual(out_p['''input_ids'''].shape[-1]       ,					6_0)
       self.assertTrue(pad_token_id in out_p['''input_ids'''])
       self.assertTrue(0 in out_p['''attention_mask'''])

       # p2
       # test automatic padding pair
       self.assertEqual(out_pa['''input_ids'''].shape[-1]       ,					5_2)
       # long slice pair doesn't have padding
       self.assertFalse(pad_token_id in out_pa['''input_ids'''][0])
       self.assertFalse(0 in out_pa['''attention_mask'''][0])
       # short slice pair does have padding
       self.assertTrue(pad_token_id in out_pa['''input_ids'''][1])
       self.assertTrue(0 in out_pa['''attention_mask'''][1])







   def   UpperCAmelCase__			(		self)				->   int:
       UpperCamelCase              =	'''$$$'''
       UpperCamelCase              =	GPTaTokenizer.from_pretrained(self.tmpdirname       ,					bos_token=lowerCamelCase_       ,					add_bos_token=lowerCamelCase_)

       UpperCamelCase              =	'''This is a simple input'''
       UpperCamelCase              =	['''This is a simple input 1''', '''This is a simple input 2''']

       UpperCamelCase              =	tokenizer.bos_token_id

       UpperCamelCase              =	tokenizer(lowerCamelCase_)
       UpperCamelCase              =	tokenizer(lowerCamelCase_)

       self.assertEqual(out_s.input_ids[0]       ,					lowerCamelCase_)
       self.assertTrue(all(o[0] == bos_token_id for o in out_sa.input_ids))

       UpperCamelCase              =	tokenizer.decode(out_s.input_ids)
       UpperCamelCase              =	tokenizer.batch_decode(out_sa.input_ids)

       self.assertEqual(decode_s.split()[0]       ,					lowerCamelCase_)
       self.assertTrue(all(d.split()[0] == bos_token for d in decode_sa))







   def   UpperCAmelCase__			(		self)				->   Dict:
       pass







   def   UpperCAmelCase__			(		self)				->   int:
       # TODO: change to self.get_tokenizers() when the fast version is implemented
       UpperCamelCase              =	[self.get_tokenizer(do_lower_case=lowerCamelCase_       ,					add_bos_token=lowerCamelCase_)]
       for tokenizer in tokenizers:
           with self.subTest(F'{tokenizer.__class__.__name__}'):
               UpperCamelCase              =	'''Encode this.'''
               UpperCamelCase              =	'''This one too please.'''
               UpperCamelCase              =	tokenizer.encode(lowerCamelCase_       ,					add_special_tokens=lowerCamelCase_)
               encoded_sequence += tokenizer.encode(lowerCamelCase_       ,					add_special_tokens=lowerCamelCase_)
               UpperCamelCase              =	tokenizer.encode_plus(
                   lowerCamelCase_       ,					lowerCamelCase_       ,					add_special_tokens=lowerCamelCase_       ,					return_special_tokens_mask=lowerCamelCase_       ,					)
               UpperCamelCase              =	encoded_sequence_dict['''input_ids''']
               UpperCamelCase              =	encoded_sequence_dict['''special_tokens_mask''']
               self.assertEqual(len(lowerCamelCase_)       ,					len(lowerCamelCase_))

               UpperCamelCase              =	[
                   (x if not special_tokens_mask[i] else None) for i, x in enumerate(lowerCamelCase_)
               ]
               UpperCamelCase              =	[x for x in filtered_sequence if x is not None]
               self.assertEqual(lowerCamelCase_       ,					lowerCamelCase_)

@require_tokenizers
class 				snake_case_			(					unittest.TestCase			):







   """simple docstring"""




   def   UpperCAmelCase__			(		self)				->   Optional[int]:
       # More context:
       # https://huggingface.co/wjmcat/opt-350m-paddle/discussions/1
       # https://huggingface.slack.com/archives/C01N44FJDHT/p1653511495183519
       # https://github.com/huggingface/transformers/pull/17088#discussion_r871246439

       UpperCamelCase              =	AutoTokenizer.from_pretrained('''facebook/opt-350m'''       ,					from_slow=lowerCamelCase_)
       UpperCamelCase              =	'''A photo of a cat'''

       UpperCamelCase              =	tokenizer.encode(
           lowerCamelCase_       ,					)
       self.assertEqual(lowerCamelCase_       ,					[2, 2_5_0, 1_3_4_5, 9, 1_0, 4_7_5_8])
       tokenizer.save_pretrained('''test_opt''')

       UpperCamelCase              =	AutoTokenizer.from_pretrained('''./test_opt''')
       UpperCamelCase              =	tokenizer.encode(
           lowerCamelCase_       ,					)
       self.assertEqual(lowerCamelCase_       ,					[2, 2_5_0, 1_3_4_5, 9, 1_0, 4_7_5_8])







   def   UpperCAmelCase__			(		self)				->   Optional[Any]:
       UpperCamelCase              =	AutoTokenizer.from_pretrained('''facebook/opt-350m'''       ,					use_slow=lowerCamelCase_)
       UpperCamelCase              =	'''A photo of a cat'''

       UpperCamelCase              =	tokenizer.encode(
           lowerCamelCase_       ,					)
       # Same as above
       self.assertEqual(lowerCamelCase_       ,					[2, 2_5_0, 1_3_4_5, 9, 1_0, 4_7_5_8])







   @unittest.skip('''This test is failing because of a bug in the fast tokenizer''')
   def   UpperCAmelCase__			(		self)				->   List[str]:
       UpperCamelCase              =	AutoTokenizer.from_pretrained('''facebook/opt-350m'''       ,					from_slow=lowerCamelCase_)

       UpperCamelCase              =	'''bos'''
       UpperCamelCase              =	tokenizer.get_vocab()['''bos''']

       UpperCamelCase              =	'''A photo of a cat'''
       UpperCamelCase              =	tokenizer.encode(
           lowerCamelCase_       ,					)
       # We changed the bos token
       self.assertEqual(lowerCamelCase_       ,					[3_1_9_5_7, 2_5_0, 1_3_4_5, 9, 1_0, 4_7_5_8])
       tokenizer.save_pretrained('''./tok''')
       UpperCamelCase              =	AutoTokenizer.from_pretrained('''./tok''')
       self.assertTrue(tokenizer.is_fast)
       UpperCamelCase              =	tokenizer.encode(
           lowerCamelCase_       ,					)
       self.assertEqual(lowerCamelCase_       ,					[3_1_9_5_7, 2_5_0, 1_3_4_5, 9, 1_0, 4_7_5_8])
"""simple docstring"""


import pytest

from datasets.utils.sharding import _distribute_shards, _number_of_shards_in_gen_kwargs, _split_gen_kwargs






@pytest.mark.parametrize(
    """kwargs, expected""" ,[
        ({"""num_shards""": 0, """max_num_jobs""": 1}, []),
        ({"""num_shards""": 10, """max_num_jobs""": 1}, [range(10		)]),
        ({"""num_shards""": 10, """max_num_jobs""": 10}, [range(lowercase ,i + 1		) for i in range(10		)]),
        ({"""num_shards""": 1, """max_num_jobs""": 10}, [range(1		)]),
        ({"""num_shards""": 10, """max_num_jobs""": 3}, [range(0 ,4		), range(4 ,7		), range(7 ,10		)]),
        ({"""num_shards""": 3, """max_num_jobs""": 10}, [range(0 ,1		), range(1 ,2		), range(2 ,3		)]),
    ] ,)
def      __UpperCAmelCase       (      lowercase ,lowercase		):







  """simple docstring"""
  _UpperCAmelCase    			=   _distribute_shards(**lowercase		)
  assert out == expected






@pytest.mark.parametrize(
    """gen_kwargs, max_num_jobs, expected""" ,[
        ({"""foo""": 0}, 10, [{"""foo""": 0}]),
        ({"""shards""": [0, 1, 2, 3]}, 1, [{"""shards""": [0, 1, 2, 3]}]),
        ({"""shards""": [0, 1, 2, 3]}, 4, [{"""shards""": [0]}, {"""shards""": [1]}, {"""shards""": [2]}, {"""shards""": [3]}]),
        ({"""shards""": [0, 1]}, 4, [{"""shards""": [0]}, {"""shards""": [1]}]),
        ({"""shards""": [0, 1, 2, 3]}, 2, [{"""shards""": [0, 1]}, {"""shards""": [2, 3]}]),
    ] ,)
def      __UpperCAmelCase       (      lowercase ,lowercase ,lowercase		):







  """simple docstring"""
  _UpperCAmelCase    			=   _split_gen_kwargs(lowercase ,lowercase		)
  assert out == expected






@pytest.mark.parametrize(
    """gen_kwargs, expected""" ,[
        ({"""foo""": 0}, 1),
        ({"""shards""": [0]}, 1),
        ({"""shards""": [0, 1, 2, 3]}, 4),
        ({"""shards""": [0, 1, 2, 3], """foo""": 0}, 4),
        ({"""shards""": [0, 1, 2, 3], """other""": (0, 1)}, 4),
        ({"""shards""": [0, 1, 2, 3], """shards2""": [0, 1]}, RuntimeError),
    ] ,)
def      __UpperCAmelCase       (      lowercase ,lowercase		):







  """simple docstring"""
  if expected is RuntimeError:
    with pytest.raises(lowercase		):
      _number_of_shards_in_gen_kwargs(lowercase		)
  else:
    _UpperCAmelCase    			=   _number_of_shards_in_gen_kwargs(lowercase		)
    assert out == expected
