


'''simple docstring'''





import json
import os
import unittest

from transformers import AutoTokenizer, GPTaTokenizer, GPTaTokenizerFast
from transformers.models.gpta.tokenization_gpta import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin

@require_tokenizers
class 						A__							(		__UpperCAmelCase   ,     unittest.TestCase  ):
						UpperCAmelCase             =						GPTaTokenizer
						UpperCAmelCase             =						GPTaTokenizerFast
						UpperCAmelCase             =						True
						UpperCAmelCase             =						{"add_prefix_space": True}
						UpperCAmelCase             =						False




						def 							__UpperCamelCase				(       self					:      Optional[int]  )							->     Dict:


										"""simple docstring"""





										super().setUp()

										# Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
										_SCREAMING_SNAKE_CASE			  =[
										    '''l''',
										    '''o''',
										    '''w''',
										    '''e''',
										    '''r''',
										    '''s''',
										    '''t''',
										    '''i''',
										    '''d''',
										    '''n''',
										    '''\u0120''',
										    '''\u0120l''',
										    '''\u0120n''',
										    '''\u0120lo''',
										    '''\u0120low''',
										    '''er''',
										    '''\u0120lowest''',
										    '''\u0120newer''',
										    '''\u0120wider''',
										    '''<unk>''',
										    '''<|endoftext|>''',
										]
										_SCREAMING_SNAKE_CASE			  =dict(zip(lowerCAmelCase_				,    range(len(lowerCAmelCase_  )  )  )  )
										_SCREAMING_SNAKE_CASE			  =['''#version: 0.2''', '''\u0120 l''', '''\u0120l o''', '''\u0120lo w''', '''e r''', '''''']
										_SCREAMING_SNAKE_CASE			  ={'''unk_token''': '''<unk>'''}

										_SCREAMING_SNAKE_CASE			  =os.path.join(self.tmpdirname				,    VOCAB_FILES_NAMES['''vocab_file''']  )
										_SCREAMING_SNAKE_CASE			  =os.path.join(self.tmpdirname				,    VOCAB_FILES_NAMES['''merges_file''']  )
										with open(self.vocab_file				,    '''w'''				,    encoding='''utf-8'''  ) as fp:
														fp.write(json.dumps(lowerCAmelCase_  ) + '''\n'''  )
										with open(self.merges_file				,    '''w'''				,    encoding='''utf-8'''  ) as fp:
														fp.write('''\n'''.join(lowerCAmelCase_  )  )




						def 							__UpperCamelCase				(       self					:      Union[str, Any]				,    **_a					:      Any  )							->     Any:


										"""simple docstring"""





										kwargs.update(self.special_tokens_map  )
										return GPTaTokenizer.from_pretrained(self.tmpdirname				,    **lowerCAmelCase_  )




						def 							__UpperCamelCase				(       self					:      str				,    **_a					:      List[str]  )							->     Optional[Any]:


										"""simple docstring"""





										kwargs.update(self.special_tokens_map  )
										return GPTaTokenizerFast.from_pretrained(self.tmpdirname				,    **lowerCAmelCase_  )




						def 							__UpperCamelCase				(       self					:      Optional[int]				,    _a					:      Tuple  )							->     str:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  ='''lower newer'''
										_SCREAMING_SNAKE_CASE			  ='''lower newer'''
										return input_text, output_text




						def 							__UpperCamelCase				(       self					:      List[Any]  )							->     List[Any]:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =GPTaTokenizer(self.vocab_file				,    self.merges_file				,    **self.special_tokens_map  )
										_SCREAMING_SNAKE_CASE			  ='''lower newer'''
										_SCREAMING_SNAKE_CASE			  =['''\u0120low''', '''er''', '''\u0120''', '''n''', '''e''', '''w''', '''er''']
										_SCREAMING_SNAKE_CASE			  =tokenizer.tokenize(lowerCAmelCase_				,    add_prefix_space=lowerCAmelCase_  )
										self.assertListEqual(lowerCAmelCase_				,    lowerCAmelCase_  )

										_SCREAMING_SNAKE_CASE			  =tokens + [tokenizer.unk_token]
										_SCREAMING_SNAKE_CASE			  =[14, 15, 10, 9, 3, 2, 15, 19]
										self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCAmelCase_  )				,    lowerCAmelCase_  )




						def 							__UpperCamelCase				(       self					:      Tuple  )							->     Tuple:


										"""simple docstring"""





										if not self.test_rust_tokenizer:
														return

										_SCREAMING_SNAKE_CASE			  =self.get_tokenizer()
										_SCREAMING_SNAKE_CASE			  =self.get_rust_tokenizer(add_prefix_space=lowerCAmelCase_  )

										_SCREAMING_SNAKE_CASE			  ='''lower newer'''

										# Testing tokenization
										_SCREAMING_SNAKE_CASE			  =tokenizer.tokenize(lowerCAmelCase_				,    add_prefix_space=lowerCAmelCase_  )
										_SCREAMING_SNAKE_CASE			  =rust_tokenizer.tokenize(lowerCAmelCase_  )
										self.assertListEqual(lowerCAmelCase_				,    lowerCAmelCase_  )

										# Testing conversion to ids without special tokens
										_SCREAMING_SNAKE_CASE			  =tokenizer.encode(lowerCAmelCase_				,    add_special_tokens=lowerCAmelCase_				,    add_prefix_space=lowerCAmelCase_  )
										_SCREAMING_SNAKE_CASE			  =rust_tokenizer.encode(lowerCAmelCase_				,    add_special_tokens=lowerCAmelCase_  )
										self.assertListEqual(lowerCAmelCase_				,    lowerCAmelCase_  )

										# Testing conversion to ids with special tokens
										_SCREAMING_SNAKE_CASE			  =self.get_rust_tokenizer(add_prefix_space=lowerCAmelCase_  )
										_SCREAMING_SNAKE_CASE			  =tokenizer.encode(lowerCAmelCase_				,    add_prefix_space=lowerCAmelCase_  )
										_SCREAMING_SNAKE_CASE			  =rust_tokenizer.encode(lowerCAmelCase_  )
										self.assertListEqual(lowerCAmelCase_				,    lowerCAmelCase_  )

										# Testing the unknown token
										_SCREAMING_SNAKE_CASE			  =tokens + [rust_tokenizer.unk_token]
										_SCREAMING_SNAKE_CASE			  =[14, 15, 10, 9, 3, 2, 15, 19]
										self.assertListEqual(rust_tokenizer.convert_tokens_to_ids(lowerCAmelCase_  )				,    lowerCAmelCase_  )




						def 							__UpperCamelCase				(       self					:      List[str]				,    *_a					:      Tuple				,    **_a					:      Any  )							->     Dict:


										"""simple docstring"""





										pass




						def 							__UpperCamelCase				(       self					:      Dict				,    _a					:      Dict=15  )							->     Any:


										"""simple docstring"""





										for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
														with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"  ):
																		_SCREAMING_SNAKE_CASE			  =self.rust_tokenizer_class.from_pretrained(lowerCAmelCase_				,    **lowerCAmelCase_  )

																		# Simple input
																		_SCREAMING_SNAKE_CASE			  ='''This is a simple input'''
																		_SCREAMING_SNAKE_CASE			  =['''This is a simple input 1''', '''This is a simple input 2''']
																		_SCREAMING_SNAKE_CASE			  =('''This is a simple input''', '''This is a pair''')
																		_SCREAMING_SNAKE_CASE			  =[
																		    ('''This is a simple input 1''', '''This is a simple input 2'''),
																		    ('''This is a simple pair 1''', '''This is a simple pair 2'''),
																		]

																		# Simple input tests
																		self.assertRaises(lowerCAmelCase_				,    tokenizer_r.encode				,    lowerCAmelCase_				,    max_length=lowerCAmelCase_				,    padding='''max_length'''  )

																		# Simple input
																		self.assertRaises(lowerCAmelCase_				,    tokenizer_r.encode_plus				,    lowerCAmelCase_				,    max_length=lowerCAmelCase_				,    padding='''max_length'''  )

																		# Simple input
																		self.assertRaises(
																		    lowerCAmelCase_				,    tokenizer_r.batch_encode_plus				,    lowerCAmelCase_				,    max_length=lowerCAmelCase_				,    padding='''max_length'''				,    )

																		# Pair input
																		self.assertRaises(lowerCAmelCase_				,    tokenizer_r.encode				,    lowerCAmelCase_				,    max_length=lowerCAmelCase_				,    padding='''max_length'''  )

																		# Pair input
																		self.assertRaises(lowerCAmelCase_				,    tokenizer_r.encode_plus				,    lowerCAmelCase_				,    max_length=lowerCAmelCase_				,    padding='''max_length'''  )

																		# Pair input
																		self.assertRaises(
																		    lowerCAmelCase_				,    tokenizer_r.batch_encode_plus				,    lowerCAmelCase_				,    max_length=lowerCAmelCase_				,    padding='''max_length'''				,    )




						def 							__UpperCamelCase				(       self					:      str  )							->     Tuple:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =GPTaTokenizer.from_pretrained(self.tmpdirname				,    pad_token='''<pad>'''  )

										# Simple input
										_SCREAMING_SNAKE_CASE			  ='''This is a simple input'''
										_SCREAMING_SNAKE_CASE			  =['''This is a simple input looooooooong''', '''This is a simple input''']
										_SCREAMING_SNAKE_CASE			  =('''This is a simple input''', '''This is a pair''')
										_SCREAMING_SNAKE_CASE			  =[
										    ('''This is a simple input loooooong''', '''This is a simple input'''),
										    ('''This is a simple pair loooooong''', '''This is a simple pair'''),
										]

										_SCREAMING_SNAKE_CASE			  =tokenizer.pad_token_id

										_SCREAMING_SNAKE_CASE			  =tokenizer(lowerCAmelCase_				,    padding='''max_length'''				,    max_length=30				,    return_tensors='''np'''  )
										_SCREAMING_SNAKE_CASE			  =tokenizer(lowerCAmelCase_				,    padding=lowerCAmelCase_				,    truncate=lowerCAmelCase_				,    return_tensors='''np'''  )
										_SCREAMING_SNAKE_CASE			  =tokenizer(*lowerCAmelCase_				,    padding='''max_length'''				,    max_length=60				,    return_tensors='''np'''  )
										_SCREAMING_SNAKE_CASE			  =tokenizer(lowerCAmelCase_				,    padding=lowerCAmelCase_				,    truncate=lowerCAmelCase_				,    return_tensors='''np'''  )

										# s
										# test single string max_length padding
										self.assertEqual(out_s['''input_ids'''].shape[-1]				,    30  )
										self.assertTrue(pad_token_id in out_s['''input_ids''']  )
										self.assertTrue(0 in out_s['''attention_mask''']  )

										# s2
										# test automatic padding
										self.assertEqual(out_sa['''input_ids'''].shape[-1]				,    33  )
										# long slice doesn't have padding
										self.assertFalse(pad_token_id in out_sa['''input_ids'''][0]  )
										self.assertFalse(0 in out_sa['''attention_mask'''][0]  )
										# short slice does have padding
										self.assertTrue(pad_token_id in out_sa['''input_ids'''][1]  )
										self.assertTrue(0 in out_sa['''attention_mask'''][1]  )

										# p
										# test single pair max_length padding
										self.assertEqual(out_p['''input_ids'''].shape[-1]				,    60  )
										self.assertTrue(pad_token_id in out_p['''input_ids''']  )
										self.assertTrue(0 in out_p['''attention_mask''']  )

										# p2
										# test automatic padding pair
										self.assertEqual(out_pa['''input_ids'''].shape[-1]				,    52  )
										# long slice pair doesn't have padding
										self.assertFalse(pad_token_id in out_pa['''input_ids'''][0]  )
										self.assertFalse(0 in out_pa['''attention_mask'''][0]  )
										# short slice pair does have padding
										self.assertTrue(pad_token_id in out_pa['''input_ids'''][1]  )
										self.assertTrue(0 in out_pa['''attention_mask'''][1]  )




						def 							__UpperCamelCase				(       self					:      Tuple  )							->     Tuple:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  ='''$$$'''
										_SCREAMING_SNAKE_CASE			  =GPTaTokenizer.from_pretrained(self.tmpdirname				,    bos_token=lowerCAmelCase_				,    add_bos_token=lowerCAmelCase_  )

										_SCREAMING_SNAKE_CASE			  ='''This is a simple input'''
										_SCREAMING_SNAKE_CASE			  =['''This is a simple input 1''', '''This is a simple input 2''']

										_SCREAMING_SNAKE_CASE			  =tokenizer.bos_token_id

										_SCREAMING_SNAKE_CASE			  =tokenizer(lowerCAmelCase_  )
										_SCREAMING_SNAKE_CASE			  =tokenizer(lowerCAmelCase_  )

										self.assertEqual(out_s.input_ids[0]				,    lowerCAmelCase_  )
										self.assertTrue(all(o[0] == bos_token_id for o in out_sa.input_ids  )  )

										_SCREAMING_SNAKE_CASE			  =tokenizer.decode(out_s.input_ids  )
										_SCREAMING_SNAKE_CASE			  =tokenizer.batch_decode(out_sa.input_ids  )

										self.assertEqual(decode_s.split()[0]				,    lowerCAmelCase_  )
										self.assertTrue(all(d.split()[0] == bos_token for d in decode_sa  )  )




						def 							__UpperCamelCase				(       self					:      str  )							->     int:


										"""simple docstring"""





										pass




						def 							__UpperCamelCase				(       self					:      Union[str, Any]  )							->     str:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =[self.get_tokenizer(do_lower_case=lowerCAmelCase_				,    add_bos_token=lowerCAmelCase_  )]
										for tokenizer in tokenizers:
														with self.subTest(f"{tokenizer.__class__.__name__}"  ):
																		_SCREAMING_SNAKE_CASE			  ='''Encode this.'''
																		_SCREAMING_SNAKE_CASE			  ='''This one too please.'''
																		_SCREAMING_SNAKE_CASE			  =tokenizer.encode(lowerCAmelCase_				,    add_special_tokens=lowerCAmelCase_  )
																		encoded_sequence += tokenizer.encode(lowerCAmelCase_				,    add_special_tokens=lowerCAmelCase_  )
																		_SCREAMING_SNAKE_CASE			  =tokenizer.encode_plus(
																		    lowerCAmelCase_				,    lowerCAmelCase_				,    add_special_tokens=lowerCAmelCase_				,    return_special_tokens_mask=lowerCAmelCase_				,    )
																		_SCREAMING_SNAKE_CASE			  =encoded_sequence_dict['''input_ids''']
																		_SCREAMING_SNAKE_CASE			  =encoded_sequence_dict['''special_tokens_mask''']
																		self.assertEqual(len(lowerCAmelCase_  )				,    len(lowerCAmelCase_  )  )

																		_SCREAMING_SNAKE_CASE			  =[
																		    (x if not special_tokens_mask[i] else None) for i, x in enumerate(lowerCAmelCase_  )
																		]
																		_SCREAMING_SNAKE_CASE			  =[x for x in filtered_sequence if x is not None]
																		self.assertEqual(lowerCAmelCase_				,    lowerCAmelCase_  )

@require_tokenizers
class 						A__							(		unittest.TestCase  ):




						def 							__UpperCamelCase				(       self					:      Optional[int]  )							->     str:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =AutoTokenizer.from_pretrained('''facebook/opt-350m'''				,    from_slow=lowerCAmelCase_  )
										_SCREAMING_SNAKE_CASE			  ='''A photo of a cat'''

										_SCREAMING_SNAKE_CASE			  =tokenizer.encode(
										    lowerCAmelCase_				,    )
										self.assertEqual(lowerCAmelCase_				,    [2, 250, 1345, 9, 10, 4758]  )
										tokenizer.save_pretrained('''test_opt'''  )

										_SCREAMING_SNAKE_CASE			  =AutoTokenizer.from_pretrained('''./test_opt'''  )
										_SCREAMING_SNAKE_CASE			  =tokenizer.encode(
										    lowerCAmelCase_				,    )
										self.assertEqual(lowerCAmelCase_				,    [2, 250, 1345, 9, 10, 4758]  )




						def 							__UpperCamelCase				(       self					:      str  )							->     List[str]:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =AutoTokenizer.from_pretrained('''facebook/opt-350m'''				,    use_slow=lowerCAmelCase_  )
										_SCREAMING_SNAKE_CASE			  ='''A photo of a cat'''

										_SCREAMING_SNAKE_CASE			  =tokenizer.encode(
										    lowerCAmelCase_				,    )
										# Same as above
										self.assertEqual(lowerCAmelCase_				,    [2, 250, 1345, 9, 10, 4758]  )




						@unittest.skip('''This test is failing because of a bug in the fast tokenizer'''  )
						def 							__UpperCamelCase				(       self					:      List[str]  )							->     Any:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =AutoTokenizer.from_pretrained('''facebook/opt-350m'''				,    from_slow=lowerCAmelCase_  )

										_SCREAMING_SNAKE_CASE			  ='''bos'''
										_SCREAMING_SNAKE_CASE			  =tokenizer.get_vocab()['''bos''']

										_SCREAMING_SNAKE_CASE			  ='''A photo of a cat'''
										_SCREAMING_SNAKE_CASE			  =tokenizer.encode(
										    lowerCAmelCase_				,    )
										# We changed the bos token
										self.assertEqual(lowerCAmelCase_				,    [3_1957, 250, 1345, 9, 10, 4758]  )
										tokenizer.save_pretrained('''./tok'''  )
										_SCREAMING_SNAKE_CASE			  =AutoTokenizer.from_pretrained('''./tok'''  )
										self.assertTrue(tokenizer.is_fast  )
										_SCREAMING_SNAKE_CASE			  =tokenizer.encode(
										    lowerCAmelCase_				,    )
										self.assertEqual(lowerCAmelCase_				,    [3_1957, 250, 1345, 9, 10, 4758]  )



import os
import sys
import tempfile

import torch

from .state import AcceleratorState
from .utils import PrecisionType, PrepareForLaunch, is_mps_available, patch_environment







def 						lowerCamelCase(   a__	,a__=()	,a__=None	,a__="no"	,a__="29500"):
				_SCREAMING_SNAKE_CASE			  =False
				_SCREAMING_SNAKE_CASE			  =False
				if any(key.startswith('''KAGGLE''') for key in os.environ.keys()):
								_SCREAMING_SNAKE_CASE			  =True
				elif "IPython" in sys.modules:
								_SCREAMING_SNAKE_CASE			  ='''google.colab''' in str(sys.modules['''IPython'''].get_ipython())

				try:
								_SCREAMING_SNAKE_CASE			  =PrecisionType(mixed_precision.lower())
				except ValueError:
								raise ValueError(
								    f"Unknown mixed_precision mode: {args.mixed_precision.lower()}. Choose between {PrecisionType.list()}.")

				if (in_colab or in_kaggle) and (os.environ.get('''TPU_NAME'''	,a__) is not None):
								# TPU launch
								import torch_xla.distributed.xla_multiprocessing as xmp

								if len(AcceleratorState._shared_state) > 0:
												raise ValueError(
												    '''To train on TPU in Colab or Kaggle Kernel, the `Accelerator` should only be initialized inside '''
												    '''your training function. Restart your notebook and make sure no cells initializes an '''
												    '''`Accelerator`.''')
								if num_processes is None:
												_SCREAMING_SNAKE_CASE			  =8

								_SCREAMING_SNAKE_CASE			  =PrepareForLaunch(a__	,distributed_type='''TPU''')
								print(f"Launching a training on {num_processes} TPU cores.")
								xmp.spawn(a__	,args=a__	,nprocs=a__	,start_method='''fork''')
				elif in_colab:
								# No need for a distributed launch otherwise as it's either CPU or one GPU.
								if torch.cuda.is_available():
												print('''Launching training on one GPU.''')
								else:
												print('''Launching training on one CPU.''')
								function(*a__)
				else:
								if num_processes is None:
												raise ValueError(
												    '''You have to specify the number of GPUs you would like to use, add `num_processes=...` to your call.''')

								if num_processes > 1:
												# Multi-GPU launch
												from torch.multiprocessing import start_processes
												from torch.multiprocessing.spawn import ProcessRaisedException

												if len(AcceleratorState._shared_state) > 0:
																raise ValueError(
																    '''To launch a multi-GPU training from your notebook, the `Accelerator` should only be initialized '''
																    '''inside your training function. Restart your notebook and make sure no cells initializes an '''
																    '''`Accelerator`.''')

												if torch.cuda.is_initialized():
																raise ValueError(
																    '''To launch a multi-GPU training from your notebook, you need to avoid running any instruction '''
																    '''using `torch.cuda` in any cell. Restart your notebook and make sure no cells use any CUDA '''
																    '''function.''')

												# torch.distributed will expect a few environment variable to be here. We set the ones common to each
												# process here (the other ones will be set be the launcher).
												with patch_environment(
												    world_size=a__	,master_addr='''127.0.01'''	,master_port=a__	,mixed_precision=a__):
																_SCREAMING_SNAKE_CASE			  =PrepareForLaunch(a__	,distributed_type='''MULTI_GPU''')
																print(f"Launching training on {num_processes} GPUs.")
																try:
																				start_processes(a__	,args=a__	,nprocs=a__	,start_method='''fork''')
																except ProcessRaisedException as e:
																				if "Cannot re-initialize CUDA in forked subprocess" in e.args[0]:
																								raise RuntimeError(
																								    '''CUDA has been initialized before the `notebook_launcher` could create a forked subprocess. '''
																								    '''This likely stems from an outside import causing issues once the `notebook_launcher()` is called. '''
																								    '''Please review your imports and test them when running the `notebook_launcher()` to identify '''
																								    '''which one is problematic.''') from e

								else:
												# No need for a distributed launch otherwise as it's either CPU, GPU or MPS.
												if is_mps_available():
																_SCREAMING_SNAKE_CASE			  ='''1'''
																print('''Launching training on MPS.''')
												elif torch.cuda.is_available():
																print('''Launching training on one GPU.''')
												else:
																print('''Launching training on CPU.''')
												function(*a__)







def 						lowerCamelCase(   a__	,a__=()	,a__=2):
				from torch.multiprocessing import start_processes

				with tempfile.NamedTemporaryFile() as tmp_file:
								# torch.distributed will expect a few environment variable to be here. We set the ones common to each
								# process here (the other ones will be set be the launcher).
								with patch_environment(
								    world_size=a__	,master_addr='''127.0.01'''	,master_port='''29500'''	,accelerate_mixed_precision='''no'''	,accelerate_debug_rdv_file=tmp_file.name	,accelerate_use_cpu='''yes'''	,):
												_SCREAMING_SNAKE_CASE			  =PrepareForLaunch(a__	,debug=a__)
												start_processes(a__	,args=a__	,nprocs=a__	,start_method='''fork''')