



'''simple docstring'''






import json
import os
import unittest

from transformers import AutoTokenizer, GPTaTokenizer, GPTaTokenizerFast
from transformers.models.gpta.tokenization_gpta import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin

@require_tokenizers
class        SCREAMING_SNAKE_CASE(							A__							,	unittest.TestCase    ):







	"""simple docstring"""






	lowerCamelCase__    				= GPTaTokenizer
	lowerCamelCase__    				= GPTaTokenizerFast
	lowerCamelCase__    				= True
	lowerCamelCase__    				= {"""add_prefix_space""": True}
	lowerCamelCase__    				= False







	def   A			(		self    :  Any	)    ->	Tuple:
								super().setUp()

								# Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
								UpperCAmelCase				: Dict											=	[
								    '''l''',
								    '''o''',
								    '''w''',
								    '''e''',
								    '''r''',
								    '''s''',
								    '''t''',
								    '''i''',
								    '''d''',
								    '''n''',
								    '''\u0120''',
								    '''\u0120l''',
								    '''\u0120n''',
								    '''\u0120lo''',
								    '''\u0120low''',
								    '''er''',
								    '''\u0120lowest''',
								    '''\u0120newer''',
								    '''\u0120wider''',
								    '''<unk>''',
								    '''<|endoftext|>''',
								]
								UpperCAmelCase				: Optional[int]											=	dict(zip(__snake_case	,    range(len(__snake_case	)	)	)	)
								UpperCAmelCase				: Dict											=	['''#version: 0.2''', '''\u0120 l''', '''\u0120l o''', '''\u0120lo w''', '''e r''', '''''']
								UpperCAmelCase				: Optional[Any]											=	{'''unk_token''': '''<unk>'''}

								UpperCAmelCase				: Dict											=	os.path.join(self.tmpdirname	,    VOCAB_FILES_NAMES['''vocab_file''']	)
								UpperCAmelCase				: Union[str, Any]											=	os.path.join(self.tmpdirname	,    VOCAB_FILES_NAMES['''merges_file''']	)
								with open(self.vocab_file	,    '''w'''	,    encoding='''utf-8'''	) as fp:
															fp.write(json.dumps(__snake_case	) + '''\n'''	)
								with open(self.merges_file	,    '''w'''	,    encoding='''utf-8'''	) as fp:
															fp.write('''\n'''.join(__snake_case	)	)







	def   A			(		self    :  int	,    **__snake_case    :  Optional[Any]	)    ->	Tuple:
								kwargs.update(self.special_tokens_map	)
								return GPTaTokenizer.from_pretrained(self.tmpdirname	,    **__snake_case	)







	def   A			(		self    :  Optional[Any]	,    **__snake_case    :  List[Any]	)    ->	str:
								kwargs.update(self.special_tokens_map	)
								return GPTaTokenizerFast.from_pretrained(self.tmpdirname	,    **__snake_case	)







	def   A			(		self    :  int	,    __snake_case    :  str	)    ->	Optional[Any]:
								UpperCAmelCase				: Any											=	'''lower newer'''
								UpperCAmelCase				: str											=	'''lower newer'''
								return input_text, output_text







	def   A			(		self    :  Any	)    ->	Dict:
								UpperCAmelCase				: Dict											=	GPTaTokenizer(self.vocab_file	,    self.merges_file	,    **self.special_tokens_map	)
								UpperCAmelCase				: Dict											=	'''lower newer'''
								UpperCAmelCase				: int											=	['''\u0120low''', '''er''', '''\u0120''', '''n''', '''e''', '''w''', '''er''']
								UpperCAmelCase				: Union[str, Any]											=	tokenizer.tokenize(__snake_case	,    add_prefix_space=__snake_case	)
								self.assertListEqual(__snake_case	,    __snake_case	)

								UpperCAmelCase				: List[Any]											=	tokens + [tokenizer.unk_token]
								UpperCAmelCase				: Dict											=	[14, 15, 10, 9, 3, 2, 15, 19]
								self.assertListEqual(tokenizer.convert_tokens_to_ids(__snake_case	)	,    __snake_case	)







	def   A			(		self    :  Dict	)    ->	Optional[int]:
								if not self.test_rust_tokenizer:
															return

								UpperCAmelCase				: Union[str, Any]											=	self.get_tokenizer()
								UpperCAmelCase				: Dict											=	self.get_rust_tokenizer(add_prefix_space=__snake_case	)

								UpperCAmelCase				: List[Any]											=	'''lower newer'''

								# Testing tokenization
								UpperCAmelCase				: Union[str, Any]											=	tokenizer.tokenize(__snake_case	,    add_prefix_space=__snake_case	)
								UpperCAmelCase				: List[str]											=	rust_tokenizer.tokenize(__snake_case	)
								self.assertListEqual(__snake_case	,    __snake_case	)

								# Testing conversion to ids without special tokens
								UpperCAmelCase				: Any											=	tokenizer.encode(__snake_case	,    add_special_tokens=__snake_case	,    add_prefix_space=__snake_case	)
								UpperCAmelCase				: Dict											=	rust_tokenizer.encode(__snake_case	,    add_special_tokens=__snake_case	)
								self.assertListEqual(__snake_case	,    __snake_case	)

								# Testing conversion to ids with special tokens
								UpperCAmelCase				: int											=	self.get_rust_tokenizer(add_prefix_space=__snake_case	)
								UpperCAmelCase				: Optional[int]											=	tokenizer.encode(__snake_case	,    add_prefix_space=__snake_case	)
								UpperCAmelCase				: Optional[int]											=	rust_tokenizer.encode(__snake_case	)
								self.assertListEqual(__snake_case	,    __snake_case	)

								# Testing the unknown token
								UpperCAmelCase				: List[str]											=	tokens + [rust_tokenizer.unk_token]
								UpperCAmelCase				: Optional[int]											=	[14, 15, 10, 9, 3, 2, 15, 19]
								self.assertListEqual(rust_tokenizer.convert_tokens_to_ids(__snake_case	)	,    __snake_case	)







	def   A			(		self    :  str	,    *__snake_case    :  List[Any]	,    **__snake_case    :  str	)    ->	Dict:
								# It's very difficult to mix/test pretokenization with byte-level
								# And get both GPT2 and Roberta to work at the same time (mostly an issue of adding a space before the string)
								pass







	def   A			(		self    :  Optional[Any]	,    __snake_case    :  List[Any]=15	)    ->	int:
								for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
															with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""	):
																						UpperCAmelCase				: Tuple											=	self.rust_tokenizer_class.from_pretrained(__snake_case	,    **__snake_case	)

																						# Simple input
																						UpperCAmelCase				: int											=	'''This is a simple input'''
																						UpperCAmelCase				: Dict											=	['''This is a simple input 1''', '''This is a simple input 2''']
																						UpperCAmelCase				: Tuple											=	('''This is a simple input''', '''This is a pair''')
																						UpperCAmelCase				: Any											=	[
																						    ('''This is a simple input 1''', '''This is a simple input 2'''),
																						    ('''This is a simple pair 1''', '''This is a simple pair 2'''),
																						]

																						# Simple input tests
																						self.assertRaises(__snake_case	,    tokenizer_r.encode	,    __snake_case	,    max_length=__snake_case	,    padding='''max_length'''	)

																						# Simple input
																						self.assertRaises(__snake_case	,    tokenizer_r.encode_plus	,    __snake_case	,    max_length=__snake_case	,    padding='''max_length'''	)

																						# Simple input
																						self.assertRaises(
																						    __snake_case	,    tokenizer_r.batch_encode_plus	,    __snake_case	,    max_length=__snake_case	,    padding='''max_length'''	,    )

																						# Pair input
																						self.assertRaises(__snake_case	,    tokenizer_r.encode	,    __snake_case	,    max_length=__snake_case	,    padding='''max_length'''	)

																						# Pair input
																						self.assertRaises(__snake_case	,    tokenizer_r.encode_plus	,    __snake_case	,    max_length=__snake_case	,    padding='''max_length'''	)

																						# Pair input
																						self.assertRaises(
																						    __snake_case	,    tokenizer_r.batch_encode_plus	,    __snake_case	,    max_length=__snake_case	,    padding='''max_length'''	,    )







	def   A			(		self    :  Optional[int]	)    ->	Optional[int]:
								UpperCAmelCase				: List[Any]											=	GPTaTokenizer.from_pretrained(self.tmpdirname	,    pad_token='''<pad>'''	)

								# Simple input
								UpperCAmelCase				: int											=	'''This is a simple input'''
								UpperCAmelCase				: List[Any]											=	['''This is a simple input looooooooong''', '''This is a simple input''']
								UpperCAmelCase				: Optional[Any]											=	('''This is a simple input''', '''This is a pair''')
								UpperCAmelCase				: List[Any]											=	[
								    ('''This is a simple input loooooong''', '''This is a simple input'''),
								    ('''This is a simple pair loooooong''', '''This is a simple pair'''),
								]

								UpperCAmelCase				: str											=	tokenizer.pad_token_id

								UpperCAmelCase				: Union[str, Any]											=	tokenizer(__snake_case	,    padding='''max_length'''	,    max_length=30	,    return_tensors='''np'''	)
								UpperCAmelCase				: Optional[int]											=	tokenizer(__snake_case	,    padding=__snake_case	,    truncate=__snake_case	,    return_tensors='''np'''	)
								UpperCAmelCase				: int											=	tokenizer(*__snake_case	,    padding='''max_length'''	,    max_length=60	,    return_tensors='''np'''	)
								UpperCAmelCase				: List[Any]											=	tokenizer(__snake_case	,    padding=__snake_case	,    truncate=__snake_case	,    return_tensors='''np'''	)

								# s
								# test single string max_length padding
								self.assertEqual(out_s['''input_ids'''].shape[-1]	,    30	)
								self.assertTrue(pad_token_id in out_s['''input_ids''']	)
								self.assertTrue(0 in out_s['''attention_mask''']	)

								# s2
								# test automatic padding
								self.assertEqual(out_sa['''input_ids'''].shape[-1]	,    33	)
								# long slice doesn't have padding
								self.assertFalse(pad_token_id in out_sa['''input_ids'''][0]	)
								self.assertFalse(0 in out_sa['''attention_mask'''][0]	)
								# short slice does have padding
								self.assertTrue(pad_token_id in out_sa['''input_ids'''][1]	)
								self.assertTrue(0 in out_sa['''attention_mask'''][1]	)

								# p
								# test single pair max_length padding
								self.assertEqual(out_p['''input_ids'''].shape[-1]	,    60	)
								self.assertTrue(pad_token_id in out_p['''input_ids''']	)
								self.assertTrue(0 in out_p['''attention_mask''']	)

								# p2
								# test automatic padding pair
								self.assertEqual(out_pa['''input_ids'''].shape[-1]	,    52	)
								# long slice pair doesn't have padding
								self.assertFalse(pad_token_id in out_pa['''input_ids'''][0]	)
								self.assertFalse(0 in out_pa['''attention_mask'''][0]	)
								# short slice pair does have padding
								self.assertTrue(pad_token_id in out_pa['''input_ids'''][1]	)
								self.assertTrue(0 in out_pa['''attention_mask'''][1]	)







	def   A			(		self    :  List[str]	)    ->	Union[str, Any]:
								UpperCAmelCase				: str											=	'''$$$'''
								UpperCAmelCase				: Optional[int]											=	GPTaTokenizer.from_pretrained(self.tmpdirname	,    bos_token=__snake_case	,    add_bos_token=__snake_case	)

								UpperCAmelCase				: List[str]											=	'''This is a simple input'''
								UpperCAmelCase				: Optional[Any]											=	['''This is a simple input 1''', '''This is a simple input 2''']

								UpperCAmelCase				: Dict											=	tokenizer.bos_token_id

								UpperCAmelCase				: int											=	tokenizer(__snake_case	)
								UpperCAmelCase				: Optional[int]											=	tokenizer(__snake_case	)

								self.assertEqual(out_s.input_ids[0]	,    __snake_case	)
								self.assertTrue(all(o[0] == bos_token_id for o in out_sa.input_ids	)	)

								UpperCAmelCase				: Dict											=	tokenizer.decode(out_s.input_ids	)
								UpperCAmelCase				: Tuple											=	tokenizer.batch_decode(out_sa.input_ids	)

								self.assertEqual(decode_s.split()[0]	,    __snake_case	)
								self.assertTrue(all(d.split()[0] == bos_token for d in decode_sa	)	)







	def   A			(		self    :  List[str]	)    ->	Dict:
								pass







	def   A			(		self    :  Union[str, Any]	)    ->	Any:
								# TODO: change to self.get_tokenizers() when the fast version is implemented
								UpperCAmelCase				: str											=	[self.get_tokenizer(do_lower_case=__snake_case	,    add_bos_token=__snake_case	)]
								for tokenizer in tokenizers:
															with self.subTest(F"""{tokenizer.__class__.__name__}"""	):
																						UpperCAmelCase				: Any											=	'''Encode this.'''
																						UpperCAmelCase				: Optional[Any]											=	'''This one too please.'''
																						UpperCAmelCase				: Dict											=	tokenizer.encode(__snake_case	,    add_special_tokens=__snake_case	)
																						encoded_sequence += tokenizer.encode(__snake_case	,    add_special_tokens=__snake_case	)
																						UpperCAmelCase				: Dict											=	tokenizer.encode_plus(
																						    __snake_case	,    __snake_case	,    add_special_tokens=__snake_case	,    return_special_tokens_mask=__snake_case	,    )
																						UpperCAmelCase				: str											=	encoded_sequence_dict['''input_ids''']
																						UpperCAmelCase				: List[str]											=	encoded_sequence_dict['''special_tokens_mask''']
																						self.assertEqual(len(__snake_case	)	,    len(__snake_case	)	)

																						UpperCAmelCase				: Dict											=	[
																						    (x if not special_tokens_mask[i] else None) for i, x in enumerate(__snake_case	)
																						]
																						UpperCAmelCase				: Optional[int]											=	[x for x in filtered_sequence if x is not None]
																						self.assertEqual(__snake_case	,    __snake_case	)

@require_tokenizers
class        SCREAMING_SNAKE_CASE(							unittest.TestCase    ):







	"""simple docstring"""






	def   A			(		self    :  int	)    ->	Tuple:
								# More context:
								# https://huggingface.co/wjmcat/opt-350m-paddle/discussions/1
								# https://huggingface.slack.com/archives/C01N44FJDHT/p1653511495183519
								# https://github.com/huggingface/transformers/pull/17088#discussion_r871246439

								UpperCAmelCase				: List[Any]											=	AutoTokenizer.from_pretrained('''facebook/opt-350m'''	,    from_slow=__snake_case	)
								UpperCAmelCase				: int											=	'''A photo of a cat'''

								UpperCAmelCase				: Optional[int]											=	tokenizer.encode(
								    __snake_case	,    )
								self.assertEqual(__snake_case	,    [2, 250, 1345, 9, 10, 4758]	)
								tokenizer.save_pretrained('''test_opt'''	)

								UpperCAmelCase				: Dict											=	AutoTokenizer.from_pretrained('''./test_opt'''	)
								UpperCAmelCase				: Union[str, Any]											=	tokenizer.encode(
								    __snake_case	,    )
								self.assertEqual(__snake_case	,    [2, 250, 1345, 9, 10, 4758]	)







	def   A			(		self    :  Any	)    ->	Optional[int]:
								UpperCAmelCase				: Dict											=	AutoTokenizer.from_pretrained('''facebook/opt-350m'''	,    use_slow=__snake_case	)
								UpperCAmelCase				: int											=	'''A photo of a cat'''

								UpperCAmelCase				: str											=	tokenizer.encode(
								    __snake_case	,    )
								# Same as above
								self.assertEqual(__snake_case	,    [2, 250, 1345, 9, 10, 4758]	)







	@unittest.skip('''This test is failing because of a bug in the fast tokenizer'''	)
	def   A			(		self    :  List[Any]	)    ->	Dict:
								UpperCAmelCase				: Dict											=	AutoTokenizer.from_pretrained('''facebook/opt-350m'''	,    from_slow=__snake_case	)

								UpperCAmelCase				: Tuple											=	'''bos'''
								UpperCAmelCase				: Any											=	tokenizer.get_vocab()['''bos''']

								UpperCAmelCase				: Dict											=	'''A photo of a cat'''
								UpperCAmelCase				: Optional[Any]											=	tokenizer.encode(
								    __snake_case	,    )
								# We changed the bos token
								self.assertEqual(__snake_case	,    [31957, 250, 1345, 9, 10, 4758]	)
								tokenizer.save_pretrained('''./tok'''	)
								UpperCAmelCase				: Tuple											=	AutoTokenizer.from_pretrained('''./tok'''	)
								self.assertTrue(tokenizer.is_fast	)
								UpperCAmelCase				: Optional[int]											=	tokenizer.encode(
								    __snake_case	,    )
								self.assertEqual(__snake_case	,    [31957, 250, 1345, 9, 10, 4758]	)










'''simple docstring'''






# Lint as: python3
import dataclasses
import re
from dataclasses import dataclass
from functools import total_ordering
from typing import Optional, Union


UpperCamelCase__:     List[str]										=    re.compile(r"^(?P<major>\d+)" r"\.(?P<minor>\d+)" r"\.(?P<patch>\d+)$")

@total_ordering
@dataclass
class        SCREAMING_SNAKE_CASE:







	"""simple docstring"""






	lowerCamelCase__    				= 42
	lowerCamelCase__    				= None
	lowerCamelCase__    				= None
	lowerCamelCase__    				= None
	lowerCamelCase__    				= None







	def   A			(		self    :  Optional[Any]	)    ->	Dict:
								UpperCAmelCase							,	UpperCAmelCase							,	UpperCAmelCase				: Any											=	_str_to_version_tuple(self.version_str	)







	def __repr__(		self    :  str	)    ->	Optional[Any]:
								return F"""{self.tuple[0]}.{self.tuple[1]}.{self.tuple[2]}"""







	@property
	def   A			(		self    :  Tuple	)    ->	Dict:
								return self.major, self.minor, self.patch







	def   A			(		self    :  str	,    __snake_case    :  Any	)    ->	Tuple:
								if isinstance(__snake_case	,    __snake_case	):
															return Version(__snake_case	)
								elif isinstance(__snake_case	,    __snake_case	):
															return other
								raise TypeError(F"""{other} (type {type(__snake_case	)}) cannot be compared to version."""	)







	def __eq__(		self    :  str	,    __snake_case    :  Union[str, Any]	)    ->	List[Any]:
								try:
															UpperCAmelCase				: Dict											=	self._validate_operand(__snake_case	)
								except (TypeError, ValueError):
															return False
								else:
															return self.tuple == other.tuple







	def __lt__(		self    :  Any	,    __snake_case    :  int	)    ->	Optional[int]:
								UpperCAmelCase				: int											=	self._validate_operand(__snake_case	)
								return self.tuple < other.tuple







	def __hash__(		self    :  List[str]	)    ->	str:
								return hash(_version_tuple_to_str(self.tuple	)	)







	@classmethod
	def   A			(		cls    :  Any	,    __snake_case    :  Tuple	)    ->	Union[str, Any]:
								UpperCAmelCase				: Optional[Any]											=	{f.name for f in dataclasses.fields(cls	)}
								return cls(**{k: v for k, v in dic.items() if k in field_names}	)







	def   A			(		self    :  Dict	)    ->	str:
								return self.version_str


def 				snake_case_  (      _lowerCAmelCase						:		Optional[Any]						)   ->     List[Any]:
							UpperCAmelCase				: Optional[Any]											=	_VERSION_REG.match(_lowerCAmelCase						)
							if not res:
														raise ValueError(f"""Invalid version '{version_str}'. Format should be x.y.z with {{x,y,z}} being digits."""						)
							return tuple(int(_lowerCAmelCase						) for v in [res.group('''major'''						), res.group('''minor'''						), res.group('''patch'''						)]						)




def 				snake_case_  (      _lowerCAmelCase						:		List[str]						)   ->     List[Any]:
							return ".".join(str(_lowerCAmelCase						) for v in version_tuple						)





