



"""simple docstring"""

import json
import os
import shutil
import tempfile
import unittest

from transformers import BatchEncoding, CanineTokenizer
from transformers.testing_utils import require_tokenizers, require_torch
from transformers.tokenization_utils import AddedToken
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin
class     lowerCAmelCase__						(	_lowerCamelCase							,							unittest.TestCase ):


	'''simple docstring'''







	_SCREAMING_SNAKE_CASE	:		List[Any]  						= CanineTokenizer
	_SCREAMING_SNAKE_CASE	:		str  						= False





	def 			_lowerCAmelCase     (      self				:       Optional[int]    )						->	int:





					"""simple docstring"""


					super().setUp()
					SCREAMING_SNAKE_CASE						:							Optional[int]							=     CanineTokenizer()
					tokenizer.save_pretrained(self.tmpdirname    )





	@cached_property
	def 			_lowerCAmelCase     (      self				:       Optional[Any]    )						->	Any:





					"""simple docstring"""


					return CanineTokenizer.from_pretrained('google/canine-s'    )





	def 			_lowerCAmelCase     (      self				:       int     ,       **_SCREAMING_SNAKE_CASE				:       Optional[Any]    )						->	CanineTokenizer:





					"""simple docstring"""


					SCREAMING_SNAKE_CASE						:							Dict							=     self.tokenizer_class.from_pretrained(self.tmpdirname     ,       **_SCREAMING_SNAKE_CASE    )
					SCREAMING_SNAKE_CASE						:							List[Any]							=     1_024
					return tokenizer





	@require_torch
	def 			_lowerCAmelCase     (      self				:       Any    )						->	Optional[int]:





					"""simple docstring"""


					SCREAMING_SNAKE_CASE						:							Any							=     self.canine_tokenizer
					SCREAMING_SNAKE_CASE						:							List[str]							=     ['Life is like a box of chocolates.', 'You never know what you\'re gonna get.']
					# fmt: off
					SCREAMING_SNAKE_CASE						:							Tuple							=     [57_344, 76, 105, 102, 101, 32, 105, 115, 32, 108, 105, 107, 101, 32, 97, 32, 98, 111, 120, 32, 111, 102, 32, 99, 104, 111, 99, 111, 108, 97, 116, 101, 115, 46, 57_345, 0, 0, 0, 0]
					# fmt: on
					SCREAMING_SNAKE_CASE						:							Dict							=     tokenizer(_SCREAMING_SNAKE_CASE     ,       padding=_SCREAMING_SNAKE_CASE     ,       return_tensors='pt'    )
					self.assertIsInstance(_SCREAMING_SNAKE_CASE     ,       _SCREAMING_SNAKE_CASE    )

					SCREAMING_SNAKE_CASE						:							Dict							=     list(batch.input_ids.numpy()[0]    )

					self.assertListEqual(_SCREAMING_SNAKE_CASE     ,       _SCREAMING_SNAKE_CASE    )

					self.assertEqual((2, 39)     ,       batch.input_ids.shape    )
					self.assertEqual((2, 39)     ,       batch.attention_mask.shape    )





	@require_torch
	def 			_lowerCAmelCase     (      self				:       Optional[Any]    )						->	Optional[Any]:





					"""simple docstring"""


					SCREAMING_SNAKE_CASE						:							Optional[int]							=     self.canine_tokenizer
					SCREAMING_SNAKE_CASE						:							Optional[Any]							=     ['Once there was a man.', 'He wrote a test in HuggingFace Tranformers.']
					SCREAMING_SNAKE_CASE						:							int							=     tokenizer(_SCREAMING_SNAKE_CASE     ,       padding=_SCREAMING_SNAKE_CASE     ,       return_tensors='pt'    )
					# check if input_ids, attention_mask and token_type_ids are returned
					self.assertIn('input_ids'     ,       _SCREAMING_SNAKE_CASE    )
					self.assertIn('attention_mask'     ,       _SCREAMING_SNAKE_CASE    )
					self.assertIn('token_type_ids'     ,       _SCREAMING_SNAKE_CASE    )





	@require_torch
	def 			_lowerCAmelCase     (      self				:       int    )						->	List[Any]:





					"""simple docstring"""


					SCREAMING_SNAKE_CASE						:							List[str]							=     self.canine_tokenizer
					SCREAMING_SNAKE_CASE						:							Any							=     [
					    'What\'s the weater?',
					    'It\'s about 25 degrees.',
					]
					SCREAMING_SNAKE_CASE						:							Optional[Any]							=     tokenizer(
					    text_target=_SCREAMING_SNAKE_CASE     ,       max_length=32     ,       padding='max_length'     ,       truncation=_SCREAMING_SNAKE_CASE     ,       return_tensors='pt'    )
					self.assertEqual(32     ,       targets['input_ids'].shape[1]    )





	def 			_lowerCAmelCase     (      self				:       str    )						->	int:





					"""simple docstring"""


					SCREAMING_SNAKE_CASE						:							Dict							=     self.get_tokenizers()
					for tokenizer in tokenizers:
									with self.subTest(f"""{tokenizer.__class__.__name__}"""    ):
													self.assertNotEqual(tokenizer.model_max_length     ,       42    )

        # Now let's start the test
					SCREAMING_SNAKE_CASE						:							str							=     self.get_tokenizers()
					for tokenizer in tokenizers:
									with self.subTest(f"""{tokenizer.__class__.__name__}"""    ):
													# Isolate this from the other tests because we save additional tokens/etc
													SCREAMING_SNAKE_CASE						:							str							=     tempfile.mkdtemp()

													SCREAMING_SNAKE_CASE						:							Any							=     ' He is very happy, UNwant\u00E9d,running'
													SCREAMING_SNAKE_CASE						:							int							=     tokenizer.encode(_SCREAMING_SNAKE_CASE     ,       add_special_tokens=_SCREAMING_SNAKE_CASE    )
													tokenizer.save_pretrained(_SCREAMING_SNAKE_CASE    )

													SCREAMING_SNAKE_CASE						:							str							=     tokenizer.__class__.from_pretrained(_SCREAMING_SNAKE_CASE    )
													SCREAMING_SNAKE_CASE						:							int							=     after_tokenizer.encode(_SCREAMING_SNAKE_CASE     ,       add_special_tokens=_SCREAMING_SNAKE_CASE    )
													self.assertListEqual(_SCREAMING_SNAKE_CASE     ,       _SCREAMING_SNAKE_CASE    )

													shutil.rmtree(_SCREAMING_SNAKE_CASE    )

					SCREAMING_SNAKE_CASE						:							Any							=     self.get_tokenizers(model_max_length=42    )
					for tokenizer in tokenizers:
									with self.subTest(f"""{tokenizer.__class__.__name__}"""    ):
													# Isolate this from the other tests because we save additional tokens/etc
													SCREAMING_SNAKE_CASE						:							int							=     tempfile.mkdtemp()

													SCREAMING_SNAKE_CASE						:							Any							=     ' He is very happy, UNwant\u00E9d,running'

													SCREAMING_SNAKE_CASE						:							Dict							=     tokenizer.additional_special_tokens

													# We can add a new special token for Canine as follows:
													SCREAMING_SNAKE_CASE						:							List[Any]							=     chr(0xe007    )
													additional_special_tokens.append(_SCREAMING_SNAKE_CASE    )
													tokenizer.add_special_tokens({'additional_special_tokens': additional_special_tokens}    )
													SCREAMING_SNAKE_CASE						:							Optional[int]							=     tokenizer.encode(_SCREAMING_SNAKE_CASE     ,       add_special_tokens=_SCREAMING_SNAKE_CASE    )
													tokenizer.save_pretrained(_SCREAMING_SNAKE_CASE    )

													SCREAMING_SNAKE_CASE						:							Tuple							=     tokenizer.__class__.from_pretrained(_SCREAMING_SNAKE_CASE    )
													SCREAMING_SNAKE_CASE						:							List[str]							=     after_tokenizer.encode(_SCREAMING_SNAKE_CASE     ,       add_special_tokens=_SCREAMING_SNAKE_CASE    )
													self.assertListEqual(_SCREAMING_SNAKE_CASE     ,       _SCREAMING_SNAKE_CASE    )
													self.assertIn(_SCREAMING_SNAKE_CASE     ,       after_tokenizer.additional_special_tokens    )
													self.assertEqual(after_tokenizer.model_max_length     ,       42    )

													SCREAMING_SNAKE_CASE						:							Tuple							=     tokenizer.__class__.from_pretrained(_SCREAMING_SNAKE_CASE     ,       model_max_length=43    )
													self.assertEqual(tokenizer.model_max_length     ,       43    )

													shutil.rmtree(_SCREAMING_SNAKE_CASE    )





	def 			_lowerCAmelCase     (      self				:       Union[str, Any]    )						->	Any:





					"""simple docstring"""


					SCREAMING_SNAKE_CASE						:							str							=     self.get_tokenizers(do_lower_case=_SCREAMING_SNAKE_CASE    )
					for tokenizer in tokenizers:
									with self.subTest(f"""{tokenizer.__class__.__name__}"""    ):
													SCREAMING_SNAKE_CASE				,						SCREAMING_SNAKE_CASE						:							Tuple							=     self.get_clean_sequence(_SCREAMING_SNAKE_CASE    )

													# a special token for Canine can be defined as follows:
													SCREAMING_SNAKE_CASE						:							List[Any]							=     0xe005
													SCREAMING_SNAKE_CASE						:							Union[str, Any]							=     chr(_SCREAMING_SNAKE_CASE    )

													tokenizer.add_special_tokens({'cls_token': special_token}    )
													SCREAMING_SNAKE_CASE						:							List[Any]							=     tokenizer.encode(_SCREAMING_SNAKE_CASE     ,       add_special_tokens=_SCREAMING_SNAKE_CASE    )
													self.assertEqual(len(_SCREAMING_SNAKE_CASE    )     ,       1    )

													SCREAMING_SNAKE_CASE						:							Tuple							=     tokenizer.decode(ids + encoded_special_token     ,       clean_up_tokenization_spaces=_SCREAMING_SNAKE_CASE    )
													SCREAMING_SNAKE_CASE						:							Union[str, Any]							=     tokenizer.encode(_SCREAMING_SNAKE_CASE     ,       add_special_tokens=_SCREAMING_SNAKE_CASE    )

													SCREAMING_SNAKE_CASE						:							Union[str, Any]							=     tokenizer.encode(_SCREAMING_SNAKE_CASE     ,       add_special_tokens=_SCREAMING_SNAKE_CASE    )
													SCREAMING_SNAKE_CASE						:							Any							=     tokenizer.encode(_SCREAMING_SNAKE_CASE     ,       add_special_tokens=_SCREAMING_SNAKE_CASE    )
													self.assertEqual(_SCREAMING_SNAKE_CASE     ,       input_encoded + special_token_id    )

													SCREAMING_SNAKE_CASE						:							int							=     tokenizer.decode(_SCREAMING_SNAKE_CASE     ,       skip_special_tokens=_SCREAMING_SNAKE_CASE    )
													self.assertTrue(special_token not in decoded    )





	def 			_lowerCAmelCase     (      self				:       Optional[Any]    )						->	Optional[int]:





					"""simple docstring"""


					SCREAMING_SNAKE_CASE						:							Any							=     self.get_tokenizers(do_lower_case=_SCREAMING_SNAKE_CASE    )
					for tokenizer in tokenizers:
									with self.subTest(f"""{tokenizer.__class__.__name__}"""    ):
													SCREAMING_SNAKE_CASE						:							Tuple							=     chr(0xe005    )
													SCREAMING_SNAKE_CASE						:							Optional[Any]							=     chr(0xe006    )

													# `add_tokens` method stores special tokens only in `tokenizer.unique_no_split_tokens`. (in tokenization_utils.py)
													tokenizer.add_tokens([SPECIAL_TOKEN_1]     ,       special_tokens=_SCREAMING_SNAKE_CASE    )
													# `add_special_tokens` method stores special tokens in `tokenizer.additional_special_tokens`,
													# which also occur in `tokenizer.all_special_tokens`. (in tokenization_utils_base.py)
													tokenizer.add_special_tokens({'additional_special_tokens': [SPECIAL_TOKEN_2]}    )

													SCREAMING_SNAKE_CASE						:							Optional[Any]							=     tokenizer.tokenize(_SCREAMING_SNAKE_CASE    )
													SCREAMING_SNAKE_CASE						:							Any							=     tokenizer.tokenize(_SCREAMING_SNAKE_CASE    )

													self.assertEqual(len(_SCREAMING_SNAKE_CASE    )     ,       1    )
													self.assertEqual(len(_SCREAMING_SNAKE_CASE    )     ,       1    )
													self.assertEqual(token_a[0]     ,       _SCREAMING_SNAKE_CASE    )
													self.assertEqual(token_a[0]     ,       _SCREAMING_SNAKE_CASE    )





	@require_tokenizers
	def 			_lowerCAmelCase     (      self				:       Optional[int]    )						->	Optional[Any]:





					"""simple docstring"""


					SCREAMING_SNAKE_CASE						:							Optional[Any]							=     self.get_tokenizers(do_lower_case=_SCREAMING_SNAKE_CASE    )
					for tokenizer in tokenizers:
									with self.subTest(f"""{tokenizer.__class__.__name__}"""    ):
													# a special token for Canine can be defined as follows:
													SCREAMING_SNAKE_CASE						:							List[str]							=     0xe006
													SCREAMING_SNAKE_CASE						:							int							=     chr(_SCREAMING_SNAKE_CASE    )

													SCREAMING_SNAKE_CASE						:							Union[str, Any]							=     AddedToken(_SCREAMING_SNAKE_CASE     ,       lstrip=_SCREAMING_SNAKE_CASE    )
													tokenizer.add_special_tokens({'additional_special_tokens': [new_token]}    )

													with tempfile.TemporaryDirectory() as tmp_dir_name:
																	tokenizer.save_pretrained(_SCREAMING_SNAKE_CASE    )
																	tokenizer.from_pretrained(_SCREAMING_SNAKE_CASE    )





	def 			_lowerCAmelCase     (      self				:       str    )						->	Tuple:





					"""simple docstring"""


					SCREAMING_SNAKE_CASE						:							int							=     []
					if self.test_slow_tokenizer:
									tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())    )

					if self.test_rust_tokenizer:
									tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())    )

					for tokenizer_class, tokenizer_utils in tokenizer_list:
									with tempfile.TemporaryDirectory() as tmp_dir:
													tokenizer_utils.save_pretrained(_SCREAMING_SNAKE_CASE    )

													with open(os.path.join(_SCREAMING_SNAKE_CASE     ,       'special_tokens_map.json'    )     ,       encoding='utf-8'    ) as json_file:
																	SCREAMING_SNAKE_CASE						:							Optional[int]							=     json.load(_SCREAMING_SNAKE_CASE    )

													with open(os.path.join(_SCREAMING_SNAKE_CASE     ,       'tokenizer_config.json'    )     ,       encoding='utf-8'    ) as json_file:
																	SCREAMING_SNAKE_CASE						:							Optional[int]							=     json.load(_SCREAMING_SNAKE_CASE    )

													# a special token for Canine can be defined as follows:
													SCREAMING_SNAKE_CASE						:							Any							=     0xe006
													SCREAMING_SNAKE_CASE						:							List[str]							=     chr(_SCREAMING_SNAKE_CASE    )

													SCREAMING_SNAKE_CASE						:							Dict							=     [new_token_a]
													SCREAMING_SNAKE_CASE						:							str							=     [new_token_a]

													with open(os.path.join(_SCREAMING_SNAKE_CASE     ,       'special_tokens_map.json'    )     ,       'w'     ,       encoding='utf-8'    ) as outfile:
																	json.dump(_SCREAMING_SNAKE_CASE     ,       _SCREAMING_SNAKE_CASE    )
													with open(os.path.join(_SCREAMING_SNAKE_CASE     ,       'tokenizer_config.json'    )     ,       'w'     ,       encoding='utf-8'    ) as outfile:
																	json.dump(_SCREAMING_SNAKE_CASE     ,       _SCREAMING_SNAKE_CASE    )

													# the following checks allow us to verify that our test works as expected, i.e. that the tokenizer takes
													# into account the new value of additional_special_tokens given in the "tokenizer_config.json" and
													# "special_tokens_map.json" files
													SCREAMING_SNAKE_CASE						:							Optional[Any]							=     tokenizer_class.from_pretrained(_SCREAMING_SNAKE_CASE     ,       extra_ids=0    )
													self.assertIn(_SCREAMING_SNAKE_CASE     ,       tokenizer_without_change_in_init.additional_special_tokens    )
													# self.assertIn("an_additional_special_token",tokenizer_without_change_in_init.get_vocab()) # ByT5Tokenization no vocab
													self.assertEqual(
													    [new_token_a]     ,       tokenizer_without_change_in_init.convert_ids_to_tokens(
													        tokenizer_without_change_in_init.convert_tokens_to_ids([new_token_a]    )    )     ,       )

													SCREAMING_SNAKE_CASE						:							List[Any]							=     0xe007
													SCREAMING_SNAKE_CASE						:							Tuple							=     chr(_SCREAMING_SNAKE_CASE    )
													# Now we test that we can change the value of additional_special_tokens in the from_pretrained
													SCREAMING_SNAKE_CASE						:							List[Any]							=     [AddedToken(_SCREAMING_SNAKE_CASE     ,       lstrip=_SCREAMING_SNAKE_CASE    )]
													SCREAMING_SNAKE_CASE						:							Tuple							=     tokenizer_class.from_pretrained(
													    _SCREAMING_SNAKE_CASE     ,       additional_special_tokens=_SCREAMING_SNAKE_CASE     ,       extra_ids=0    )

													self.assertIn(_SCREAMING_SNAKE_CASE     ,       tokenizer.additional_special_tokens    )
													# self.assertIn(new_token_2,tokenizer.get_vocab()) # ByT5Tokenization no vocab
													self.assertEqual(
													    [new_token_a]     ,       tokenizer.convert_ids_to_tokens(tokenizer.convert_tokens_to_ids([new_token_a]    )    )    )





	@require_tokenizers
	def 			_lowerCAmelCase     (      self				:       str    )						->	List[Any]:





					"""simple docstring"""


					SCREAMING_SNAKE_CASE						:							Tuple							=     self.get_tokenizers(do_lower_case=_SCREAMING_SNAKE_CASE    )
					for tokenizer in tokenizers:
									with self.subTest(f"""{tokenizer.__class__.__name__}"""    ):
													SCREAMING_SNAKE_CASE						:							List[Any]							=     'hello world'
													if self.space_between_special_tokens:
																	SCREAMING_SNAKE_CASE						:							Union[str, Any]							=     '[CLS] hello world [SEP]'
													else:
																	SCREAMING_SNAKE_CASE						:							Tuple							=     input
													SCREAMING_SNAKE_CASE						:							List[Any]							=     tokenizer.encode(_SCREAMING_SNAKE_CASE     ,       add_special_tokens=_SCREAMING_SNAKE_CASE    )
													SCREAMING_SNAKE_CASE						:							int							=     tokenizer.decode(_SCREAMING_SNAKE_CASE     ,       spaces_between_special_tokens=self.space_between_special_tokens    )
													self.assertIn(_SCREAMING_SNAKE_CASE     ,       [output, output.lower()]    )





	def 			_lowerCAmelCase     (      self				:       Tuple    )						->	Tuple:





					"""simple docstring"""


					SCREAMING_SNAKE_CASE						:							List[Any]							=     self.get_tokenizers()
					for tokenizer in tokenizers:
									with self.subTest(f"""{tokenizer.__class__.__name__}"""    ):
													SCREAMING_SNAKE_CASE						:							str							=     [
													    'bos_token',
													    'eos_token',
													    'unk_token',
													    'sep_token',
													    'pad_token',
													    'cls_token',
													    'mask_token',
													]

					SCREAMING_SNAKE_CASE						:							Optional[int]							=     'a'
					SCREAMING_SNAKE_CASE						:							Optional[int]							=     ord(_SCREAMING_SNAKE_CASE    )

					for attr in attributes_list:
									setattr(_SCREAMING_SNAKE_CASE     ,       attr + '_id'     ,       _SCREAMING_SNAKE_CASE    )
									self.assertEqual(getattr(_SCREAMING_SNAKE_CASE     ,       _SCREAMING_SNAKE_CASE    )     ,       _SCREAMING_SNAKE_CASE    )
									self.assertEqual(getattr(_SCREAMING_SNAKE_CASE     ,       attr + '_id'    )     ,       _SCREAMING_SNAKE_CASE    )

									setattr(_SCREAMING_SNAKE_CASE     ,       attr + '_id'     ,       _SCREAMING_SNAKE_CASE    )
									self.assertEqual(getattr(_SCREAMING_SNAKE_CASE     ,       _SCREAMING_SNAKE_CASE    )     ,       _SCREAMING_SNAKE_CASE    )
									self.assertEqual(getattr(_SCREAMING_SNAKE_CASE     ,       attr + '_id'    )     ,       _SCREAMING_SNAKE_CASE    )

					setattr(_SCREAMING_SNAKE_CASE     ,       'additional_special_tokens_ids'     ,       []    )
					self.assertListEqual(getattr(_SCREAMING_SNAKE_CASE     ,       'additional_special_tokens'    )     ,       []    )
					self.assertListEqual(getattr(_SCREAMING_SNAKE_CASE     ,       'additional_special_tokens_ids'    )     ,       []    )

					SCREAMING_SNAKE_CASE						:							List[Any]							=     0xe006
					SCREAMING_SNAKE_CASE						:							Tuple							=     chr(_SCREAMING_SNAKE_CASE    )
					setattr(_SCREAMING_SNAKE_CASE     ,       'additional_special_tokens_ids'     ,       [additional_special_token_id]    )
					self.assertListEqual(getattr(_SCREAMING_SNAKE_CASE     ,       'additional_special_tokens'    )     ,       [additional_special_token]    )
					self.assertListEqual(getattr(_SCREAMING_SNAKE_CASE     ,       'additional_special_tokens_ids'    )     ,       [additional_special_token_id]    )





	def 			_lowerCAmelCase     (      self				:       Optional[int]    )						->	List[Any]:





					"""simple docstring"""


					pass





	def 			_lowerCAmelCase     (      self				:       Any    )						->	str:





					"""simple docstring"""


					pass





	def 			_lowerCAmelCase     (      self				:       Tuple    )						->	Any:





					"""simple docstring"""


					pass





	def 			_lowerCAmelCase     (      self				:       Optional[Any]    )						->	Optional[Any]:





					"""simple docstring"""


					pass





	def 			_lowerCAmelCase     (      self				:       Union[str, Any]    )						->	Optional[int]:





					"""simple docstring"""


					pass





	def 			_lowerCAmelCase     (      self				:       Any    )						->	Tuple:





					"""simple docstring"""


					pass





	def 			_lowerCAmelCase     (      self				:       Tuple    )						->	Any:





					"""simple docstring"""


					pass







	def 			_lowerCAmelCase     (      self				:       int    )						->	List[str]:





					"""simple docstring"""


					pass










"""simple docstring"""

def  __snake_case							(	__A      :						int							,		__A      :						int     )     ->       float:






				'''simple docstring'''
				return base * power(__A							,		(exponent - 1)     ) if exponent else 1


if __name__ == "__main__":
							print('Raise base to the power of exponent using recursion...')
							A_						:    str				=      int(input('Enter the base: ').strip())
							A_						:    Dict				=      int(input('Enter the exponent: ').strip())
							A_						:    List[str]				=      power(base, abs(exponent))
							if exponent < 0:  # power() does not properly deal w/ negative exponents
														A_						:    Any				=      1 / result
							print(f'''{base} to the power of {exponent} is {result}''')





