
'''simple docstring'''
import os
import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from huggingface_hub.file_download import http_get
from requests.exceptions import HTTPError

from transformers import (
    AlbertTokenizer,
    AutoTokenizer,
    BertTokenizer,
    BertTokenizerFast,
    GPTaTokenizerFast,
    is_tokenizers_available,
)
from transformers.testing_utils import TOKEN, USER, is_staging_test, require_tokenizers
from transformers.tokenization_utils import Trie


sys.path.append(str(Path(__file__).parent.parent / 'utils'))

from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
						from test_module.custom_tokenization_fast import CustomTokenizerFast


class 			SCREAMING_SNAKE_CASE__      (     unittest.TestCase):







				def 						UpperCAmelCase_    (       self     )->		Union[str, Any]:







											'''simple docstring'''






											UpperCamelCase   			=     mock.Mock()
											UpperCamelCase   			=     500
											UpperCamelCase   			=     {}
											UpperCamelCase   			=     HTTPError
											UpperCamelCase   			=     {}

											# Download this model to make sure it's in the cache.
											UpperCamelCase   			=     BertTokenizer.from_pretrained('hf-internal-testing/tiny-random-bert'     )

											# Under the mock environment we get a 500 error when trying to reach the tokenizer.
											with mock.patch('requests.Session.request'						,				return_value=A_     ) as mock_head:
																		UpperCamelCase   			=     BertTokenizer.from_pretrained('hf-internal-testing/tiny-random-bert'     )
																		# This check we did call the fake head request
																		mock_head.assert_called()







				@require_tokenizers
				def 						UpperCAmelCase_    (       self     )->		int:







											'''simple docstring'''






											UpperCamelCase   			=     mock.Mock()
											UpperCamelCase   			=     500
											UpperCamelCase   			=     {}
											UpperCamelCase   			=     HTTPError
											UpperCamelCase   			=     {}

											# Download this model to make sure it's in the cache.
											UpperCamelCase   			=     GPTaTokenizerFast.from_pretrained('gpt2'     )

											# Under the mock environment we get a 500 error when trying to reach the tokenizer.
											with mock.patch('requests.Session.request'						,				return_value=A_     ) as mock_head:
																		UpperCamelCase   			=     GPTaTokenizerFast.from_pretrained('gpt2'     )
																		# This check we did call the fake head request
																		mock_head.assert_called()







				def 						UpperCAmelCase_    (       self     )->		Tuple:







											'''simple docstring'''






											try:
																		UpperCamelCase   			=     tempfile.mktemp()
																		with open(A_						,				'wb'     ) as f:
																									http_get('https://huggingface.co/albert-base-v1/resolve/main/spiece.model'						,				A_     )

																		UpperCamelCase   			=     AlbertTokenizer.from_pretrained(A_     )
											finally:
																		os.remove(A_     )

											# Supporting this legacy load introduced a weird bug where the tokenizer would load local files if they are in
											# the current folder and have the right name.
											if os.path.isfile('tokenizer.json'     ):
																		# We skip the test if the user has a `tokenizer.json` in this folder to avoid deleting it.
																		return
											try:
																		with open('tokenizer.json'						,				'wb'     ) as f:
																									http_get('https://huggingface.co/hf-internal-testing/tiny-random-bert/blob/main/tokenizer.json'						,				A_     )
																		UpperCamelCase   			=     AutoTokenizer.from_pretrained('hf-internal-testing/tiny-random-gpt2'     )
																		# The tiny random BERT has a vocab size of 1024, tiny gpt2 as a vocab size of 1000
																		self.assertEqual(tokenizer.vocab_size						,				1000     )
																		# Tokenizer should depend on the remote checkpoint, not the local tokenizer.json file.

											finally:
																		os.remove('tokenizer.json'     )







				def 						UpperCAmelCase_    (       self     )->		Any:







											'''simple docstring'''






											UpperCamelCase   			=     AlbertTokenizer.from_pretrained('https://huggingface.co/albert-base-v1/resolve/main/spiece.model'     )


@is_staging_test
class 			SCREAMING_SNAKE_CASE__      (     unittest.TestCase):
				lowerCAmelCase_     			=							["""[UNK]""", """[CLS]""", """[SEP]""", """[PAD]""", """[MASK]""", """bla""", """blou"""]







				@classmethod
				def 						UpperCAmelCase_    (       cls     )->		int:







											'''simple docstring'''






											UpperCamelCase   			=     TOKEN
											HfFolder.save_token(A_     )







				@classmethod
				def 						UpperCAmelCase_    (       cls     )->		Optional[Any]:







											'''simple docstring'''






											try:
																		delete_repo(token=cls._token						,				repo_id='test-tokenizer'     )
											except HTTPError:
																		pass

											try:
																		delete_repo(token=cls._token						,				repo_id='valid_org/test-tokenizer-org'     )
											except HTTPError:
																		pass

											try:
																		delete_repo(token=cls._token						,				repo_id='test-dynamic-tokenizer'     )
											except HTTPError:
																		pass







				def 						UpperCAmelCase_    (       self     )->		List[str]:







											'''simple docstring'''






											with tempfile.TemporaryDirectory() as tmp_dir:
																		UpperCamelCase   			=     os.path.join(A_						,				'vocab.txt'     )
																		with open(A_						,				'w'						,				encoding='utf-8'     ) as vocab_writer:
																									vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]     )     )
																		UpperCamelCase   			=     BertTokenizer(A_     )

											tokenizer.push_to_hub('test-tokenizer'						,				use_auth_token=self._token     )
											UpperCamelCase   			=     BertTokenizer.from_pretrained(F'''{USER}/test-tokenizer'''     )
											self.assertDictEqual(new_tokenizer.vocab						,				tokenizer.vocab     )

											# Reset repo
											delete_repo(token=self._token						,				repo_id='test-tokenizer'     )

											# Push to hub via save_pretrained
											with tempfile.TemporaryDirectory() as tmp_dir:
																		tokenizer.save_pretrained(A_						,				repo_id='test-tokenizer'						,				push_to_hub=A_						,				use_auth_token=self._token     )

											UpperCamelCase   			=     BertTokenizer.from_pretrained(F'''{USER}/test-tokenizer'''     )
											self.assertDictEqual(new_tokenizer.vocab						,				tokenizer.vocab     )







				def 						UpperCAmelCase_    (       self     )->		Optional[int]:







											'''simple docstring'''






											with tempfile.TemporaryDirectory() as tmp_dir:
																		UpperCamelCase   			=     os.path.join(A_						,				'vocab.txt'     )
																		with open(A_						,				'w'						,				encoding='utf-8'     ) as vocab_writer:
																									vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]     )     )
																		UpperCamelCase   			=     BertTokenizer(A_     )

											tokenizer.push_to_hub('valid_org/test-tokenizer-org'						,				use_auth_token=self._token     )
											UpperCamelCase   			=     BertTokenizer.from_pretrained('valid_org/test-tokenizer-org'     )
											self.assertDictEqual(new_tokenizer.vocab						,				tokenizer.vocab     )

											# Reset repo
											delete_repo(token=self._token						,				repo_id='valid_org/test-tokenizer-org'     )

											# Push to hub via save_pretrained
											with tempfile.TemporaryDirectory() as tmp_dir:
																		tokenizer.save_pretrained(
																		    A_						,				repo_id='valid_org/test-tokenizer-org'						,				push_to_hub=A_						,				use_auth_token=self._token     )

											UpperCamelCase   			=     BertTokenizer.from_pretrained('valid_org/test-tokenizer-org'     )
											self.assertDictEqual(new_tokenizer.vocab						,				tokenizer.vocab     )







				@require_tokenizers
				def 						UpperCAmelCase_    (       self     )->		Optional[int]:







											'''simple docstring'''






											CustomTokenizer.register_for_auto_class()
											with tempfile.TemporaryDirectory() as tmp_dir:
																		UpperCamelCase   			=     os.path.join(A_						,				'vocab.txt'     )
																		with open(A_						,				'w'						,				encoding='utf-8'     ) as vocab_writer:
																									vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]     )     )
																		UpperCamelCase   			=     CustomTokenizer(A_     )

											# No fast custom tokenizer
											tokenizer.push_to_hub('test-dynamic-tokenizer'						,				use_auth_token=self._token     )

											UpperCamelCase   			=     AutoTokenizer.from_pretrained(F'''{USER}/test-dynamic-tokenizer'''						,				trust_remote_code=A_     )
											# Can't make an isinstance check because the new_model.config is from the CustomTokenizer class of a dynamic module
											self.assertEqual(tokenizer.__class__.__name__						,				'CustomTokenizer'     )

											# Fast and slow custom tokenizer
											CustomTokenizerFast.register_for_auto_class()
											with tempfile.TemporaryDirectory() as tmp_dir:
																		UpperCamelCase   			=     os.path.join(A_						,				'vocab.txt'     )
																		with open(A_						,				'w'						,				encoding='utf-8'     ) as vocab_writer:
																									vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]     )     )

																		UpperCamelCase   			=     BertTokenizerFast.from_pretrained(A_     )
																		bert_tokenizer.save_pretrained(A_     )
																		UpperCamelCase   			=     CustomTokenizerFast.from_pretrained(A_     )

											tokenizer.push_to_hub('test-dynamic-tokenizer'						,				use_auth_token=self._token     )

											UpperCamelCase   			=     AutoTokenizer.from_pretrained(F'''{USER}/test-dynamic-tokenizer'''						,				trust_remote_code=A_     )
											# Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
											self.assertEqual(tokenizer.__class__.__name__						,				'CustomTokenizerFast'     )
											UpperCamelCase   			=     AutoTokenizer.from_pretrained(
											    F'''{USER}/test-dynamic-tokenizer'''						,				use_fast=A_						,				trust_remote_code=A_     )
											# Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
											self.assertEqual(tokenizer.__class__.__name__						,				'CustomTokenizer'     )


class 			SCREAMING_SNAKE_CASE__      (     unittest.TestCase):







				def 						UpperCAmelCase_    (       self     )->		Dict:







											'''simple docstring'''






											UpperCamelCase   			=     Trie()
											trie.add('Hello 友達'     )
											self.assertEqual(trie.data						,				{'H': {'e': {'l': {'l': {'o': {' ': {'友': {'達': {'': 1}}}}}}}}}     )
											trie.add('Hello'     )
											trie.data
											self.assertEqual(trie.data						,				{'H': {'e': {'l': {'l': {'o': {'': 1, ' ': {'友': {'達': {'': 1}}}}}}}}}     )







				def 						UpperCAmelCase_    (       self     )->		List[Any]:







											'''simple docstring'''






											UpperCamelCase   			=     Trie()
											self.assertEqual(trie.split('[CLS] This is a extra_id_100'     )						,				['[CLS] This is a extra_id_100']     )
											trie.add('[CLS]'     )
											trie.add('extra_id_1'     )
											trie.add('extra_id_100'     )
											self.assertEqual(trie.split('[CLS] This is a extra_id_100'     )						,				['[CLS]', ' This is a ', 'extra_id_100']     )







				def 						UpperCAmelCase_    (       self     )->		List[Any]:







											'''simple docstring'''






											UpperCamelCase   			=     Trie()
											trie.add('A'     )
											self.assertEqual(trie.split('ABC'     )						,				['A', 'BC']     )
											self.assertEqual(trie.split('BCA'     )						,				['BC', 'A']     )







				def 						UpperCAmelCase_    (       self     )->		Optional[Any]:







											'''simple docstring'''






											UpperCamelCase   			=     Trie()
											trie.add('TOKEN]'     )
											trie.add('[SPECIAL_TOKEN]'     )
											self.assertEqual(trie.split('This is something [SPECIAL_TOKEN]'     )						,				['This is something ', '[SPECIAL_TOKEN]']     )







				def 						UpperCAmelCase_    (       self     )->		Any:







											'''simple docstring'''






											UpperCamelCase   			=     Trie()
											trie.add('A'     )
											trie.add('P'     )
											trie.add('[SPECIAL_TOKEN]'     )
											self.assertEqual(trie.split('This is something [SPECIAL_TOKEN]'     )						,				['This is something ', '[SPECIAL_TOKEN]']     )







				def 						UpperCAmelCase_    (       self     )->		List[str]:







											'''simple docstring'''






											UpperCamelCase   			=     Trie()
											trie.add('AB'     )
											trie.add('B'     )
											trie.add('C'     )
											self.assertEqual(trie.split('ABC'     )						,				['AB', 'C']     )







				def 						UpperCAmelCase_    (       self     )->		Optional[int]:







											'''simple docstring'''






											UpperCamelCase   			=     Trie()
											trie.add('ABC'     )
											trie.add('B'     )
											trie.add('CD'     )
											self.assertEqual(trie.split('ABCD'     )						,				['ABC', 'D']     )







				def 						UpperCAmelCase_    (       self     )->		Dict:







											'''simple docstring'''






											UpperCamelCase   			=     Trie()
											UpperCamelCase   			=     trie.cut_text('ABC'						,				[0, 0, 2, 1, 2, 3]     )
											self.assertEqual(A_						,				['AB', 'C']     )






'''simple docstring'''
import colorsys

from PIL import Image  # type: ignore







def 		A_(     A						:		float		,     A						:		float		,     A						:		int):
							UpperCamelCase   			=     x
							UpperCamelCase   			=     y
							for step in range(A):  # noqa: B007
														UpperCamelCase   			=     a * a - b * b + x
														UpperCamelCase   			=     2 * a * b + y
														UpperCamelCase   			=     a_new

														# divergence happens for all complex number with an absolute value
														# greater than 4
														if a * a + b * b > 4:
																					break
							return step / (max_step - 1)







def 		A_(     A						:		float):
							if distance == 1:
														return (0, 0, 0)
							else:
														return (255, 255, 255)







def 		A_(     A						:		float):
							if distance == 1:
														return (0, 0, 0)
							else:
														return tuple(round(i * 255) for i in colorsys.hsv_to_rgb(A		,     1		,     1))







def 		A_(     A						:		int = 800		,     A						:		int = 600		,     A						:		float = -0.6		,     A						:		float = 0		,     A						:		float = 3.2		,     A						:		int = 50		,     A						:		bool = True		,     ):
							UpperCamelCase   			=     Image.new('RGB'		,     (image_width, image_height))
							UpperCamelCase   			=     img.load()

							# loop through the image-coordinates
							for image_x in range(A):
														for image_y in range(A):
																					# determine the figure-coordinates based on the image-coordinates
																					UpperCamelCase   			=     figure_width / image_width * image_height
																					UpperCamelCase   			=     figure_center_x + (image_x / image_width - 0.5) * figure_width
																					UpperCamelCase   			=     figure_center_y + (image_y / image_height - 0.5) * figure_height

																					UpperCamelCase   			=     get_distance(A		,     A		,     A)

																					# color the corresponding pixel based on the selected coloring-function
																					if use_distance_color_coding:
																												UpperCamelCase   			=     get_color_coded_rgb(A)
																					else:
																												UpperCamelCase   			=     get_black_and_white_rgb(A)

							return img


if __name__ == "__main__":
						import doctest

						doctest.testmod()

						# colored version, full figure
						lowerCAmelCase					:    Any           = get_image()

						# uncomment for colored version, different section, zoomed in
						# img = get_image(figure_center_x = -0.6, figure_center_y = -0.4,
						# figure_width = 0.8)

						# uncomment for black and white version, full figure
						# img = get_image(use_distance_color_coding = False)

						# uncomment to save the image
						# img.save("mandelbrot.png")

						img.show()




