


import os
import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from huggingface_hub.file_download import http_get
from requests.exceptions import HTTPError

from transformers import (
    AlbertTokenizer,
    AutoTokenizer,
    BertTokenizer,
    BertTokenizerFast,
    GPTaTokenizerFast,
    is_tokenizers_available,
)
from transformers.testing_utils import TOKEN, USER, is_staging_test, require_tokenizers
from transformers.tokenization_utils import Trie


sys.path.append(str(Path(__file__).parent.parent / '''utils'''))

from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
		from test_module.custom_tokenization_fast import CustomTokenizerFast


class 					lowerCAmelCase_    (       unittest.TestCase):




						def  _snake_case					(	self						:   Optional[Any]						)      ->Tuple:






									"""simple docstring"""






									a__ :Union[str, Any]						  =    mock.Mock()
									a__ :Optional[Any]						  =    500
									a__ :Union[str, Any]						  =    {}
									a__ :List[str]						  =    HTTPError
									a__ :Union[str, Any]						  =    {}

									# Download this model to make sure it's in the cache.
									a__ :Tuple						  =    BertTokenizer.from_pretrained("hf-internal-testing/tiny-random-bert"						)

									# Under the mock environment we get a 500 error when trying to reach the tokenizer.
									with mock.patch("requests.Session.request"   ,     return_value=__A						) as mock_head:
												a__ :Optional[int]						  =    BertTokenizer.from_pretrained("hf-internal-testing/tiny-random-bert"						)
												# This check we did call the fake head request
												mock_head.assert_called()




						@require_tokenizers
						def  _snake_case					(	self						:   Any						)      ->List[str]:






									"""simple docstring"""






									a__ :Union[str, Any]						  =    mock.Mock()
									a__ :str						  =    500
									a__ :Optional[Any]						  =    {}
									a__ :Dict						  =    HTTPError
									a__ :List[Any]						  =    {}

									# Download this model to make sure it's in the cache.
									a__ :Any						  =    GPTaTokenizerFast.from_pretrained("gpt2"						)

									# Under the mock environment we get a 500 error when trying to reach the tokenizer.
									with mock.patch("requests.Session.request"   ,     return_value=__A						) as mock_head:
												a__ :Any						  =    GPTaTokenizerFast.from_pretrained("gpt2"						)
												# This check we did call the fake head request
												mock_head.assert_called()




						def  _snake_case					(	self						:   str						)      ->Union[str, Any]:






									"""simple docstring"""






									try:
												a__ :Optional[int]						  =    tempfile.mktemp()
												with open(__A   ,     "wb"						) as f:
															http_get("https://huggingface.co/albert-base-v1/resolve/main/spiece.model"   ,     __A						)

												a__ :Tuple						  =    AlbertTokenizer.from_pretrained(__A						)
									finally:
												os.remove(__A						)

									# Supporting this legacy load introduced a weird bug where the tokenizer would load local files if they are in
									# the current folder and have the right name.
									if os.path.isfile("tokenizer.json"						):
												# We skip the test if the user has a `tokenizer.json` in this folder to avoid deleting it.
												return
									try:
												with open("tokenizer.json"   ,     "wb"						) as f:
															http_get("https://huggingface.co/hf-internal-testing/tiny-random-bert/blob/main/tokenizer.json"   ,     __A						)
												a__ :Tuple						  =    AutoTokenizer.from_pretrained("hf-internal-testing/tiny-random-gpt2"						)
												# The tiny random BERT has a vocab size of 1024, tiny gpt2 as a vocab size of 1000
												self.assertEqual(tokenizer.vocab_size   ,     1000						)
												# Tokenizer should depend on the remote checkpoint, not the local tokenizer.json file.

									finally:
												os.remove("tokenizer.json"						)






						def  _snake_case					(	self						:   Union[str, Any]						)      ->Any:






									"""simple docstring"""






									a__ :Optional[Any]						  =    AlbertTokenizer.from_pretrained("https://huggingface.co/albert-base-v1/resolve/main/spiece.model"						)






@is_staging_test
class 					lowerCAmelCase_    (       unittest.TestCase):
						lowerCamelCase_         =     ['[UNK]', '[CLS]', '[SEP]', '[PAD]', '[MASK]', 'bla', 'blou']




						@classmethod
						def  _snake_case					(	cls						:   int						)      ->Dict:






									"""simple docstring"""






									a__ :str						  =    TOKEN
									HfFolder.save_token(__A						)




						@classmethod
						def  _snake_case					(	cls						:   Optional[int]						)      ->List[Any]:






									"""simple docstring"""






									try:
												delete_repo(token=cls._token   ,     repo_id="test-tokenizer"						)
									except HTTPError:
												pass

									try:
												delete_repo(token=cls._token   ,     repo_id="valid_org/test-tokenizer-org"						)
									except HTTPError:
												pass

									try:
												delete_repo(token=cls._token   ,     repo_id="test-dynamic-tokenizer"						)
									except HTTPError:
												pass




						def  _snake_case					(	self						:   List[str]						)      ->Any:






									"""simple docstring"""






									with tempfile.TemporaryDirectory() as tmp_dir:
												a__ :List[Any]						  =    os.path.join(__A   ,     "vocab.txt"						)
												with open(__A   ,     "w"   ,     encoding="utf-8"						) as vocab_writer:
															vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]						)						)
												a__ :Optional[int]						  =    BertTokenizer(__A						)

									tokenizer.push_to_hub("test-tokenizer"   ,     use_auth_token=self._token						)
									a__ :Dict						  =    BertTokenizer.from_pretrained(F'''{USER}/test-tokenizer'''						)
									self.assertDictEqual(new_tokenizer.vocab   ,     tokenizer.vocab						)

									# Reset repo
									delete_repo(token=self._token   ,     repo_id="test-tokenizer"						)

									# Push to hub via save_pretrained
									with tempfile.TemporaryDirectory() as tmp_dir:
												tokenizer.save_pretrained(__A   ,     repo_id="test-tokenizer"   ,     push_to_hub=__A   ,     use_auth_token=self._token						)

									a__ :Union[str, Any]						  =    BertTokenizer.from_pretrained(F'''{USER}/test-tokenizer'''						)
									self.assertDictEqual(new_tokenizer.vocab   ,     tokenizer.vocab						)




						def  _snake_case					(	self						:   Dict						)      ->int:






									"""simple docstring"""






									with tempfile.TemporaryDirectory() as tmp_dir:
												a__ :Optional[Any]						  =    os.path.join(__A   ,     "vocab.txt"						)
												with open(__A   ,     "w"   ,     encoding="utf-8"						) as vocab_writer:
															vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]						)						)
												a__ :List[Any]						  =    BertTokenizer(__A						)

									tokenizer.push_to_hub("valid_org/test-tokenizer-org"   ,     use_auth_token=self._token						)
									a__ :str						  =    BertTokenizer.from_pretrained("valid_org/test-tokenizer-org"						)
									self.assertDictEqual(new_tokenizer.vocab   ,     tokenizer.vocab						)

									# Reset repo
									delete_repo(token=self._token   ,     repo_id="valid_org/test-tokenizer-org"						)

									# Push to hub via save_pretrained
									with tempfile.TemporaryDirectory() as tmp_dir:
												tokenizer.save_pretrained(
												    __A   ,     repo_id="valid_org/test-tokenizer-org"   ,     push_to_hub=__A   ,     use_auth_token=self._token						)

									a__ :Optional[int]						  =    BertTokenizer.from_pretrained("valid_org/test-tokenizer-org"						)
									self.assertDictEqual(new_tokenizer.vocab   ,     tokenizer.vocab						)






						@require_tokenizers
						def  _snake_case					(	self						:   Optional[int]						)      ->List[Any]:






									"""simple docstring"""






									CustomTokenizer.register_for_auto_class()
									with tempfile.TemporaryDirectory() as tmp_dir:
												a__ :Any						  =    os.path.join(__A   ,     "vocab.txt"						)
												with open(__A   ,     "w"   ,     encoding="utf-8"						) as vocab_writer:
															vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]						)						)
												a__ :Optional[Any]						  =    CustomTokenizer(__A						)

									# No fast custom tokenizer
									tokenizer.push_to_hub("test-dynamic-tokenizer"   ,     use_auth_token=self._token						)

									a__ :Optional[int]						  =    AutoTokenizer.from_pretrained(F'''{USER}/test-dynamic-tokenizer'''   ,     trust_remote_code=__A						)
									# Can't make an isinstance check because the new_model.config is from the CustomTokenizer class of a dynamic module
									self.assertEqual(tokenizer.__class__.__name__   ,     "CustomTokenizer"						)

									# Fast and slow custom tokenizer
									CustomTokenizerFast.register_for_auto_class()
									with tempfile.TemporaryDirectory() as tmp_dir:
												a__ :Union[str, Any]						  =    os.path.join(__A   ,     "vocab.txt"						)
												with open(__A   ,     "w"   ,     encoding="utf-8"						) as vocab_writer:
															vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]						)						)

												a__ :Dict						  =    BertTokenizerFast.from_pretrained(__A						)
												bert_tokenizer.save_pretrained(__A						)
												a__ :Optional[int]						  =    CustomTokenizerFast.from_pretrained(__A						)

									tokenizer.push_to_hub("test-dynamic-tokenizer"   ,     use_auth_token=self._token						)

									a__ :Tuple						  =    AutoTokenizer.from_pretrained(F'''{USER}/test-dynamic-tokenizer'''   ,     trust_remote_code=__A						)
									# Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
									self.assertEqual(tokenizer.__class__.__name__   ,     "CustomTokenizerFast"						)
									a__ :Tuple						  =    AutoTokenizer.from_pretrained(
									    F'''{USER}/test-dynamic-tokenizer'''   ,     use_fast=__A   ,     trust_remote_code=__A						)
									# Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
									self.assertEqual(tokenizer.__class__.__name__   ,     "CustomTokenizer"						)






class 					lowerCAmelCase_    (       unittest.TestCase):




						def  _snake_case					(	self						:   int						)      ->List[Any]:






									"""simple docstring"""






									a__ :List[str]						  =    Trie()
									trie.add("Hello 友達"						)
									self.assertEqual(trie.data   ,     {"H": {"e": {"l": {"l": {"o": {" ": {"友": {"達": {"": 1}}}}}}}}}						)
									trie.add("Hello"						)
									trie.data
									self.assertEqual(trie.data   ,     {"H": {"e": {"l": {"l": {"o": {"": 1, " ": {"友": {"達": {"": 1}}}}}}}}}						)




						def  _snake_case					(	self						:   Union[str, Any]						)      ->str:






									"""simple docstring"""






									a__ :Optional[Any]						  =    Trie()
									self.assertEqual(trie.split("[CLS] This is a extra_id_100"						)   ,     ["[CLS] This is a extra_id_100"]						)
									trie.add("[CLS]"						)
									trie.add("extra_id_1"						)
									trie.add("extra_id_100"						)
									self.assertEqual(trie.split("[CLS] This is a extra_id_100"						)   ,     ["[CLS]", " This is a ", "extra_id_100"]						)




						def  _snake_case					(	self						:   Tuple						)      ->List[str]:






									"""simple docstring"""






									a__ :str						  =    Trie()
									trie.add("A"						)
									self.assertEqual(trie.split("ABC"						)   ,     ["A", "BC"]						)
									self.assertEqual(trie.split("BCA"						)   ,     ["BC", "A"]						)




						def  _snake_case					(	self						:   Union[str, Any]						)      ->List[str]:






									"""simple docstring"""






									a__ :int						  =    Trie()
									trie.add("TOKEN]"						)
									trie.add("[SPECIAL_TOKEN]"						)
									self.assertEqual(trie.split("This is something [SPECIAL_TOKEN]"						)   ,     ["This is something ", "[SPECIAL_TOKEN]"]						)




						def  _snake_case					(	self						:   str						)      ->str:






									"""simple docstring"""






									a__ :List[Any]						  =    Trie()
									trie.add("A"						)
									trie.add("P"						)
									trie.add("[SPECIAL_TOKEN]"						)
									self.assertEqual(trie.split("This is something [SPECIAL_TOKEN]"						)   ,     ["This is something ", "[SPECIAL_TOKEN]"]						)




						def  _snake_case					(	self						:   List[str]						)      ->Any:






									"""simple docstring"""






									a__ :Union[str, Any]						  =    Trie()
									trie.add("AB"						)
									trie.add("B"						)
									trie.add("C"						)
									self.assertEqual(trie.split("ABC"						)   ,     ["AB", "C"]						)




						def  _snake_case					(	self						:   Optional[Any]						)      ->str:






									"""simple docstring"""






									a__ :int						  =    Trie()
									trie.add("ABC"						)
									trie.add("B"						)
									trie.add("CD"						)
									self.assertEqual(trie.split("ABCD"						)   ,     ["ABC", "D"]						)






						def  _snake_case					(	self						:   Tuple						)      ->List[str]:






									"""simple docstring"""






									a__ :List[str]						  =    Trie()
									a__ :str						  =    trie.cut_text("ABC"   ,     [0, 0, 2, 1, 2, 3]						)
									self.assertEqual(__A   ,     ["AB", "C"]						)






def 				lowerCamelCase__	(      a	:   int = 100				)    ->	int:





			"""simple docstring"""
			a__ :Tuple						  =    n * (n + 1) * (2 * n + 1) / 6
			a__ :Tuple						  =    (n * (n + 1) / 2) ** 2
			return int(square_of_sum - sum_of_squares				)


if __name__ == "__main__":
		print(f'''{solution() = }''')


