



import os
import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from huggingface_hub.file_download import http_get
from requests.exceptions import HTTPError

from transformers import (
    AlbertTokenizer,
    AutoTokenizer,
    BertTokenizer,
    BertTokenizerFast,
    GPTaTokenizerFast,
    is_tokenizers_available,
)
from transformers.testing_utils import TOKEN, USER, is_staging_test, require_tokenizers
from transformers.tokenization_utils import Trie


sys.path.append(str(Path(__file__).parent.parent / 'utils'))

from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
		from test_module.custom_tokenization_fast import CustomTokenizerFast




class 							lowerCAmelCase__  (	unittest.TestCase						):







					def  _lowercase			(       self      :     List[str]):
											# A mock response for an HTTP head request to emulate server down
											A__	:							Optional[Any]						    = mock.Mock()
											A__	:							int						    = 500
											A__	:							Any						    = {}
											A__	:							List[str]						    = HTTPError
											A__	:							Optional[int]						    = {}

											# Download this model to make sure it's in the cache.
											A__	:							Optional[int]						    = BertTokenizer.from_pretrained("hf-internal-testing/tiny-random-bert")

											# Under the mock environment we get a 500 error when trying to reach the tokenizer.
											with mock.patch("requests.Session.request"			,		return_value=_A) as mock_head:
																	A__	:							Dict						    = BertTokenizer.from_pretrained("hf-internal-testing/tiny-random-bert")
																	# This check we did call the fake head request
																	mock_head.assert_called()







					@require_tokenizers
					def  _lowercase			(       self      :     List[Any]):
											# A mock response for an HTTP head request to emulate server down
											A__	:							Union[str, Any]						    = mock.Mock()
											A__	:							Any						    = 500
											A__	:							Optional[Any]						    = {}
											A__	:							Optional[int]						    = HTTPError
											A__	:							Any						    = {}

											# Download this model to make sure it's in the cache.
											A__	:							int						    = GPTaTokenizerFast.from_pretrained("gpt2")

											# Under the mock environment we get a 500 error when trying to reach the tokenizer.
											with mock.patch("requests.Session.request"			,		return_value=_A) as mock_head:
																	A__	:							str						    = GPTaTokenizerFast.from_pretrained("gpt2")
																	# This check we did call the fake head request
																	mock_head.assert_called()







					def  _lowercase			(       self      :     Any):
											# This test is for deprecated behavior and can be removed in v5
											try:
																	A__	:							Optional[int]						    = tempfile.mktemp()
																	with open(_A			,		"wb") as f:
																							http_get("https://huggingface.co/albert-base-v1/resolve/main/spiece.model"			,		_A)

																	A__	:							Tuple						    = AlbertTokenizer.from_pretrained(_A)
											finally:
																	os.remove(_A)

											# Supporting this legacy load introduced a weird bug where the tokenizer would load local files if they are in
											# the current folder and have the right name.
											if os.path.isfile("tokenizer.json"):
																	# We skip the test if the user has a `tokenizer.json` in this folder to avoid deleting it.
																	return
											try:
																	with open("tokenizer.json"			,		"wb") as f:
																							http_get("https://huggingface.co/hf-internal-testing/tiny-random-bert/blob/main/tokenizer.json"			,		_A)
																	A__	:							int						    = AutoTokenizer.from_pretrained("hf-internal-testing/tiny-random-gpt2")
																	# The tiny random BERT has a vocab size of 1024, tiny gpt2 as a vocab size of 1000
																	self.assertEqual(tokenizer.vocab_size			,		1000)
																	# Tokenizer should depend on the remote checkpoint, not the local tokenizer.json file.

											finally:
																	os.remove("tokenizer.json")







					def  _lowercase			(       self      :     Optional[int]):
											# This test is for deprecated behavior and can be removed in v5
											A__	:							Tuple						    = AlbertTokenizer.from_pretrained("https://huggingface.co/albert-base-v1/resolve/main/spiece.model")




@is_staging_test
class 							lowerCAmelCase__  (	unittest.TestCase						):
					__A     :    Any           =						['[UNK]', '[CLS]', '[SEP]', '[PAD]', '[MASK]', 'bla', 'blou']







					@classmethod
					def  _lowercase			(       cls      :     int):
											A__	:							Optional[Any]						    = TOKEN
											HfFolder.save_token(_A)







					@classmethod
					def  _lowercase			(       cls      :     Optional[int]):
											try:
																	delete_repo(token=cls._token			,		repo_id="test-tokenizer")
											except HTTPError:
																	pass

											try:
																	delete_repo(token=cls._token			,		repo_id="valid_org/test-tokenizer-org")
											except HTTPError:
																	pass

											try:
																	delete_repo(token=cls._token			,		repo_id="test-dynamic-tokenizer")
											except HTTPError:
																	pass







					def  _lowercase			(       self      :     List[str]):
											with tempfile.TemporaryDirectory() as tmp_dir:
																	A__	:							Optional[Any]						    = os.path.join(_A			,		"vocab.txt")
																	with open(_A			,		"w"			,		encoding="utf-8") as vocab_writer:
																							vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]))
																	A__	:							Optional[Any]						    = BertTokenizer(_A)

											tokenizer.push_to_hub("test-tokenizer"			,		use_auth_token=self._token)
											A__	:							Optional[Any]						    = BertTokenizer.from_pretrained(F'{USER}/test-tokenizer')
											self.assertDictEqual(new_tokenizer.vocab			,		tokenizer.vocab)

											# Reset repo
											delete_repo(token=self._token			,		repo_id="test-tokenizer")

											# Push to hub via save_pretrained
											with tempfile.TemporaryDirectory() as tmp_dir:
																	tokenizer.save_pretrained(_A			,		repo_id="test-tokenizer"			,		push_to_hub=_A			,		use_auth_token=self._token)

											A__	:							Optional[int]						    = BertTokenizer.from_pretrained(F'{USER}/test-tokenizer')
											self.assertDictEqual(new_tokenizer.vocab			,		tokenizer.vocab)







					def  _lowercase			(       self      :     Tuple):
											with tempfile.TemporaryDirectory() as tmp_dir:
																	A__	:							Dict						    = os.path.join(_A			,		"vocab.txt")
																	with open(_A			,		"w"			,		encoding="utf-8") as vocab_writer:
																							vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]))
																	A__	:							int						    = BertTokenizer(_A)

											tokenizer.push_to_hub("valid_org/test-tokenizer-org"			,		use_auth_token=self._token)
											A__	:							Tuple						    = BertTokenizer.from_pretrained("valid_org/test-tokenizer-org")
											self.assertDictEqual(new_tokenizer.vocab			,		tokenizer.vocab)

											# Reset repo
											delete_repo(token=self._token			,		repo_id="valid_org/test-tokenizer-org")

											# Push to hub via save_pretrained
											with tempfile.TemporaryDirectory() as tmp_dir:
																	tokenizer.save_pretrained(
																	    _A			,		repo_id="valid_org/test-tokenizer-org"			,		push_to_hub=_A			,		use_auth_token=self._token)

											A__	:							Optional[Any]						    = BertTokenizer.from_pretrained("valid_org/test-tokenizer-org")
											self.assertDictEqual(new_tokenizer.vocab			,		tokenizer.vocab)







					@require_tokenizers
					def  _lowercase			(       self      :     Any):
											CustomTokenizer.register_for_auto_class()
											with tempfile.TemporaryDirectory() as tmp_dir:
																	A__	:							Union[str, Any]						    = os.path.join(_A			,		"vocab.txt")
																	with open(_A			,		"w"			,		encoding="utf-8") as vocab_writer:
																							vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]))
																	A__	:							int						    = CustomTokenizer(_A)

											# No fast custom tokenizer
											tokenizer.push_to_hub("test-dynamic-tokenizer"			,		use_auth_token=self._token)

											A__	:							Tuple						    = AutoTokenizer.from_pretrained(F'{USER}/test-dynamic-tokenizer'			,		trust_remote_code=_A)
											# Can't make an isinstance check because the new_model.config is from the CustomTokenizer class of a dynamic module
											self.assertEqual(tokenizer.__class__.__name__			,		"CustomTokenizer")

											# Fast and slow custom tokenizer
											CustomTokenizerFast.register_for_auto_class()
											with tempfile.TemporaryDirectory() as tmp_dir:
																	A__	:							str						    = os.path.join(_A			,		"vocab.txt")
																	with open(_A			,		"w"			,		encoding="utf-8") as vocab_writer:
																							vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]))

																	A__	:							Dict						    = BertTokenizerFast.from_pretrained(_A)
																	bert_tokenizer.save_pretrained(_A)
																	A__	:							List[str]						    = CustomTokenizerFast.from_pretrained(_A)

											tokenizer.push_to_hub("test-dynamic-tokenizer"			,		use_auth_token=self._token)

											A__	:							Any						    = AutoTokenizer.from_pretrained(F'{USER}/test-dynamic-tokenizer'			,		trust_remote_code=_A)
											# Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
											self.assertEqual(tokenizer.__class__.__name__			,		"CustomTokenizerFast")
											A__	:							List[str]						    = AutoTokenizer.from_pretrained(
											    F'{USER}/test-dynamic-tokenizer'			,		use_fast=_A			,		trust_remote_code=_A)
											# Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
											self.assertEqual(tokenizer.__class__.__name__			,		"CustomTokenizer")




class 							lowerCAmelCase__  (	unittest.TestCase						):







					def  _lowercase			(       self      :     List[str]):
											A__	:							Optional[int]						    = Trie()
											trie.add("Hello 友達")
											self.assertEqual(trie.data			,		{"H": {"e": {"l": {"l": {"o": {" ": {"友": {"達": {"": 1}}}}}}}}})
											trie.add("Hello")
											trie.data
											self.assertEqual(trie.data			,		{"H": {"e": {"l": {"l": {"o": {"": 1, " ": {"友": {"達": {"": 1}}}}}}}}})







					def  _lowercase			(       self      :     Optional[Any]):
											A__	:							List[Any]						    = Trie()
											self.assertEqual(trie.split("[CLS] This is a extra_id_100")			,		["[CLS] This is a extra_id_100"])
											trie.add("[CLS]")
											trie.add("extra_id_1")
											trie.add("extra_id_100")
											self.assertEqual(trie.split("[CLS] This is a extra_id_100")			,		["[CLS]", " This is a ", "extra_id_100"])







					def  _lowercase			(       self      :     str):
											A__	:							List[Any]						    = Trie()
											trie.add("A")
											self.assertEqual(trie.split("ABC")			,		["A", "BC"])
											self.assertEqual(trie.split("BCA")			,		["BC", "A"])







					def  _lowercase			(       self      :     Tuple):
											A__	:							Optional[Any]						    = Trie()
											trie.add("TOKEN]")
											trie.add("[SPECIAL_TOKEN]")
											self.assertEqual(trie.split("This is something [SPECIAL_TOKEN]")			,		["This is something ", "[SPECIAL_TOKEN]"])







					def  _lowercase			(       self      :     Optional[Any]):
											A__	:							str						    = Trie()
											trie.add("A")
											trie.add("P")
											trie.add("[SPECIAL_TOKEN]")
											self.assertEqual(trie.split("This is something [SPECIAL_TOKEN]")			,		["This is something ", "[SPECIAL_TOKEN]"])







					def  _lowercase			(       self      :     Tuple):
											A__	:							Union[str, Any]						    = Trie()
											trie.add("AB")
											trie.add("B")
											trie.add("C")
											self.assertEqual(trie.split("ABC")			,		["AB", "C"])







					def  _lowercase			(       self      :     Union[str, Any]):
											A__	:							List[str]						    = Trie()
											trie.add("ABC")
											trie.add("B")
											trie.add("CD")
											self.assertEqual(trie.split("ABCD")			,		["ABC", "D"])







					def  _lowercase			(       self      :     Union[str, Any]):
											# Even if the offsets are wrong, we necessarily output correct string
											# parts.
											A__	:							Optional[Any]						    = Trie()
											A__	:							Any						    = trie.cut_text("ABC"			,		[0, 0, 2, 1, 2, 3])
											self.assertEqual(_A			,		["AB", "C"])




import os
import sys
from contextlib import contextmanager


# Windows only
if os.name == "nt":
		import ctypes
		import msvcrt  # noqa




		class 							lowerCAmelCase__  (	ctypes.Structure						):
							# _fields is a specific attr expected by ctypes
							__A     :    Optional[Any]           =						[('size', ctypes.c_int), ('visible', ctypes.c_byte)]
def 	snake_case__       (					)		->		List[Any]:







						"""simple docstring"""

						if os.name == "nt":
												A__	:							Optional[Any]						    = CursorInfo()
												A__	:							Tuple						    = ctypes.windll.kernelaa.GetStdHandle(-1_1       )
												ctypes.windll.kernelaa.GetConsoleCursorInfo(__lowercase							,     ctypes.byref(__lowercase       )       )
												A__	:							Any						    = False
												ctypes.windll.kernelaa.SetConsoleCursorInfo(__lowercase							,     ctypes.byref(__lowercase       )       )
						elif os.name == "posix":
												sys.stdout.write("\033[?25l"       )
												sys.stdout.flush()
def 	snake_case__       (					)		->		Dict:







						"""simple docstring"""

						if os.name == "nt":
												A__	:							List[str]						    = CursorInfo()
												A__	:							Optional[int]						    = ctypes.windll.kernelaa.GetStdHandle(-1_1       )
												ctypes.windll.kernelaa.GetConsoleCursorInfo(__lowercase							,     ctypes.byref(__lowercase       )       )
												A__	:							int						    = True
												ctypes.windll.kernelaa.SetConsoleCursorInfo(__lowercase							,     ctypes.byref(__lowercase       )       )
						elif os.name == "posix":
												sys.stdout.write("\033[?25h"       )
												sys.stdout.flush()




@contextmanager
def 	snake_case__       (					)		->		Optional[int]:







						"""simple docstring"""

						try:
												hide_cursor()
												yield
						finally:
												show_cursor()