



import os
import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from huggingface_hub.file_download import http_get
from requests.exceptions import HTTPError

from transformers import (
    AlbertTokenizer,
    AutoTokenizer,
    BertTokenizer,
    BertTokenizerFast,
    GPTaTokenizerFast,
    is_tokenizers_available,
)
from transformers.testing_utils import TOKEN, USER, is_staging_test, require_tokenizers
from transformers.tokenization_utils import Trie


sys.path.append(str(Path(__file__).parent.parent / '''utils'''))

from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
	from test_module.custom_tokenization_fast import CustomTokenizerFast






class 						lowercase		(					unittest.TestCase):






	"""simple docstring"""


	def     _SCREAMING_SNAKE_CASE						(      self      :		Dict	)     ->					Union[str, Any]:
							# A mock response for an HTTP head request to emulate server down
							UpperCAmelCase_=  mock.Mock()
							UpperCAmelCase_=  500
							UpperCAmelCase_=  {}
							UpperCAmelCase_=  HTTPError
							UpperCAmelCase_=  {}

							# Download this model to make sure it's in the cache.
							UpperCAmelCase_=  BertTokenizer.from_pretrained("""hf-internal-testing/tiny-random-bert"""	)

							# Under the mock environment we get a 500 error when trying to reach the tokenizer.
							with mock.patch("""requests.Session.request"""		,    return_value=_lowerCamelCase	) as mock_head:
													UpperCAmelCase_=  BertTokenizer.from_pretrained("""hf-internal-testing/tiny-random-bert"""	)
													# This check we did call the fake head request
													mock_head.assert_called()




	@require_tokenizers
	def     _SCREAMING_SNAKE_CASE						(      self      :		Tuple	)     ->					List[str]:
							# A mock response for an HTTP head request to emulate server down
							UpperCAmelCase_=  mock.Mock()
							UpperCAmelCase_=  500
							UpperCAmelCase_=  {}
							UpperCAmelCase_=  HTTPError
							UpperCAmelCase_=  {}

							# Download this model to make sure it's in the cache.
							UpperCAmelCase_=  GPTaTokenizerFast.from_pretrained("""gpt2"""	)

							# Under the mock environment we get a 500 error when trying to reach the tokenizer.
							with mock.patch("""requests.Session.request"""		,    return_value=_lowerCamelCase	) as mock_head:
													UpperCAmelCase_=  GPTaTokenizerFast.from_pretrained("""gpt2"""	)
													# This check we did call the fake head request
													mock_head.assert_called()




	def     _SCREAMING_SNAKE_CASE						(      self      :		Union[str, Any]	)     ->					Dict:
							# This test is for deprecated behavior and can be removed in v5
							try:
													UpperCAmelCase_=  tempfile.mktemp()
													with open(_lowerCamelCase		,    """wb"""	) as f:
																			http_get("""https://huggingface.co/albert-base-v1/resolve/main/spiece.model"""		,    _lowerCamelCase	)

													UpperCAmelCase_=  AlbertTokenizer.from_pretrained(_lowerCamelCase	)
							finally:
													os.remove(_lowerCamelCase	)

							# Supporting this legacy load introduced a weird bug where the tokenizer would load local files if they are in
							# the current folder and have the right name.
							if os.path.isfile("""tokenizer.json"""	):
													# We skip the test if the user has a `tokenizer.json` in this folder to avoid deleting it.
													return
							try:
													with open("""tokenizer.json"""		,    """wb"""	) as f:
																			http_get("""https://huggingface.co/hf-internal-testing/tiny-random-bert/blob/main/tokenizer.json"""		,    _lowerCamelCase	)
													UpperCAmelCase_=  AutoTokenizer.from_pretrained("""hf-internal-testing/tiny-random-gpt2"""	)
													# The tiny random BERT has a vocab size of 1024, tiny gpt2 as a vocab size of 1000
													self.assertEqual(tokenizer.vocab_size		,    1_000	)
													# Tokenizer should depend on the remote checkpoint, not the local tokenizer.json file.

							finally:
													os.remove("""tokenizer.json"""	)




	def     _SCREAMING_SNAKE_CASE						(      self      :		Union[str, Any]	)     ->					Union[str, Any]:
							# This test is for deprecated behavior and can be removed in v5
							UpperCAmelCase_=  AlbertTokenizer.from_pretrained("""https://huggingface.co/albert-base-v1/resolve/main/spiece.model"""	)






@is_staging_test
class 						lowercase		(					unittest.TestCase):






	"""simple docstring"""


	a__ :   Tuple						  =				['''[UNK]''', '''[CLS]''', '''[SEP]''', '''[PAD]''', '''[MASK]''', '''bla''', '''blou''']




	@classmethod
	def     _SCREAMING_SNAKE_CASE						(      cls      :		List[Any]	)     ->					Any:
							UpperCAmelCase_=  TOKEN
							HfFolder.save_token(_lowerCamelCase	)




	@classmethod
	def     _SCREAMING_SNAKE_CASE						(      cls      :		str	)     ->					Dict:
							try:
													delete_repo(token=cls._token		,    repo_id="""test-tokenizer"""	)
							except HTTPError:
													pass

							try:
													delete_repo(token=cls._token		,    repo_id="""valid_org/test-tokenizer-org"""	)
							except HTTPError:
													pass

							try:
													delete_repo(token=cls._token		,    repo_id="""test-dynamic-tokenizer"""	)
							except HTTPError:
													pass




	def     _SCREAMING_SNAKE_CASE						(      self      :		List[Any]	)     ->					Union[str, Any]:
							with tempfile.TemporaryDirectory() as tmp_dir:
													UpperCAmelCase_=  os.path.join(_lowerCamelCase		,    """vocab.txt"""	)
													with open(_lowerCamelCase		,    """w"""		,    encoding="""utf-8"""	) as vocab_writer:
																			vocab_writer.write("""""".join([x + """\n""" for x in self.vocab_tokens]	)	)
													UpperCAmelCase_=  BertTokenizer(_lowerCamelCase	)

							tokenizer.push_to_hub("""test-tokenizer"""		,    use_auth_token=self._token	)
							UpperCAmelCase_=  BertTokenizer.from_pretrained(F"""{USER}/test-tokenizer"""	)
							self.assertDictEqual(new_tokenizer.vocab		,    tokenizer.vocab	)

							# Reset repo
							delete_repo(token=self._token		,    repo_id="""test-tokenizer"""	)

							# Push to hub via save_pretrained
							with tempfile.TemporaryDirectory() as tmp_dir:
													tokenizer.save_pretrained(_lowerCamelCase		,    repo_id="""test-tokenizer"""		,    push_to_hub=_lowerCamelCase		,    use_auth_token=self._token	)

							UpperCAmelCase_=  BertTokenizer.from_pretrained(F"""{USER}/test-tokenizer"""	)
							self.assertDictEqual(new_tokenizer.vocab		,    tokenizer.vocab	)




	def     _SCREAMING_SNAKE_CASE						(      self      :		Any	)     ->					List[Any]:
							with tempfile.TemporaryDirectory() as tmp_dir:
													UpperCAmelCase_=  os.path.join(_lowerCamelCase		,    """vocab.txt"""	)
													with open(_lowerCamelCase		,    """w"""		,    encoding="""utf-8"""	) as vocab_writer:
																			vocab_writer.write("""""".join([x + """\n""" for x in self.vocab_tokens]	)	)
													UpperCAmelCase_=  BertTokenizer(_lowerCamelCase	)

							tokenizer.push_to_hub("""valid_org/test-tokenizer-org"""		,    use_auth_token=self._token	)
							UpperCAmelCase_=  BertTokenizer.from_pretrained("""valid_org/test-tokenizer-org"""	)
							self.assertDictEqual(new_tokenizer.vocab		,    tokenizer.vocab	)

							# Reset repo
							delete_repo(token=self._token		,    repo_id="""valid_org/test-tokenizer-org"""	)

							# Push to hub via save_pretrained
							with tempfile.TemporaryDirectory() as tmp_dir:
													tokenizer.save_pretrained(
													    _lowerCamelCase		,    repo_id="""valid_org/test-tokenizer-org"""		,    push_to_hub=_lowerCamelCase		,    use_auth_token=self._token	)

							UpperCAmelCase_=  BertTokenizer.from_pretrained("""valid_org/test-tokenizer-org"""	)
							self.assertDictEqual(new_tokenizer.vocab		,    tokenizer.vocab	)




	@require_tokenizers
	def     _SCREAMING_SNAKE_CASE						(      self      :		int	)     ->					Dict:
							CustomTokenizer.register_for_auto_class()
							with tempfile.TemporaryDirectory() as tmp_dir:
													UpperCAmelCase_=  os.path.join(_lowerCamelCase		,    """vocab.txt"""	)
													with open(_lowerCamelCase		,    """w"""		,    encoding="""utf-8"""	) as vocab_writer:
																			vocab_writer.write("""""".join([x + """\n""" for x in self.vocab_tokens]	)	)
													UpperCAmelCase_=  CustomTokenizer(_lowerCamelCase	)

							# No fast custom tokenizer
							tokenizer.push_to_hub("""test-dynamic-tokenizer"""		,    use_auth_token=self._token	)

							UpperCAmelCase_=  AutoTokenizer.from_pretrained(F"""{USER}/test-dynamic-tokenizer"""		,    trust_remote_code=_lowerCamelCase	)
							# Can't make an isinstance check because the new_model.config is from the CustomTokenizer class of a dynamic module
							self.assertEqual(tokenizer.__class__.__name__		,    """CustomTokenizer"""	)

							# Fast and slow custom tokenizer
							CustomTokenizerFast.register_for_auto_class()
							with tempfile.TemporaryDirectory() as tmp_dir:
													UpperCAmelCase_=  os.path.join(_lowerCamelCase		,    """vocab.txt"""	)
													with open(_lowerCamelCase		,    """w"""		,    encoding="""utf-8"""	) as vocab_writer:
																			vocab_writer.write("""""".join([x + """\n""" for x in self.vocab_tokens]	)	)

													UpperCAmelCase_=  BertTokenizerFast.from_pretrained(_lowerCamelCase	)
													bert_tokenizer.save_pretrained(_lowerCamelCase	)
													UpperCAmelCase_=  CustomTokenizerFast.from_pretrained(_lowerCamelCase	)

							tokenizer.push_to_hub("""test-dynamic-tokenizer"""		,    use_auth_token=self._token	)

							UpperCAmelCase_=  AutoTokenizer.from_pretrained(F"""{USER}/test-dynamic-tokenizer"""		,    trust_remote_code=_lowerCamelCase	)
							# Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
							self.assertEqual(tokenizer.__class__.__name__		,    """CustomTokenizerFast"""	)
							UpperCAmelCase_=  AutoTokenizer.from_pretrained(
							    F"""{USER}/test-dynamic-tokenizer"""		,    use_fast=_lowerCamelCase		,    trust_remote_code=_lowerCamelCase	)
							# Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
							self.assertEqual(tokenizer.__class__.__name__		,    """CustomTokenizer"""	)






class 						lowercase		(					unittest.TestCase):






	"""simple docstring"""


	def     _SCREAMING_SNAKE_CASE						(      self      :		List[Any]	)     ->					str:
							UpperCAmelCase_=  Trie()
							trie.add("""Hello 友達"""	)
							self.assertEqual(trie.data		,    {"""H""": {"""e""": {"""l""": {"""l""": {"""o""": {""" """: {"""友""": {"""達""": {"""""": 1}}}}}}}}}	)
							trie.add("""Hello"""	)
							trie.data
							self.assertEqual(trie.data		,    {"""H""": {"""e""": {"""l""": {"""l""": {"""o""": {"""""": 1, """ """: {"""友""": {"""達""": {"""""": 1}}}}}}}}}	)




	def     _SCREAMING_SNAKE_CASE						(      self      :		Optional[Any]	)     ->					Dict:
							UpperCAmelCase_=  Trie()
							self.assertEqual(trie.split("""[CLS] This is a extra_id_100"""	)		,    ["""[CLS] This is a extra_id_100"""]	)
							trie.add("""[CLS]"""	)
							trie.add("""extra_id_1"""	)
							trie.add("""extra_id_100"""	)
							self.assertEqual(trie.split("""[CLS] This is a extra_id_100"""	)		,    ["""[CLS]""", """ This is a """, """extra_id_100"""]	)




	def     _SCREAMING_SNAKE_CASE						(      self      :		Dict	)     ->					Optional[int]:
							UpperCAmelCase_=  Trie()
							trie.add("""A"""	)
							self.assertEqual(trie.split("""ABC"""	)		,    ["""A""", """BC"""]	)
							self.assertEqual(trie.split("""BCA"""	)		,    ["""BC""", """A"""]	)




	def     _SCREAMING_SNAKE_CASE						(      self      :		List[str]	)     ->					Tuple:
							UpperCAmelCase_=  Trie()
							trie.add("""TOKEN]"""	)
							trie.add("""[SPECIAL_TOKEN]"""	)
							self.assertEqual(trie.split("""This is something [SPECIAL_TOKEN]"""	)		,    ["""This is something """, """[SPECIAL_TOKEN]"""]	)




	def     _SCREAMING_SNAKE_CASE						(      self      :		Optional[int]	)     ->					Union[str, Any]:
							UpperCAmelCase_=  Trie()
							trie.add("""A"""	)
							trie.add("""P"""	)
							trie.add("""[SPECIAL_TOKEN]"""	)
							self.assertEqual(trie.split("""This is something [SPECIAL_TOKEN]"""	)		,    ["""This is something """, """[SPECIAL_TOKEN]"""]	)




	def     _SCREAMING_SNAKE_CASE						(      self      :		str	)     ->					Union[str, Any]:
							UpperCAmelCase_=  Trie()
							trie.add("""AB"""	)
							trie.add("""B"""	)
							trie.add("""C"""	)
							self.assertEqual(trie.split("""ABC"""	)		,    ["""AB""", """C"""]	)




	def     _SCREAMING_SNAKE_CASE						(      self      :		Union[str, Any]	)     ->					Optional[Any]:
							UpperCAmelCase_=  Trie()
							trie.add("""ABC"""	)
							trie.add("""B"""	)
							trie.add("""CD"""	)
							self.assertEqual(trie.split("""ABCD"""	)		,    ["""ABC""", """D"""]	)




	def     _SCREAMING_SNAKE_CASE						(      self      :		int	)     ->					Dict:
							# Even if the offsets are wrong, we necessarily output correct string
							# parts.
							UpperCAmelCase_=  Trie()
							UpperCAmelCase_=  trie.cut_text("""ABC"""		,    [0, 0, 2, 1, 2, 3]	)
							self.assertEqual(_lowerCamelCase		,    ["""AB""", """C"""]	)





"""simple docstring"""


def      lowerCamelCase_					(   __lowerCAmelCase							)				->   bool:




						'''simple docstring'''



						return number & 1 == 0


if __name__ == "__main__":
	import doctest

	doctest.testmod()


