



import os
import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from huggingface_hub.file_download import http_get
from requests.exceptions import HTTPError

from transformers import (
    AlbertTokenizer,
    AutoTokenizer,
    BertTokenizer,
    BertTokenizerFast,
    GPTaTokenizerFast,
    is_tokenizers_available,
)
from transformers.testing_utils import TOKEN, USER, is_staging_test, require_tokenizers
from transformers.tokenization_utils import Trie


sys.path.append(str(Path(__file__).parent.parent / '''utils'''))

from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
	from test_module.custom_tokenization_fast import CustomTokenizerFast






class 		lowercase						(			unittest.TestCase):






	"""simple docstring"""


	def   _SCREAMING_SNAKE_CASE		(     self     :						Optional[Any]					)      ->			str:
							# A mock response for an HTTP head request to emulate server down
							UpperCAmelCase_=       mock.Mock()
							UpperCAmelCase_=       500
							UpperCAmelCase_=       {}
							UpperCAmelCase_=       HTTPError
							UpperCAmelCase_=       {}

							# Download this model to make sure it's in the cache.
							UpperCAmelCase_=       BertTokenizer.from_pretrained("""hf-internal-testing/tiny-random-bert"""					)

							# Under the mock environment we get a 500 error when trying to reach the tokenizer.
							with mock.patch("""requests.Session.request"""						,  return_value=__UpperCAmelCase					) as mock_head:
													UpperCAmelCase_=       BertTokenizer.from_pretrained("""hf-internal-testing/tiny-random-bert"""					)
													# This check we did call the fake head request
													mock_head.assert_called()




	@require_tokenizers
	def   _SCREAMING_SNAKE_CASE		(     self     :						Dict					)      ->			Optional[Any]:
							# A mock response for an HTTP head request to emulate server down
							UpperCAmelCase_=       mock.Mock()
							UpperCAmelCase_=       500
							UpperCAmelCase_=       {}
							UpperCAmelCase_=       HTTPError
							UpperCAmelCase_=       {}

							# Download this model to make sure it's in the cache.
							UpperCAmelCase_=       GPTaTokenizerFast.from_pretrained("""gpt2"""					)

							# Under the mock environment we get a 500 error when trying to reach the tokenizer.
							with mock.patch("""requests.Session.request"""						,  return_value=__UpperCAmelCase					) as mock_head:
													UpperCAmelCase_=       GPTaTokenizerFast.from_pretrained("""gpt2"""					)
													# This check we did call the fake head request
													mock_head.assert_called()




	def   _SCREAMING_SNAKE_CASE		(     self     :						int					)      ->			List[Any]:
							# This test is for deprecated behavior and can be removed in v5
							try:
													UpperCAmelCase_=       tempfile.mktemp()
													with open(__UpperCAmelCase						,  """wb"""					) as f:
																			http_get("""https://huggingface.co/albert-base-v1/resolve/main/spiece.model"""						,  __UpperCAmelCase					)

													UpperCAmelCase_=       AlbertTokenizer.from_pretrained(__UpperCAmelCase					)
							finally:
													os.remove(__UpperCAmelCase					)

							# Supporting this legacy load introduced a weird bug where the tokenizer would load local files if they are in
							# the current folder and have the right name.
							if os.path.isfile("""tokenizer.json"""					):
													# We skip the test if the user has a `tokenizer.json` in this folder to avoid deleting it.
													return
							try:
													with open("""tokenizer.json"""						,  """wb"""					) as f:
																			http_get("""https://huggingface.co/hf-internal-testing/tiny-random-bert/blob/main/tokenizer.json"""						,  __UpperCAmelCase					)
													UpperCAmelCase_=       AutoTokenizer.from_pretrained("""hf-internal-testing/tiny-random-gpt2"""					)
													# The tiny random BERT has a vocab size of 1024, tiny gpt2 as a vocab size of 1000
													self.assertEqual(tokenizer.vocab_size						,  1_000					)
													# Tokenizer should depend on the remote checkpoint, not the local tokenizer.json file.

							finally:
													os.remove("""tokenizer.json"""					)




	def   _SCREAMING_SNAKE_CASE		(     self     :						str					)      ->			Union[str, Any]:
							# This test is for deprecated behavior and can be removed in v5
							UpperCAmelCase_=       AlbertTokenizer.from_pretrained("""https://huggingface.co/albert-base-v1/resolve/main/spiece.model"""					)






@is_staging_test
class 		lowercase						(			unittest.TestCase):






	"""simple docstring"""


	a__				: Any		       =							["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]", "bla", "blou"]




	@classmethod
	def   _SCREAMING_SNAKE_CASE		(     cls     :						Optional[int]					)      ->			Optional[Any]:
							UpperCAmelCase_=       TOKEN
							HfFolder.save_token(__UpperCAmelCase					)




	@classmethod
	def   _SCREAMING_SNAKE_CASE		(     cls     :						str					)      ->			Optional[int]:
							try:
													delete_repo(token=cls._token						,  repo_id="""test-tokenizer"""					)
							except HTTPError:
													pass

							try:
													delete_repo(token=cls._token						,  repo_id="""valid_org/test-tokenizer-org"""					)
							except HTTPError:
													pass

							try:
													delete_repo(token=cls._token						,  repo_id="""test-dynamic-tokenizer"""					)
							except HTTPError:
													pass




	def   _SCREAMING_SNAKE_CASE		(     self     :						Optional[int]					)      ->			Optional[Any]:
							with tempfile.TemporaryDirectory() as tmp_dir:
													UpperCAmelCase_=       os.path.join(__UpperCAmelCase						,  """vocab.txt"""					)
													with open(__UpperCAmelCase						,  """w"""						,  encoding="""utf-8"""					) as vocab_writer:
																			vocab_writer.write("""""".join([x + """\n""" for x in self.vocab_tokens]					)					)
													UpperCAmelCase_=       BertTokenizer(__UpperCAmelCase					)

							tokenizer.push_to_hub("""test-tokenizer"""						,  use_auth_token=self._token					)
							UpperCAmelCase_=       BertTokenizer.from_pretrained(F"""{USER}/test-tokenizer"""					)
							self.assertDictEqual(new_tokenizer.vocab						,  tokenizer.vocab					)

							# Reset repo
							delete_repo(token=self._token						,  repo_id="""test-tokenizer"""					)

							# Push to hub via save_pretrained
							with tempfile.TemporaryDirectory() as tmp_dir:
													tokenizer.save_pretrained(__UpperCAmelCase						,  repo_id="""test-tokenizer"""						,  push_to_hub=__UpperCAmelCase						,  use_auth_token=self._token					)

							UpperCAmelCase_=       BertTokenizer.from_pretrained(F"""{USER}/test-tokenizer"""					)
							self.assertDictEqual(new_tokenizer.vocab						,  tokenizer.vocab					)




	def   _SCREAMING_SNAKE_CASE		(     self     :						List[Any]					)      ->			Tuple:
							with tempfile.TemporaryDirectory() as tmp_dir:
													UpperCAmelCase_=       os.path.join(__UpperCAmelCase						,  """vocab.txt"""					)
													with open(__UpperCAmelCase						,  """w"""						,  encoding="""utf-8"""					) as vocab_writer:
																			vocab_writer.write("""""".join([x + """\n""" for x in self.vocab_tokens]					)					)
													UpperCAmelCase_=       BertTokenizer(__UpperCAmelCase					)

							tokenizer.push_to_hub("""valid_org/test-tokenizer-org"""						,  use_auth_token=self._token					)
							UpperCAmelCase_=       BertTokenizer.from_pretrained("""valid_org/test-tokenizer-org"""					)
							self.assertDictEqual(new_tokenizer.vocab						,  tokenizer.vocab					)

							# Reset repo
							delete_repo(token=self._token						,  repo_id="""valid_org/test-tokenizer-org"""					)

							# Push to hub via save_pretrained
							with tempfile.TemporaryDirectory() as tmp_dir:
													tokenizer.save_pretrained(
													    __UpperCAmelCase						,  repo_id="""valid_org/test-tokenizer-org"""						,  push_to_hub=__UpperCAmelCase						,  use_auth_token=self._token					)

							UpperCAmelCase_=       BertTokenizer.from_pretrained("""valid_org/test-tokenizer-org"""					)
							self.assertDictEqual(new_tokenizer.vocab						,  tokenizer.vocab					)




	@require_tokenizers
	def   _SCREAMING_SNAKE_CASE		(     self     :						List[str]					)      ->			Optional[int]:
							CustomTokenizer.register_for_auto_class()
							with tempfile.TemporaryDirectory() as tmp_dir:
													UpperCAmelCase_=       os.path.join(__UpperCAmelCase						,  """vocab.txt"""					)
													with open(__UpperCAmelCase						,  """w"""						,  encoding="""utf-8"""					) as vocab_writer:
																			vocab_writer.write("""""".join([x + """\n""" for x in self.vocab_tokens]					)					)
													UpperCAmelCase_=       CustomTokenizer(__UpperCAmelCase					)

							# No fast custom tokenizer
							tokenizer.push_to_hub("""test-dynamic-tokenizer"""						,  use_auth_token=self._token					)

							UpperCAmelCase_=       AutoTokenizer.from_pretrained(F"""{USER}/test-dynamic-tokenizer"""						,  trust_remote_code=__UpperCAmelCase					)
							# Can't make an isinstance check because the new_model.config is from the CustomTokenizer class of a dynamic module
							self.assertEqual(tokenizer.__class__.__name__						,  """CustomTokenizer"""					)

							# Fast and slow custom tokenizer
							CustomTokenizerFast.register_for_auto_class()
							with tempfile.TemporaryDirectory() as tmp_dir:
													UpperCAmelCase_=       os.path.join(__UpperCAmelCase						,  """vocab.txt"""					)
													with open(__UpperCAmelCase						,  """w"""						,  encoding="""utf-8"""					) as vocab_writer:
																			vocab_writer.write("""""".join([x + """\n""" for x in self.vocab_tokens]					)					)

													UpperCAmelCase_=       BertTokenizerFast.from_pretrained(__UpperCAmelCase					)
													bert_tokenizer.save_pretrained(__UpperCAmelCase					)
													UpperCAmelCase_=       CustomTokenizerFast.from_pretrained(__UpperCAmelCase					)

							tokenizer.push_to_hub("""test-dynamic-tokenizer"""						,  use_auth_token=self._token					)

							UpperCAmelCase_=       AutoTokenizer.from_pretrained(F"""{USER}/test-dynamic-tokenizer"""						,  trust_remote_code=__UpperCAmelCase					)
							# Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
							self.assertEqual(tokenizer.__class__.__name__						,  """CustomTokenizerFast"""					)
							UpperCAmelCase_=       AutoTokenizer.from_pretrained(
							    F"""{USER}/test-dynamic-tokenizer"""						,  use_fast=__UpperCAmelCase						,  trust_remote_code=__UpperCAmelCase					)
							# Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
							self.assertEqual(tokenizer.__class__.__name__						,  """CustomTokenizer"""					)






class 		lowercase						(			unittest.TestCase):






	"""simple docstring"""


	def   _SCREAMING_SNAKE_CASE		(     self     :						List[Any]					)      ->			Tuple:
							UpperCAmelCase_=       Trie()
							trie.add("""Hello 友達"""					)
							self.assertEqual(trie.data						,  {"""H""": {"""e""": {"""l""": {"""l""": {"""o""": {""" """: {"""友""": {"""達""": {"""""": 1}}}}}}}}}					)
							trie.add("""Hello"""					)
							trie.data
							self.assertEqual(trie.data						,  {"""H""": {"""e""": {"""l""": {"""l""": {"""o""": {"""""": 1, """ """: {"""友""": {"""達""": {"""""": 1}}}}}}}}}					)




	def   _SCREAMING_SNAKE_CASE		(     self     :						Optional[Any]					)      ->			List[str]:
							UpperCAmelCase_=       Trie()
							self.assertEqual(trie.split("""[CLS] This is a extra_id_100"""					)						,  ["""[CLS] This is a extra_id_100"""]					)
							trie.add("""[CLS]"""					)
							trie.add("""extra_id_1"""					)
							trie.add("""extra_id_100"""					)
							self.assertEqual(trie.split("""[CLS] This is a extra_id_100"""					)						,  ["""[CLS]""", """ This is a """, """extra_id_100"""]					)




	def   _SCREAMING_SNAKE_CASE		(     self     :						Any					)      ->			Dict:
							UpperCAmelCase_=       Trie()
							trie.add("""A"""					)
							self.assertEqual(trie.split("""ABC"""					)						,  ["""A""", """BC"""]					)
							self.assertEqual(trie.split("""BCA"""					)						,  ["""BC""", """A"""]					)




	def   _SCREAMING_SNAKE_CASE		(     self     :						Optional[Any]					)      ->			List[Any]:
							UpperCAmelCase_=       Trie()
							trie.add("""TOKEN]"""					)
							trie.add("""[SPECIAL_TOKEN]"""					)
							self.assertEqual(trie.split("""This is something [SPECIAL_TOKEN]"""					)						,  ["""This is something """, """[SPECIAL_TOKEN]"""]					)




	def   _SCREAMING_SNAKE_CASE		(     self     :						List[Any]					)      ->			int:
							UpperCAmelCase_=       Trie()
							trie.add("""A"""					)
							trie.add("""P"""					)
							trie.add("""[SPECIAL_TOKEN]"""					)
							self.assertEqual(trie.split("""This is something [SPECIAL_TOKEN]"""					)						,  ["""This is something """, """[SPECIAL_TOKEN]"""]					)




	def   _SCREAMING_SNAKE_CASE		(     self     :						List[str]					)      ->			List[Any]:
							UpperCAmelCase_=       Trie()
							trie.add("""AB"""					)
							trie.add("""B"""					)
							trie.add("""C"""					)
							self.assertEqual(trie.split("""ABC"""					)						,  ["""AB""", """C"""]					)




	def   _SCREAMING_SNAKE_CASE		(     self     :						Optional[int]					)      ->			Dict:
							UpperCAmelCase_=       Trie()
							trie.add("""ABC"""					)
							trie.add("""B"""					)
							trie.add("""CD"""					)
							self.assertEqual(trie.split("""ABCD"""					)						,  ["""ABC""", """D"""]					)




	def   _SCREAMING_SNAKE_CASE		(     self     :						Dict					)      ->			Dict:
							# Even if the offsets are wrong, we necessarily output correct string
							# parts.
							UpperCAmelCase_=       Trie()
							UpperCAmelCase_=       trie.cut_text("""ABC"""						,  [0, 0, 2, 1, 2, 3]					)
							self.assertEqual(__UpperCAmelCase						,  ["""AB""", """C"""]					)









import warnings
from functools import wraps
from typing import Callable



def 	__a					(   lowerCAmelCase_       : Callable    )				->			Callable:






						'''simple docstring'''



						@wraps(lowerCAmelCase_    )
						def _inner_fn(*lowerCAmelCase_       : List[Any]      ,**lowerCAmelCase_       : Tuple    ):
												warnings.warn(
												    (F"""'{fn.__name__}' is experimental and might be subject to breaking changes in the future.""")      ,lowerCAmelCase_      ,)
												return fn(*lowerCAmelCase_      ,**lowerCAmelCase_    )

						return _inner_fn




