


'''simple docstring'''



import os
import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from huggingface_hub.file_download import http_get
from requests.exceptions import HTTPError

from transformers import (
    AlbertTokenizer,
    AutoTokenizer,
    BertTokenizer,
    BertTokenizerFast,
    GPTaTokenizerFast,
    is_tokenizers_available,
)
from transformers.testing_utils import TOKEN, USER, is_staging_test, require_tokenizers
from transformers.tokenization_utils import Trie


sys.path.append(str(Path(__file__).parent.parent / 'utils'))

from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
		from test_module.custom_tokenization_fast import CustomTokenizerFast


class 		_snake_case					(     unittest.TestCase	):




					def 				snake_case__						(      self):
								# A mock response for an HTTP head request to emulate server down
								UpperCAmelCase__  :					Optional[int]        =     mock.Mock()
								UpperCAmelCase__  :					List[str]        =     500
								UpperCAmelCase__  :					int        =     {}
								UpperCAmelCase__  :					Union[str, Any]        =     HTTPError
								UpperCAmelCase__  :					int        =     {}

								# Download this model to make sure it's in the cache.
								UpperCAmelCase__  :					List[Any]        =     BertTokenizer.from_pretrained("""hf-internal-testing/tiny-random-bert""")

								# Under the mock environment we get a 500 error when trying to reach the tokenizer.
								with mock.patch("""requests.Session.request"""							,							return_value=_lowerCamelCase) as mock_head:
											UpperCAmelCase__  :					Optional[int]        =     BertTokenizer.from_pretrained("""hf-internal-testing/tiny-random-bert""")
											# This check we did call the fake head request
											mock_head.assert_called()




					@require_tokenizers
					def 				snake_case__						(      self):
								# A mock response for an HTTP head request to emulate server down
								UpperCAmelCase__  :					Optional[int]        =     mock.Mock()
								UpperCAmelCase__  :					Tuple        =     500
								UpperCAmelCase__  :					Tuple        =     {}
								UpperCAmelCase__  :					Any        =     HTTPError
								UpperCAmelCase__  :					Optional[int]        =     {}

								# Download this model to make sure it's in the cache.
								UpperCAmelCase__  :					Any        =     GPTaTokenizerFast.from_pretrained("""gpt2""")

								# Under the mock environment we get a 500 error when trying to reach the tokenizer.
								with mock.patch("""requests.Session.request"""							,							return_value=_lowerCamelCase) as mock_head:
											UpperCAmelCase__  :					str        =     GPTaTokenizerFast.from_pretrained("""gpt2""")
											# This check we did call the fake head request
											mock_head.assert_called()




					def 				snake_case__						(      self):
								# This test is for deprecated behavior and can be removed in v5
								try:
											UpperCAmelCase__  :					Optional[int]        =     tempfile.mktemp()
											with open(_lowerCamelCase							,							"""wb""") as f:
														http_get("""https://huggingface.co/albert-base-v1/resolve/main/spiece.model"""							,							_lowerCamelCase)

											UpperCAmelCase__  :					str        =     AlbertTokenizer.from_pretrained(_lowerCamelCase)
								finally:
											os.remove(_lowerCamelCase)

								# Supporting this legacy load introduced a weird bug where the tokenizer would load local files if they are in
								# the current folder and have the right name.
								if os.path.isfile("""tokenizer.json"""):
											# We skip the test if the user has a `tokenizer.json` in this folder to avoid deleting it.
											return
								try:
											with open("""tokenizer.json"""							,							"""wb""") as f:
														http_get("""https://huggingface.co/hf-internal-testing/tiny-random-bert/blob/main/tokenizer.json"""							,							_lowerCamelCase)
											UpperCAmelCase__  :					Optional[Any]        =     AutoTokenizer.from_pretrained("""hf-internal-testing/tiny-random-gpt2""")
											# The tiny random BERT has a vocab size of 1024, tiny gpt2 as a vocab size of 1000
											self.assertEqual(tokenizer.vocab_size							,							1000)
											# Tokenizer should depend on the remote checkpoint, not the local tokenizer.json file.

								finally:
											os.remove("""tokenizer.json""")




					def 				snake_case__						(      self):
								# This test is for deprecated behavior and can be removed in v5
								UpperCAmelCase__  :					Any        =     AlbertTokenizer.from_pretrained("""https://huggingface.co/albert-base-v1/resolve/main/spiece.model""")


@is_staging_test
class 		_snake_case					(     unittest.TestCase	):
					lowerCAmelCase      :Optional[Any]   =					['''[UNK]''', '''[CLS]''', '''[SEP]''', '''[PAD]''', '''[MASK]''', '''bla''', '''blou''']




					@classmethod
					def 				snake_case__						(      cls):
								UpperCAmelCase__  :					Dict        =     TOKEN
								HfFolder.save_token(_lowerCamelCase)




					@classmethod
					def 				snake_case__						(      cls):
								try:
											delete_repo(token=cls._token							,							repo_id="""test-tokenizer""")
								except HTTPError:
											pass

								try:
											delete_repo(token=cls._token							,							repo_id="""valid_org/test-tokenizer-org""")
								except HTTPError:
											pass

								try:
											delete_repo(token=cls._token							,							repo_id="""test-dynamic-tokenizer""")
								except HTTPError:
											pass




					def 				snake_case__						(      self):
								with tempfile.TemporaryDirectory() as tmp_dir:
											UpperCAmelCase__  :					Optional[Any]        =     os.path.join(_lowerCamelCase							,							"""vocab.txt""")
											with open(_lowerCamelCase							,							"""w"""							,							encoding="""utf-8""") as vocab_writer:
														vocab_writer.write("""""".join([x + """\n""" for x in self.vocab_tokens]))
											UpperCAmelCase__  :					Any        =     BertTokenizer(_lowerCamelCase)

								tokenizer.push_to_hub("""test-tokenizer"""							,							use_auth_token=self._token)
								UpperCAmelCase__  :					List[Any]        =     BertTokenizer.from_pretrained(f'''{USER}/test-tokenizer''')
								self.assertDictEqual(new_tokenizer.vocab							,							tokenizer.vocab)

								# Reset repo
								delete_repo(token=self._token							,							repo_id="""test-tokenizer""")

								# Push to hub via save_pretrained
								with tempfile.TemporaryDirectory() as tmp_dir:
											tokenizer.save_pretrained(_lowerCamelCase							,							repo_id="""test-tokenizer"""							,							push_to_hub=_lowerCamelCase							,							use_auth_token=self._token)

								UpperCAmelCase__  :					Any        =     BertTokenizer.from_pretrained(f'''{USER}/test-tokenizer''')
								self.assertDictEqual(new_tokenizer.vocab							,							tokenizer.vocab)




					def 				snake_case__						(      self):
								with tempfile.TemporaryDirectory() as tmp_dir:
											UpperCAmelCase__  :					List[str]        =     os.path.join(_lowerCamelCase							,							"""vocab.txt""")
											with open(_lowerCamelCase							,							"""w"""							,							encoding="""utf-8""") as vocab_writer:
														vocab_writer.write("""""".join([x + """\n""" for x in self.vocab_tokens]))
											UpperCAmelCase__  :					int        =     BertTokenizer(_lowerCamelCase)

								tokenizer.push_to_hub("""valid_org/test-tokenizer-org"""							,							use_auth_token=self._token)
								UpperCAmelCase__  :					Dict        =     BertTokenizer.from_pretrained("""valid_org/test-tokenizer-org""")
								self.assertDictEqual(new_tokenizer.vocab							,							tokenizer.vocab)

								# Reset repo
								delete_repo(token=self._token							,							repo_id="""valid_org/test-tokenizer-org""")

								# Push to hub via save_pretrained
								with tempfile.TemporaryDirectory() as tmp_dir:
											tokenizer.save_pretrained(
											    _lowerCamelCase							,							repo_id="""valid_org/test-tokenizer-org"""							,							push_to_hub=_lowerCamelCase							,							use_auth_token=self._token)

								UpperCAmelCase__  :					List[Any]        =     BertTokenizer.from_pretrained("""valid_org/test-tokenizer-org""")
								self.assertDictEqual(new_tokenizer.vocab							,							tokenizer.vocab)




					@require_tokenizers
					def 				snake_case__						(      self):
								CustomTokenizer.register_for_auto_class()
								with tempfile.TemporaryDirectory() as tmp_dir:
											UpperCAmelCase__  :					Optional[int]        =     os.path.join(_lowerCamelCase							,							"""vocab.txt""")
											with open(_lowerCamelCase							,							"""w"""							,							encoding="""utf-8""") as vocab_writer:
														vocab_writer.write("""""".join([x + """\n""" for x in self.vocab_tokens]))
											UpperCAmelCase__  :					List[Any]        =     CustomTokenizer(_lowerCamelCase)

								# No fast custom tokenizer
								tokenizer.push_to_hub("""test-dynamic-tokenizer"""							,							use_auth_token=self._token)

								UpperCAmelCase__  :					int        =     AutoTokenizer.from_pretrained(f'''{USER}/test-dynamic-tokenizer'''							,							trust_remote_code=_lowerCamelCase)
								# Can't make an isinstance check because the new_model.config is from the CustomTokenizer class of a dynamic module
								self.assertEqual(tokenizer.__class__.__name__							,							"""CustomTokenizer""")

								# Fast and slow custom tokenizer
								CustomTokenizerFast.register_for_auto_class()
								with tempfile.TemporaryDirectory() as tmp_dir:
											UpperCAmelCase__  :					str        =     os.path.join(_lowerCamelCase							,							"""vocab.txt""")
											with open(_lowerCamelCase							,							"""w"""							,							encoding="""utf-8""") as vocab_writer:
														vocab_writer.write("""""".join([x + """\n""" for x in self.vocab_tokens]))

											UpperCAmelCase__  :					Union[str, Any]        =     BertTokenizerFast.from_pretrained(_lowerCamelCase)
											bert_tokenizer.save_pretrained(_lowerCamelCase)
											UpperCAmelCase__  :					Dict        =     CustomTokenizerFast.from_pretrained(_lowerCamelCase)

								tokenizer.push_to_hub("""test-dynamic-tokenizer"""							,							use_auth_token=self._token)

								UpperCAmelCase__  :					List[Any]        =     AutoTokenizer.from_pretrained(f'''{USER}/test-dynamic-tokenizer'''							,							trust_remote_code=_lowerCamelCase)
								# Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
								self.assertEqual(tokenizer.__class__.__name__							,							"""CustomTokenizerFast""")
								UpperCAmelCase__  :					Any        =     AutoTokenizer.from_pretrained(
								    f'''{USER}/test-dynamic-tokenizer'''							,							use_fast=_lowerCamelCase							,							trust_remote_code=_lowerCamelCase)
								# Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
								self.assertEqual(tokenizer.__class__.__name__							,							"""CustomTokenizer""")


class 		_snake_case					(     unittest.TestCase	):




					def 				snake_case__						(      self):
								UpperCAmelCase__  :					List[Any]        =     Trie()
								trie.add("""Hello 友達""")
								self.assertEqual(trie.data							,							{"""H""": {"""e""": {"""l""": {"""l""": {"""o""": {""" """: {"""友""": {"""達""": {"""""": 1}}}}}}}}})
								trie.add("""Hello""")
								trie.data
								self.assertEqual(trie.data							,							{"""H""": {"""e""": {"""l""": {"""l""": {"""o""": {"""""": 1, """ """: {"""友""": {"""達""": {"""""": 1}}}}}}}}})




					def 				snake_case__						(      self):
								UpperCAmelCase__  :					Optional[Any]        =     Trie()
								self.assertEqual(trie.split("""[CLS] This is a extra_id_100""")							,							["""[CLS] This is a extra_id_100"""])
								trie.add("""[CLS]""")
								trie.add("""extra_id_1""")
								trie.add("""extra_id_100""")
								self.assertEqual(trie.split("""[CLS] This is a extra_id_100""")							,							["""[CLS]""", """ This is a """, """extra_id_100"""])




					def 				snake_case__						(      self):
								UpperCAmelCase__  :					str        =     Trie()
								trie.add("""A""")
								self.assertEqual(trie.split("""ABC""")							,							["""A""", """BC"""])
								self.assertEqual(trie.split("""BCA""")							,							["""BC""", """A"""])




					def 				snake_case__						(      self):
								UpperCAmelCase__  :					str        =     Trie()
								trie.add("""TOKEN]""")
								trie.add("""[SPECIAL_TOKEN]""")
								self.assertEqual(trie.split("""This is something [SPECIAL_TOKEN]""")							,							["""This is something """, """[SPECIAL_TOKEN]"""])




					def 				snake_case__						(      self):
								UpperCAmelCase__  :					List[str]        =     Trie()
								trie.add("""A""")
								trie.add("""P""")
								trie.add("""[SPECIAL_TOKEN]""")
								self.assertEqual(trie.split("""This is something [SPECIAL_TOKEN]""")							,							["""This is something """, """[SPECIAL_TOKEN]"""])




					def 				snake_case__						(      self):
								UpperCAmelCase__  :					int        =     Trie()
								trie.add("""AB""")
								trie.add("""B""")
								trie.add("""C""")
								self.assertEqual(trie.split("""ABC""")							,							["""AB""", """C"""])




					def 				snake_case__						(      self):
								UpperCAmelCase__  :					Union[str, Any]        =     Trie()
								trie.add("""ABC""")
								trie.add("""B""")
								trie.add("""CD""")
								self.assertEqual(trie.split("""ABCD""")							,							["""ABC""", """D"""])




					def 				snake_case__						(      self):
								# Even if the offsets are wrong, we necessarily output correct string
								# parts.
								UpperCAmelCase__  :					Union[str, Any]        =     Trie()
								UpperCAmelCase__  :					Union[str, Any]        =     trie.cut_text("""ABC"""							,							[0, 0, 2, 1, 2, 3])
								self.assertEqual(_lowerCamelCase							,							["""AB""", """C"""])



'''simple docstring'''



from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tokenizers_available, is_torch_available


__A	={
    'configuration_nezha': ['NEZHA_PRETRAINED_CONFIG_ARCHIVE_MAP', 'NezhaConfig'],
}

try:
		if not is_torch_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		__A	=[
		    'NEZHA_PRETRAINED_MODEL_ARCHIVE_LIST',
		    'NezhaForNextSentencePrediction',
		    'NezhaForMaskedLM',
		    'NezhaForPreTraining',
		    'NezhaForMultipleChoice',
		    'NezhaForQuestionAnswering',
		    'NezhaForSequenceClassification',
		    'NezhaForTokenClassification',
		    'NezhaModel',
		    'NezhaPreTrainedModel',
		]


if TYPE_CHECKING:
		from .configuration_nezha import NEZHA_PRETRAINED_CONFIG_ARCHIVE_MAP, NezhaConfig

		try:
				if not is_torch_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				from .modeling_nezha import (
				    NEZHA_PRETRAINED_MODEL_ARCHIVE_LIST,
				    NezhaForMaskedLM,
				    NezhaForMultipleChoice,
				    NezhaForNextSentencePrediction,
				    NezhaForPreTraining,
				    NezhaForQuestionAnswering,
				    NezhaForSequenceClassification,
				    NezhaForTokenClassification,
				    NezhaModel,
				    NezhaPreTrainedModel,
				)


else:
		import sys

		__A	=_LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)