





import os
import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from huggingface_hub.file_download import http_get
from requests.exceptions import HTTPError

from transformers import (
    AlbertTokenizer,
    AutoTokenizer,
    BertTokenizer,
    BertTokenizerFast,
    GPTaTokenizerFast,
    is_tokenizers_available,
)
from transformers.testing_utils import TOKEN, USER, is_staging_test, require_tokenizers
from transformers.tokenization_utils import Trie


sys.path.append(str(Path(__file__).parent.parent / '''utils'''))

from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
	from test_module.custom_tokenization_fast import CustomTokenizerFast



class  __lowerCAmelCase	(      unittest.TestCase					):






			def       lowerCamelCase  (self   )					->       str:


							'''simple docstring'''



							snake_case_						:   Dict           =  mock.Mock()
							snake_case_						:   Tuple           =  500
							snake_case_						:   Any           =  {}
							snake_case_						:   Optional[int]           =  HTTPError
							snake_case_						:   Any           =  {}

							# Download this model to make sure it's in the cache.
							snake_case_						:   Tuple           =  BertTokenizer.from_pretrained('''hf-internal-testing/tiny-random-bert'''   )

							# Under the mock environment we get a 500 error when trying to reach the tokenizer.
							with mock.patch('''requests.Session.request''' ,							return_value=_SCREAMING_SNAKE_CASE   ) as mock_head:
											snake_case_						:   Union[str, Any]           =  BertTokenizer.from_pretrained('''hf-internal-testing/tiny-random-bert'''   )
											# This check we did call the fake head request
											mock_head.assert_called()






			@require_tokenizers
			def       lowerCamelCase  (self   )					->       List[Any]:


							'''simple docstring'''



							snake_case_						:   Optional[Any]           =  mock.Mock()
							snake_case_						:   Optional[Any]           =  500
							snake_case_						:   List[str]           =  {}
							snake_case_						:   int           =  HTTPError
							snake_case_						:   Optional[int]           =  {}

							# Download this model to make sure it's in the cache.
							snake_case_						:   Dict           =  GPTaTokenizerFast.from_pretrained('''gpt2'''   )

							# Under the mock environment we get a 500 error when trying to reach the tokenizer.
							with mock.patch('''requests.Session.request''' ,							return_value=_SCREAMING_SNAKE_CASE   ) as mock_head:
											snake_case_						:   List[str]           =  GPTaTokenizerFast.from_pretrained('''gpt2'''   )
											# This check we did call the fake head request
											mock_head.assert_called()






			def       lowerCamelCase  (self   )					->       List[str]:


							'''simple docstring'''



							try:
											snake_case_						:   Union[str, Any]           =  tempfile.mktemp()
											with open(_SCREAMING_SNAKE_CASE ,							'''wb'''   ) as f:
															http_get('''https://huggingface.co/albert-base-v1/resolve/main/spiece.model''' ,							_SCREAMING_SNAKE_CASE   )

											snake_case_						:   Any           =  AlbertTokenizer.from_pretrained(_SCREAMING_SNAKE_CASE   )
							finally:
											os.remove(_SCREAMING_SNAKE_CASE   )

							# Supporting this legacy load introduced a weird bug where the tokenizer would load local files if they are in
							# the current folder and have the right name.
							if os.path.isfile('''tokenizer.json'''   ):
											# We skip the test if the user has a `tokenizer.json` in this folder to avoid deleting it.
											return
							try:
											with open('''tokenizer.json''' ,							'''wb'''   ) as f:
															http_get('''https://huggingface.co/hf-internal-testing/tiny-random-bert/blob/main/tokenizer.json''' ,							_SCREAMING_SNAKE_CASE   )
											snake_case_						:   Optional[Any]           =  AutoTokenizer.from_pretrained('''hf-internal-testing/tiny-random-gpt2'''   )
											# The tiny random BERT has a vocab size of 1024, tiny gpt2 as a vocab size of 1000
											self.assertEqual(tokenizer.vocab_size ,							1000   )
											# Tokenizer should depend on the remote checkpoint, not the local tokenizer.json file.

							finally:
											os.remove('''tokenizer.json'''   )






			def       lowerCamelCase  (self   )					->       Tuple:


							'''simple docstring'''



							snake_case_						:   List[str]           =  AlbertTokenizer.from_pretrained('''https://huggingface.co/albert-base-v1/resolve/main/spiece.model'''   )




@is_staging_test
class  __lowerCAmelCase	(      unittest.TestCase					):
			lowerCamelCase_					:					Optional[Any]	      =				["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]", "bla", "blou"]






			@classmethod
			def       lowerCamelCase  (cls   )					->       Dict:


							'''simple docstring'''



							snake_case_						:   Optional[int]           =  TOKEN
							HfFolder.save_token(_SCREAMING_SNAKE_CASE   )






			@classmethod
			def       lowerCamelCase  (cls   )					->       Tuple:


							'''simple docstring'''



							try:
											delete_repo(token=cls._token ,							repo_id='''test-tokenizer'''   )
							except HTTPError:
											pass

							try:
											delete_repo(token=cls._token ,							repo_id='''valid_org/test-tokenizer-org'''   )
							except HTTPError:
											pass

							try:
											delete_repo(token=cls._token ,							repo_id='''test-dynamic-tokenizer'''   )
							except HTTPError:
											pass






			def       lowerCamelCase  (self   )					->       List[str]:


							'''simple docstring'''



							with tempfile.TemporaryDirectory() as tmp_dir:
											snake_case_						:   int           =  os.path.join(_SCREAMING_SNAKE_CASE ,							'''vocab.txt'''   )
											with open(_SCREAMING_SNAKE_CASE ,							'''w''' ,							encoding='''utf-8'''   ) as vocab_writer:
															vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]   )   )
											snake_case_						:   Dict           =  BertTokenizer(_SCREAMING_SNAKE_CASE   )

							tokenizer.push_to_hub('''test-tokenizer''' ,							use_auth_token=self._token   )
							snake_case_						:   Optional[Any]           =  BertTokenizer.from_pretrained(F'''{USER}/test-tokenizer'''   )
							self.assertDictEqual(new_tokenizer.vocab ,							tokenizer.vocab   )

							# Reset repo
							delete_repo(token=self._token ,							repo_id='''test-tokenizer'''   )

							# Push to hub via save_pretrained
							with tempfile.TemporaryDirectory() as tmp_dir:
											tokenizer.save_pretrained(_SCREAMING_SNAKE_CASE ,							repo_id='''test-tokenizer''' ,							push_to_hub=_SCREAMING_SNAKE_CASE ,							use_auth_token=self._token   )

							snake_case_						:   int           =  BertTokenizer.from_pretrained(F'''{USER}/test-tokenizer'''   )
							self.assertDictEqual(new_tokenizer.vocab ,							tokenizer.vocab   )






			def       lowerCamelCase  (self   )					->       List[str]:


							'''simple docstring'''



							with tempfile.TemporaryDirectory() as tmp_dir:
											snake_case_						:   str           =  os.path.join(_SCREAMING_SNAKE_CASE ,							'''vocab.txt'''   )
											with open(_SCREAMING_SNAKE_CASE ,							'''w''' ,							encoding='''utf-8'''   ) as vocab_writer:
															vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]   )   )
											snake_case_						:   int           =  BertTokenizer(_SCREAMING_SNAKE_CASE   )

							tokenizer.push_to_hub('''valid_org/test-tokenizer-org''' ,							use_auth_token=self._token   )
							snake_case_						:   Tuple           =  BertTokenizer.from_pretrained('''valid_org/test-tokenizer-org'''   )
							self.assertDictEqual(new_tokenizer.vocab ,							tokenizer.vocab   )

							# Reset repo
							delete_repo(token=self._token ,							repo_id='''valid_org/test-tokenizer-org'''   )

							# Push to hub via save_pretrained
							with tempfile.TemporaryDirectory() as tmp_dir:
											tokenizer.save_pretrained(
											    _SCREAMING_SNAKE_CASE ,							repo_id='''valid_org/test-tokenizer-org''' ,							push_to_hub=_SCREAMING_SNAKE_CASE ,							use_auth_token=self._token   )

							snake_case_						:   List[str]           =  BertTokenizer.from_pretrained('''valid_org/test-tokenizer-org'''   )
							self.assertDictEqual(new_tokenizer.vocab ,							tokenizer.vocab   )






			@require_tokenizers
			def       lowerCamelCase  (self   )					->       List[Any]:


							'''simple docstring'''



							CustomTokenizer.register_for_auto_class()
							with tempfile.TemporaryDirectory() as tmp_dir:
											snake_case_						:   Any           =  os.path.join(_SCREAMING_SNAKE_CASE ,							'''vocab.txt'''   )
											with open(_SCREAMING_SNAKE_CASE ,							'''w''' ,							encoding='''utf-8'''   ) as vocab_writer:
															vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]   )   )
											snake_case_						:   Any           =  CustomTokenizer(_SCREAMING_SNAKE_CASE   )

							# No fast custom tokenizer
							tokenizer.push_to_hub('''test-dynamic-tokenizer''' ,							use_auth_token=self._token   )

							snake_case_						:   Any           =  AutoTokenizer.from_pretrained(F'''{USER}/test-dynamic-tokenizer''' ,							trust_remote_code=_SCREAMING_SNAKE_CASE   )
							# Can't make an isinstance check because the new_model.config is from the CustomTokenizer class of a dynamic module
							self.assertEqual(tokenizer.__class__.__name__ ,							'''CustomTokenizer'''   )

							# Fast and slow custom tokenizer
							CustomTokenizerFast.register_for_auto_class()
							with tempfile.TemporaryDirectory() as tmp_dir:
											snake_case_						:   Union[str, Any]           =  os.path.join(_SCREAMING_SNAKE_CASE ,							'''vocab.txt'''   )
											with open(_SCREAMING_SNAKE_CASE ,							'''w''' ,							encoding='''utf-8'''   ) as vocab_writer:
															vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]   )   )

											snake_case_						:   int           =  BertTokenizerFast.from_pretrained(_SCREAMING_SNAKE_CASE   )
											bert_tokenizer.save_pretrained(_SCREAMING_SNAKE_CASE   )
											snake_case_						:   Dict           =  CustomTokenizerFast.from_pretrained(_SCREAMING_SNAKE_CASE   )

							tokenizer.push_to_hub('''test-dynamic-tokenizer''' ,							use_auth_token=self._token   )

							snake_case_						:   Optional[Any]           =  AutoTokenizer.from_pretrained(F'''{USER}/test-dynamic-tokenizer''' ,							trust_remote_code=_SCREAMING_SNAKE_CASE   )
							# Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
							self.assertEqual(tokenizer.__class__.__name__ ,							'''CustomTokenizerFast'''   )
							snake_case_						:   int           =  AutoTokenizer.from_pretrained(
							    F'''{USER}/test-dynamic-tokenizer''' ,							use_fast=_SCREAMING_SNAKE_CASE ,							trust_remote_code=_SCREAMING_SNAKE_CASE   )
							# Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
							self.assertEqual(tokenizer.__class__.__name__ ,							'''CustomTokenizer'''   )




class  __lowerCAmelCase	(      unittest.TestCase					):






			def       lowerCamelCase  (self   )					->       Optional[int]:


							'''simple docstring'''



							snake_case_						:   Optional[Any]           =  Trie()
							trie.add('''Hello 友達'''   )
							self.assertEqual(trie.data ,							{'''H''': {'''e''': {'''l''': {'''l''': {'''o''': {''' ''': {'''友''': {'''達''': {'''''': 1}}}}}}}}}   )
							trie.add('''Hello'''   )
							trie.data
							self.assertEqual(trie.data ,							{'''H''': {'''e''': {'''l''': {'''l''': {'''o''': {'''''': 1, ''' ''': {'''友''': {'''達''': {'''''': 1}}}}}}}}}   )






			def       lowerCamelCase  (self   )					->       List[Any]:


							'''simple docstring'''



							snake_case_						:   Dict           =  Trie()
							self.assertEqual(trie.split('''[CLS] This is a extra_id_100'''   ) ,							['''[CLS] This is a extra_id_100''']   )
							trie.add('''[CLS]'''   )
							trie.add('''extra_id_1'''   )
							trie.add('''extra_id_100'''   )
							self.assertEqual(trie.split('''[CLS] This is a extra_id_100'''   ) ,							['''[CLS]''', ''' This is a ''', '''extra_id_100''']   )






			def       lowerCamelCase  (self   )					->       List[str]:


							'''simple docstring'''



							snake_case_						:   int           =  Trie()
							trie.add('''A'''   )
							self.assertEqual(trie.split('''ABC'''   ) ,							['''A''', '''BC''']   )
							self.assertEqual(trie.split('''BCA'''   ) ,							['''BC''', '''A''']   )






			def       lowerCamelCase  (self   )					->       Union[str, Any]:


							'''simple docstring'''



							snake_case_						:   int           =  Trie()
							trie.add('''TOKEN]'''   )
							trie.add('''[SPECIAL_TOKEN]'''   )
							self.assertEqual(trie.split('''This is something [SPECIAL_TOKEN]'''   ) ,							['''This is something ''', '''[SPECIAL_TOKEN]''']   )






			def       lowerCamelCase  (self   )					->       int:


							'''simple docstring'''



							snake_case_						:   Union[str, Any]           =  Trie()
							trie.add('''A'''   )
							trie.add('''P'''   )
							trie.add('''[SPECIAL_TOKEN]'''   )
							self.assertEqual(trie.split('''This is something [SPECIAL_TOKEN]'''   ) ,							['''This is something ''', '''[SPECIAL_TOKEN]''']   )






			def       lowerCamelCase  (self   )					->       Optional[int]:


							'''simple docstring'''



							snake_case_						:   str           =  Trie()
							trie.add('''AB'''   )
							trie.add('''B'''   )
							trie.add('''C'''   )
							self.assertEqual(trie.split('''ABC'''   ) ,							['''AB''', '''C''']   )






			def       lowerCamelCase  (self   )					->       List[Any]:


							'''simple docstring'''



							snake_case_						:   Optional[int]           =  Trie()
							trie.add('''ABC'''   )
							trie.add('''B'''   )
							trie.add('''CD'''   )
							self.assertEqual(trie.split('''ABCD'''   ) ,							['''ABC''', '''D''']   )






			def       lowerCamelCase  (self   )					->       int:


							'''simple docstring'''



							snake_case_						:   str           =  Trie()
							snake_case_						:   Optional[Any]           =  trie.cut_text('''ABC''' ,							[0, 0, 2, 1, 2, 3]   )
							self.assertEqual(_SCREAMING_SNAKE_CASE ,							['''AB''', '''C''']   )











from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_sentencepiece_available,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


lowerCAmelCase_    =    {'''configuration_xglm''': ['''XGLM_PRETRAINED_CONFIG_ARCHIVE_MAP''', '''XGLMConfig''']}

try:
	if not is_sentencepiece_available():
		raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
	pass
else:
	lowerCAmelCase_    =    ['''XGLMTokenizer''']

try:
	if not is_tokenizers_available():
		raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
	pass
else:
	lowerCAmelCase_    =    ['''XGLMTokenizerFast''']

try:
	if not is_torch_available():
		raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
	pass
else:
	lowerCAmelCase_    =    [
	    '''XGLM_PRETRAINED_MODEL_ARCHIVE_LIST''',
	    '''XGLMForCausalLM''',
	    '''XGLMModel''',
	    '''XGLMPreTrainedModel''',
	]


try:
	if not is_flax_available():
		raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
	pass
else:
	lowerCAmelCase_    =    [
	    '''FlaxXGLMForCausalLM''',
	    '''FlaxXGLMModel''',
	    '''FlaxXGLMPreTrainedModel''',
	]


try:
	if not is_tf_available():
		raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
	pass
else:
	lowerCAmelCase_    =    [
	    '''TF_XGLM_PRETRAINED_MODEL_ARCHIVE_LIST''',
	    '''TFXGLMForCausalLM''',
	    '''TFXGLMModel''',
	    '''TFXGLMPreTrainedModel''',
	]


if TYPE_CHECKING:
	from .configuration_xglm import XGLM_PRETRAINED_CONFIG_ARCHIVE_MAP, XGLMConfig

	try:
		if not is_sentencepiece_available():
			raise OptionalDependencyNotAvailable()
	except OptionalDependencyNotAvailable:
		pass
	else:
		from .tokenization_xglm import XGLMTokenizer

	try:
		if not is_tokenizers_available():
			raise OptionalDependencyNotAvailable()
	except OptionalDependencyNotAvailable:
		pass
	else:
		from .tokenization_xglm_fast import XGLMTokenizerFast

	try:
		if not is_torch_available():
			raise OptionalDependencyNotAvailable()
	except OptionalDependencyNotAvailable:
		pass
	else:
		from .modeling_xglm import XGLM_PRETRAINED_MODEL_ARCHIVE_LIST, XGLMForCausalLM, XGLMModel, XGLMPreTrainedModel

	try:
		if not is_flax_available():
			raise OptionalDependencyNotAvailable()
	except OptionalDependencyNotAvailable:
		pass
	else:
		from .modeling_flax_xglm import FlaxXGLMForCausalLM, FlaxXGLMModel, FlaxXGLMPreTrainedModel

	try:
		if not is_tf_available():
			raise OptionalDependencyNotAvailable()
	except OptionalDependencyNotAvailable:
		pass
	else:
		from .modeling_tf_xglm import (
		    TF_XGLM_PRETRAINED_MODEL_ARCHIVE_LIST,
		    TFXGLMForCausalLM,
		    TFXGLMModel,
		    TFXGLMPreTrainedModel,
		)


else:
	import sys

	lowerCAmelCase_    =    _LazyModule(__name__, globals()['''__file__'''], _import_structure)
