



import os
import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from huggingface_hub.file_download import http_get
from requests.exceptions import HTTPError

from transformers import (
    AlbertTokenizer,
    AutoTokenizer,
    BertTokenizer,
    BertTokenizerFast,
    GPTaTokenizerFast,
    is_tokenizers_available,
)
from transformers.testing_utils import TOKEN, USER, is_staging_test, require_tokenizers
from transformers.tokenization_utils import Trie


sys.path.append(str(Path(__file__).parent.parent / """utils"""))

from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
      from test_module.custom_tokenization_fast import CustomTokenizerFast






class    SCREAMING_SNAKE_CASE			(							unittest.TestCase					):





       def 						UpperCamelCase_					(   self		:		List[Any]	):

           '''simple docstring'''
           __a       =		mock.Mock()
           __a       =		500
           __a       =		{}
           __a       =		HTTPError
           __a       =		{}

           # Download this model to make sure it's in the cache.
           __a       =		BertTokenizer.from_pretrained("""hf-internal-testing/tiny-random-bert"""	)

           # Under the mock environment we get a 500 error when trying to reach the tokenizer.
           with mock.patch("""requests.Session.request"""      ,				return_value=__lowercase	) as mock_head:
               __a       =		BertTokenizer.from_pretrained("""hf-internal-testing/tiny-random-bert"""	)
               # This check we did call the fake head request
               mock_head.assert_called()





       @require_tokenizers
       def 						UpperCamelCase_					(   self		:		int	):

           '''simple docstring'''
           __a       =		mock.Mock()
           __a       =		500
           __a       =		{}
           __a       =		HTTPError
           __a       =		{}

           # Download this model to make sure it's in the cache.
           __a       =		GPTaTokenizerFast.from_pretrained("""gpt2"""	)

           # Under the mock environment we get a 500 error when trying to reach the tokenizer.
           with mock.patch("""requests.Session.request"""      ,				return_value=__lowercase	) as mock_head:
               __a       =		GPTaTokenizerFast.from_pretrained("""gpt2"""	)
               # This check we did call the fake head request
               mock_head.assert_called()





       def 						UpperCamelCase_					(   self		:		str	):

           '''simple docstring'''




           try:
               __a       =		tempfile.mktemp()
               with open(__lowercase      ,				"""wb"""	) as f:
                   http_get("""https://huggingface.co/albert-base-v1/resolve/main/spiece.model"""      ,				__lowercase	)

               __a       =		AlbertTokenizer.from_pretrained(__lowercase	)
           finally:
               os.remove(__lowercase	)

           # Supporting this legacy load introduced a weird bug where the tokenizer would load local files if they are in
           # the current folder and have the right name.
           if os.path.isfile("""tokenizer.json"""	):
               # We skip the test if the user has a `tokenizer.json` in this folder to avoid deleting it.
               return
           try:
               with open("""tokenizer.json"""      ,				"""wb"""	) as f:
                   http_get("""https://huggingface.co/hf-internal-testing/tiny-random-bert/blob/main/tokenizer.json"""      ,				__lowercase	)
               __a       =		AutoTokenizer.from_pretrained("""hf-internal-testing/tiny-random-gpt2"""	)
               # The tiny random BERT has a vocab size of 1024, tiny gpt2 as a vocab size of 1000
               self.assertEqual(tokenizer.vocab_size      ,				1000	)
               # Tokenizer should depend on the remote checkpoint, not the local tokenizer.json file.

           finally:
               os.remove("""tokenizer.json"""	)







       def 						UpperCamelCase_					(   self		:		Optional[Any]	):

           '''simple docstring'''
           __a       =		AlbertTokenizer.from_pretrained("""https://huggingface.co/albert-base-v1/resolve/main/spiece.model"""	)






@is_staging_test
class    SCREAMING_SNAKE_CASE			(							unittest.TestCase					):
       __lowerCamelCase	:	List[str]    =["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]", "bla", "blou"]





       @classmethod
       def 						UpperCamelCase_					(   cls		:		Optional[int]	):

           '''simple docstring'''
           __a       =		TOKEN
           HfFolder.save_token(__lowercase	)





       @classmethod
       def 						UpperCamelCase_					(   cls		:		Optional[Any]	):

           '''simple docstring'''




           try:
               delete_repo(token=cls._token      ,				repo_id="""test-tokenizer"""	)
           except HTTPError:
               pass

           try:
               delete_repo(token=cls._token      ,				repo_id="""valid_org/test-tokenizer-org"""	)
           except HTTPError:
               pass

           try:
               delete_repo(token=cls._token      ,				repo_id="""test-dynamic-tokenizer"""	)
           except HTTPError:
               pass





       def 						UpperCamelCase_					(   self		:		Optional[Any]	):

           '''simple docstring'''




           with tempfile.TemporaryDirectory() as tmp_dir:
               __a       =		os.path.join(__lowercase      ,				"""vocab.txt"""	)
               with open(__lowercase      ,				"""w"""      ,				encoding="""utf-8"""	) as vocab_writer:
                   vocab_writer.write("""""".join([x + """\n""" for x in self.vocab_tokens]	)	)
               __a       =		BertTokenizer(__lowercase	)

           tokenizer.push_to_hub("""test-tokenizer"""      ,				use_auth_token=self._token	)
           __a       =		BertTokenizer.from_pretrained(F"{USER}/test-tokenizer"	)
           self.assertDictEqual(new_tokenizer.vocab      ,				tokenizer.vocab	)

           # Reset repo
           delete_repo(token=self._token      ,				repo_id="""test-tokenizer"""	)

           # Push to hub via save_pretrained
           with tempfile.TemporaryDirectory() as tmp_dir:
               tokenizer.save_pretrained(__lowercase      ,				repo_id="""test-tokenizer"""      ,				push_to_hub=__lowercase      ,				use_auth_token=self._token	)

           __a       =		BertTokenizer.from_pretrained(F"{USER}/test-tokenizer"	)
           self.assertDictEqual(new_tokenizer.vocab      ,				tokenizer.vocab	)





       def 						UpperCamelCase_					(   self		:		Union[str, Any]	):

           '''simple docstring'''




           with tempfile.TemporaryDirectory() as tmp_dir:
               __a       =		os.path.join(__lowercase      ,				"""vocab.txt"""	)
               with open(__lowercase      ,				"""w"""      ,				encoding="""utf-8"""	) as vocab_writer:
                   vocab_writer.write("""""".join([x + """\n""" for x in self.vocab_tokens]	)	)
               __a       =		BertTokenizer(__lowercase	)

           tokenizer.push_to_hub("""valid_org/test-tokenizer-org"""      ,				use_auth_token=self._token	)
           __a       =		BertTokenizer.from_pretrained("""valid_org/test-tokenizer-org"""	)
           self.assertDictEqual(new_tokenizer.vocab      ,				tokenizer.vocab	)

           # Reset repo
           delete_repo(token=self._token      ,				repo_id="""valid_org/test-tokenizer-org"""	)

           # Push to hub via save_pretrained
           with tempfile.TemporaryDirectory() as tmp_dir:
               tokenizer.save_pretrained(
                   __lowercase      ,				repo_id="""valid_org/test-tokenizer-org"""      ,				push_to_hub=__lowercase      ,				use_auth_token=self._token	)

           __a       =		BertTokenizer.from_pretrained("""valid_org/test-tokenizer-org"""	)
           self.assertDictEqual(new_tokenizer.vocab      ,				tokenizer.vocab	)







       @require_tokenizers
       def 						UpperCamelCase_					(   self		:		int	):

           '''simple docstring'''
           CustomTokenizer.register_for_auto_class()
           with tempfile.TemporaryDirectory() as tmp_dir:
               __a       =		os.path.join(__lowercase      ,				"""vocab.txt"""	)
               with open(__lowercase      ,				"""w"""      ,				encoding="""utf-8"""	) as vocab_writer:
                   vocab_writer.write("""""".join([x + """\n""" for x in self.vocab_tokens]	)	)
               __a       =		CustomTokenizer(__lowercase	)

           # No fast custom tokenizer
           tokenizer.push_to_hub("""test-dynamic-tokenizer"""      ,				use_auth_token=self._token	)

           __a       =		AutoTokenizer.from_pretrained(F"{USER}/test-dynamic-tokenizer"      ,				trust_remote_code=__lowercase	)
           # Can't make an isinstance check because the new_model.config is from the CustomTokenizer class of a dynamic module
           self.assertEqual(tokenizer.__class__.__name__      ,				"""CustomTokenizer"""	)

           # Fast and slow custom tokenizer
           CustomTokenizerFast.register_for_auto_class()
           with tempfile.TemporaryDirectory() as tmp_dir:
               __a       =		os.path.join(__lowercase      ,				"""vocab.txt"""	)
               with open(__lowercase      ,				"""w"""      ,				encoding="""utf-8"""	) as vocab_writer:
                   vocab_writer.write("""""".join([x + """\n""" for x in self.vocab_tokens]	)	)

               __a       =		BertTokenizerFast.from_pretrained(__lowercase	)
               bert_tokenizer.save_pretrained(__lowercase	)
               __a       =		CustomTokenizerFast.from_pretrained(__lowercase	)

           tokenizer.push_to_hub("""test-dynamic-tokenizer"""      ,				use_auth_token=self._token	)

           __a       =		AutoTokenizer.from_pretrained(F"{USER}/test-dynamic-tokenizer"      ,				trust_remote_code=__lowercase	)
           # Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
           self.assertEqual(tokenizer.__class__.__name__      ,				"""CustomTokenizerFast"""	)
           __a       =		AutoTokenizer.from_pretrained(
               F"{USER}/test-dynamic-tokenizer"      ,				use_fast=__lowercase      ,				trust_remote_code=__lowercase	)
           # Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
           self.assertEqual(tokenizer.__class__.__name__      ,				"""CustomTokenizer"""	)






class    SCREAMING_SNAKE_CASE			(							unittest.TestCase					):





       def 						UpperCamelCase_					(   self		:		int	):

           '''simple docstring'''
           __a       =		Trie()
           trie.add("""Hello 友達"""	)
           self.assertEqual(trie.data      ,				{"""H""": {"""e""": {"""l""": {"""l""": {"""o""": {""" """: {"""友""": {"""達""": {"""""": 1}}}}}}}}}	)
           trie.add("""Hello"""	)
           trie.data
           self.assertEqual(trie.data      ,				{"""H""": {"""e""": {"""l""": {"""l""": {"""o""": {"""""": 1, """ """: {"""友""": {"""達""": {"""""": 1}}}}}}}}}	)





       def 						UpperCamelCase_					(   self		:		Union[str, Any]	):

           '''simple docstring'''
           __a       =		Trie()
           self.assertEqual(trie.split("""[CLS] This is a extra_id_100"""	)      ,				["""[CLS] This is a extra_id_100"""]	)
           trie.add("""[CLS]"""	)
           trie.add("""extra_id_1"""	)
           trie.add("""extra_id_100"""	)
           self.assertEqual(trie.split("""[CLS] This is a extra_id_100"""	)      ,				["""[CLS]""", """ This is a """, """extra_id_100"""]	)





       def 						UpperCamelCase_					(   self		:		Tuple	):

           '''simple docstring'''
           __a       =		Trie()
           trie.add("""A"""	)
           self.assertEqual(trie.split("""ABC"""	)      ,				["""A""", """BC"""]	)
           self.assertEqual(trie.split("""BCA"""	)      ,				["""BC""", """A"""]	)





       def 						UpperCamelCase_					(   self		:		Union[str, Any]	):

           '''simple docstring'''
           __a       =		Trie()
           trie.add("""TOKEN]"""	)
           trie.add("""[SPECIAL_TOKEN]"""	)
           self.assertEqual(trie.split("""This is something [SPECIAL_TOKEN]"""	)      ,				["""This is something """, """[SPECIAL_TOKEN]"""]	)





       def 						UpperCamelCase_					(   self		:		Optional[Any]	):

           '''simple docstring'''
           __a       =		Trie()
           trie.add("""A"""	)
           trie.add("""P"""	)
           trie.add("""[SPECIAL_TOKEN]"""	)
           self.assertEqual(trie.split("""This is something [SPECIAL_TOKEN]"""	)      ,				["""This is something """, """[SPECIAL_TOKEN]"""]	)





       def 						UpperCamelCase_					(   self		:		List[Any]	):

           '''simple docstring'''
           __a       =		Trie()
           trie.add("""AB"""	)
           trie.add("""B"""	)
           trie.add("""C"""	)
           self.assertEqual(trie.split("""ABC"""	)      ,				["""AB""", """C"""]	)





       def 						UpperCamelCase_					(   self		:		Union[str, Any]	):

           '''simple docstring'''
           __a       =		Trie()
           trie.add("""ABC"""	)
           trie.add("""B"""	)
           trie.add("""CD"""	)
           self.assertEqual(trie.split("""ABCD"""	)      ,				["""ABC""", """D"""]	)







       def 						UpperCamelCase_					(   self		:		Optional[int]	):

           '''simple docstring'''
           __a       =		Trie()
           __a       =		trie.cut_text("""ABC"""      ,				[0, 0, 2, 1, 2, 3]	)
           self.assertEqual(__lowercase      ,				["""AB""", """C"""]	)





from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_flax_available, is_torch_available


lowerCamelCase__			= {"""configuration_speech_encoder_decoder""": ["""SpeechEncoderDecoderConfig"""]}

try:
						if not is_torch_available():
												raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
						pass
else:
						lowerCamelCase__			= ["""SpeechEncoderDecoderModel"""]

try:
						if not is_flax_available():
												raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
						pass
else:
						lowerCamelCase__			= ["""FlaxSpeechEncoderDecoderModel"""]

if TYPE_CHECKING:
						from .configuration_speech_encoder_decoder import SpeechEncoderDecoderConfig

						try:
												if not is_torch_available():
																		raise OptionalDependencyNotAvailable()
						except OptionalDependencyNotAvailable:
												pass
						else:
												from .modeling_speech_encoder_decoder import SpeechEncoderDecoderModel

						try:
												if not is_flax_available():
																		raise OptionalDependencyNotAvailable()
						except OptionalDependencyNotAvailable:
												pass
						else:
												from .modeling_flax_speech_encoder_decoder import FlaxSpeechEncoderDecoderModel

else:
						import sys

						lowerCamelCase__			= _LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)
