



"""simple docstring"""





import os
import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from huggingface_hub.file_download import http_get
from requests.exceptions import HTTPError

from transformers import (
    AlbertTokenizer,
    AutoTokenizer,
    BertTokenizer,
    BertTokenizerFast,
    GPTaTokenizerFast,
    is_tokenizers_available,
)
from transformers.testing_utils import TOKEN, USER, is_staging_test, require_tokenizers
from transformers.tokenization_utils import Trie


sys.path.append(str(Path(__file__).parent.parent / '''utils'''))

from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
					from test_module.custom_tokenization_fast import CustomTokenizerFast




class       lowercase__      (   unittest.TestCase     ):
							'''simple docstring'''







							def 							lowerCamelCase_ (	self    )      ->				Union[str, Any]:
									_UpperCAmelCase							=		mock.Mock()
									_UpperCAmelCase							=		500
									_UpperCAmelCase							=		{}
									_UpperCAmelCase							=		HTTPError
									_UpperCAmelCase							=		{}

									# Download this model to make sure it's in the cache.
									_UpperCAmelCase							=		BertTokenizer.from_pretrained('hf-internal-testing/tiny-random-bert'    )

									# Under the mock environment we get a 500 error when trying to reach the tokenizer.
									with mock.patch('requests.Session.request'						,							return_value=__UpperCamelCase    ) as mock_head:
											_UpperCAmelCase							=		BertTokenizer.from_pretrained('hf-internal-testing/tiny-random-bert'    )
											# This check we did call the fake head request
											mock_head.assert_called()






							@require_tokenizers
							def 							lowerCamelCase_ (	self    )      ->				Tuple:
									_UpperCAmelCase							=		mock.Mock()
									_UpperCAmelCase							=		500
									_UpperCAmelCase							=		{}
									_UpperCAmelCase							=		HTTPError
									_UpperCAmelCase							=		{}

									# Download this model to make sure it's in the cache.
									_UpperCAmelCase							=		GPTaTokenizerFast.from_pretrained('gpt2'    )

									# Under the mock environment we get a 500 error when trying to reach the tokenizer.
									with mock.patch('requests.Session.request'						,							return_value=__UpperCamelCase    ) as mock_head:
											_UpperCAmelCase							=		GPTaTokenizerFast.from_pretrained('gpt2'    )
											# This check we did call the fake head request
											mock_head.assert_called()






							def 							lowerCamelCase_ (	self    )      ->				str:


									try:
											_UpperCAmelCase							=		tempfile.mktemp()
											with open(__UpperCamelCase						,							'wb'    ) as f:
													http_get('https://huggingface.co/albert-base-v1/resolve/main/spiece.model'						,							__UpperCamelCase    )

											_UpperCAmelCase							=		AlbertTokenizer.from_pretrained(__UpperCamelCase    )
									finally:
											os.remove(__UpperCamelCase    )

									# Supporting this legacy load introduced a weird bug where the tokenizer would load local files if they are in
									# the current folder and have the right name.
									if os.path.isfile('tokenizer.json'    ):
											# We skip the test if the user has a `tokenizer.json` in this folder to avoid deleting it.
											return
									try:
											with open('tokenizer.json'						,							'wb'    ) as f:
													http_get('https://huggingface.co/hf-internal-testing/tiny-random-bert/blob/main/tokenizer.json'						,							__UpperCamelCase    )
											_UpperCAmelCase							=		AutoTokenizer.from_pretrained('hf-internal-testing/tiny-random-gpt2'    )
											# The tiny random BERT has a vocab size of 1024, tiny gpt2 as a vocab size of 1000
											self.assertEqual(tokenizer.vocab_size						,							1000    )
											# Tokenizer should depend on the remote checkpoint, not the local tokenizer.json file.

									finally:
											os.remove('tokenizer.json'    )






							def 							lowerCamelCase_ (	self    )      ->				List[str]:
									_UpperCAmelCase							=		AlbertTokenizer.from_pretrained('https://huggingface.co/albert-base-v1/resolve/main/spiece.model'    )







@is_staging_test
class       lowercase__      (   unittest.TestCase     ):
							'''simple docstring'''







							_UpperCAmelCase   					=       ['''[UNK]''', '''[CLS]''', '''[SEP]''', '''[PAD]''', '''[MASK]''', '''bla''', '''blou''']






							@classmethod
							def 							lowerCamelCase_ (	cls    )      ->				Any:
									_UpperCAmelCase							=		TOKEN
									HfFolder.save_token(__UpperCamelCase    )






							@classmethod
							def 							lowerCamelCase_ (	cls    )      ->				Optional[int]:


									try:
											delete_repo(token=cls._token						,							repo_id='test-tokenizer'    )
									except HTTPError:
											pass

									try:
											delete_repo(token=cls._token						,							repo_id='valid_org/test-tokenizer-org'    )
									except HTTPError:
											pass

									try:
											delete_repo(token=cls._token						,							repo_id='test-dynamic-tokenizer'    )
									except HTTPError:
											pass






							def 							lowerCamelCase_ (	self    )      ->				List[str]:


									with tempfile.TemporaryDirectory() as tmp_dir:
											_UpperCAmelCase							=		os.path.join(__UpperCamelCase						,							'vocab.txt'    )
											with open(__UpperCamelCase						,							'w'						,							encoding='utf-8'    ) as vocab_writer:
													vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]    )    )
											_UpperCAmelCase							=		BertTokenizer(__UpperCamelCase    )

									tokenizer.push_to_hub('test-tokenizer'						,							use_auth_token=self._token    )
									_UpperCAmelCase							=		BertTokenizer.from_pretrained(f'{USER}/test-tokenizer'    )
									self.assertDictEqual(new_tokenizer.vocab						,							tokenizer.vocab    )

									# Reset repo
									delete_repo(token=self._token						,							repo_id='test-tokenizer'    )

									# Push to hub via save_pretrained
									with tempfile.TemporaryDirectory() as tmp_dir:
											tokenizer.save_pretrained(__UpperCamelCase						,							repo_id='test-tokenizer'						,							push_to_hub=__UpperCamelCase						,							use_auth_token=self._token    )

									_UpperCAmelCase							=		BertTokenizer.from_pretrained(f'{USER}/test-tokenizer'    )
									self.assertDictEqual(new_tokenizer.vocab						,							tokenizer.vocab    )






							def 							lowerCamelCase_ (	self    )      ->				Tuple:


									with tempfile.TemporaryDirectory() as tmp_dir:
											_UpperCAmelCase							=		os.path.join(__UpperCamelCase						,							'vocab.txt'    )
											with open(__UpperCamelCase						,							'w'						,							encoding='utf-8'    ) as vocab_writer:
													vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]    )    )
											_UpperCAmelCase							=		BertTokenizer(__UpperCamelCase    )

									tokenizer.push_to_hub('valid_org/test-tokenizer-org'						,							use_auth_token=self._token    )
									_UpperCAmelCase							=		BertTokenizer.from_pretrained('valid_org/test-tokenizer-org'    )
									self.assertDictEqual(new_tokenizer.vocab						,							tokenizer.vocab    )

									# Reset repo
									delete_repo(token=self._token						,							repo_id='valid_org/test-tokenizer-org'    )

									# Push to hub via save_pretrained
									with tempfile.TemporaryDirectory() as tmp_dir:
											tokenizer.save_pretrained(
											    __UpperCamelCase						,							repo_id='valid_org/test-tokenizer-org'						,							push_to_hub=__UpperCamelCase						,							use_auth_token=self._token    )

									_UpperCAmelCase							=		BertTokenizer.from_pretrained('valid_org/test-tokenizer-org'    )
									self.assertDictEqual(new_tokenizer.vocab						,							tokenizer.vocab    )






							@require_tokenizers
							def 							lowerCamelCase_ (	self    )      ->				Dict:
									CustomTokenizer.register_for_auto_class()
									with tempfile.TemporaryDirectory() as tmp_dir:
											_UpperCAmelCase							=		os.path.join(__UpperCamelCase						,							'vocab.txt'    )
											with open(__UpperCamelCase						,							'w'						,							encoding='utf-8'    ) as vocab_writer:
													vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]    )    )
											_UpperCAmelCase							=		CustomTokenizer(__UpperCamelCase    )

									# No fast custom tokenizer
									tokenizer.push_to_hub('test-dynamic-tokenizer'						,							use_auth_token=self._token    )

									_UpperCAmelCase							=		AutoTokenizer.from_pretrained(f'{USER}/test-dynamic-tokenizer'						,							trust_remote_code=__UpperCamelCase    )
									# Can't make an isinstance check because the new_model.config is from the CustomTokenizer class of a dynamic module
									self.assertEqual(tokenizer.__class__.__name__						,							'CustomTokenizer'    )

									# Fast and slow custom tokenizer
									CustomTokenizerFast.register_for_auto_class()
									with tempfile.TemporaryDirectory() as tmp_dir:
											_UpperCAmelCase							=		os.path.join(__UpperCamelCase						,							'vocab.txt'    )
											with open(__UpperCamelCase						,							'w'						,							encoding='utf-8'    ) as vocab_writer:
													vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]    )    )

											_UpperCAmelCase							=		BertTokenizerFast.from_pretrained(__UpperCamelCase    )
											bert_tokenizer.save_pretrained(__UpperCamelCase    )
											_UpperCAmelCase							=		CustomTokenizerFast.from_pretrained(__UpperCamelCase    )

									tokenizer.push_to_hub('test-dynamic-tokenizer'						,							use_auth_token=self._token    )

									_UpperCAmelCase							=		AutoTokenizer.from_pretrained(f'{USER}/test-dynamic-tokenizer'						,							trust_remote_code=__UpperCamelCase    )
									# Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
									self.assertEqual(tokenizer.__class__.__name__						,							'CustomTokenizerFast'    )
									_UpperCAmelCase							=		AutoTokenizer.from_pretrained(
									    f'{USER}/test-dynamic-tokenizer'						,							use_fast=__UpperCamelCase						,							trust_remote_code=__UpperCamelCase    )
									# Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
									self.assertEqual(tokenizer.__class__.__name__						,							'CustomTokenizer'    )







class       lowercase__      (   unittest.TestCase     ):
							'''simple docstring'''







							def 							lowerCamelCase_ (	self    )      ->				List[str]:
									_UpperCAmelCase							=		Trie()
									trie.add('Hello 友達'    )
									self.assertEqual(trie.data						,							{'H': {'e': {'l': {'l': {'o': {' ': {'友': {'達': {'': 1}}}}}}}}}    )
									trie.add('Hello'    )
									trie.data
									self.assertEqual(trie.data						,							{'H': {'e': {'l': {'l': {'o': {'': 1, ' ': {'友': {'達': {'': 1}}}}}}}}}    )






							def 							lowerCamelCase_ (	self    )      ->				Tuple:
									_UpperCAmelCase							=		Trie()
									self.assertEqual(trie.split('[CLS] This is a extra_id_100'    )						,							['[CLS] This is a extra_id_100']    )
									trie.add('[CLS]'    )
									trie.add('extra_id_1'    )
									trie.add('extra_id_100'    )
									self.assertEqual(trie.split('[CLS] This is a extra_id_100'    )						,							['[CLS]', ' This is a ', 'extra_id_100']    )






							def 							lowerCamelCase_ (	self    )      ->				Optional[int]:
									_UpperCAmelCase							=		Trie()
									trie.add('A'    )
									self.assertEqual(trie.split('ABC'    )						,							['A', 'BC']    )
									self.assertEqual(trie.split('BCA'    )						,							['BC', 'A']    )






							def 							lowerCamelCase_ (	self    )      ->				Dict:
									_UpperCAmelCase							=		Trie()
									trie.add('TOKEN]'    )
									trie.add('[SPECIAL_TOKEN]'    )
									self.assertEqual(trie.split('This is something [SPECIAL_TOKEN]'    )						,							['This is something ', '[SPECIAL_TOKEN]']    )






							def 							lowerCamelCase_ (	self    )      ->				Optional[int]:
									_UpperCAmelCase							=		Trie()
									trie.add('A'    )
									trie.add('P'    )
									trie.add('[SPECIAL_TOKEN]'    )
									self.assertEqual(trie.split('This is something [SPECIAL_TOKEN]'    )						,							['This is something ', '[SPECIAL_TOKEN]']    )






							def 							lowerCamelCase_ (	self    )      ->				str:
									_UpperCAmelCase							=		Trie()
									trie.add('AB'    )
									trie.add('B'    )
									trie.add('C'    )
									self.assertEqual(trie.split('ABC'    )						,							['AB', 'C']    )






							def 							lowerCamelCase_ (	self    )      ->				Optional[Any]:
									_UpperCAmelCase							=		Trie()
									trie.add('ABC'    )
									trie.add('B'    )
									trie.add('CD'    )
									self.assertEqual(trie.split('ABCD'    )						,							['ABC', 'D']    )






							def 							lowerCamelCase_ (	self    )      ->				Union[str, Any]:
									_UpperCAmelCase							=		Trie()
									_UpperCAmelCase							=		trie.cut_text('ABC'						,							[0, 0, 2, 1, 2, 3]    )
									self.assertEqual(__UpperCamelCase						,							['AB', 'C']    )





from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_sentencepiece_available,
    is_speech_available,
    is_tf_available,
    is_torch_available,
)


A									=  {
    'configuration_speech_to_text': ['SPEECH_TO_TEXT_PRETRAINED_CONFIG_ARCHIVE_MAP', 'Speech2TextConfig'],
    'processing_speech_to_text': ['Speech2TextProcessor'],
}

try:
		if not is_sentencepiece_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		A									=  ['Speech2TextTokenizer']

try:
		if not is_speech_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		A									=  ['Speech2TextFeatureExtractor']

try:
		if not is_tf_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		A									=  [
		    'TF_SPEECH_TO_TEXT_PRETRAINED_MODEL_ARCHIVE_LIST',
		    'TFSpeech2TextForConditionalGeneration',
		    'TFSpeech2TextModel',
		    'TFSpeech2TextPreTrainedModel',
		]

try:
		if not is_torch_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		A									=  [
		    'SPEECH_TO_TEXT_PRETRAINED_MODEL_ARCHIVE_LIST',
		    'Speech2TextForConditionalGeneration',
		    'Speech2TextModel',
		    'Speech2TextPreTrainedModel',
		]


if TYPE_CHECKING:
		from .configuration_speech_to_text import SPEECH_TO_TEXT_PRETRAINED_CONFIG_ARCHIVE_MAP, SpeechaTextConfig
		from .processing_speech_to_text import SpeechaTextProcessor

		try:
				if not is_sentencepiece_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				from .tokenization_speech_to_text import SpeechaTextTokenizer

		try:
				if not is_speech_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				from .feature_extraction_speech_to_text import SpeechaTextFeatureExtractor

		try:
				if not is_tf_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				from .modeling_tf_speech_to_text import (
				    TF_SPEECH_TO_TEXT_PRETRAINED_MODEL_ARCHIVE_LIST,
				    TFSpeechaTextForConditionalGeneration,
				    TFSpeechaTextModel,
				    TFSpeechaTextPreTrainedModel,
				)

		try:
				if not is_torch_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				from .modeling_speech_to_text import (
				    SPEECH_TO_TEXT_PRETRAINED_MODEL_ARCHIVE_LIST,
				    SpeechaTextForConditionalGeneration,
				    SpeechaTextModel,
				    SpeechaTextPreTrainedModel,
				)

else:
		import sys

		A									=  _LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)
