"""simple docstring"""







import os
import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from huggingface_hub.file_download import http_get
from requests.exceptions import HTTPError

from transformers import (
    AlbertTokenizer,
    AutoTokenizer,
    BertTokenizer,
    BertTokenizerFast,
    GPTaTokenizerFast,
    is_tokenizers_available,
)
from transformers.testing_utils import TOKEN, USER, is_staging_test, require_tokenizers
from transformers.tokenization_utils import Trie


sys.path.append(str(Path(__file__).parent.parent / "utils"))

from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
	from test_module.custom_tokenization_fast import CustomTokenizerFast

class 	UpperCamelCase    (					unittest.TestCase		):



				def       _lowercase  (self       :   Optional[Any])					->			Tuple:
					__snake_case    :							Dict   	=				mock.Mock()
					__snake_case    :							int   	=				5_00
					__snake_case    :							int   	=				{}
					__snake_case    :							List[Any]   	=				HTTPError
					__snake_case    :							Dict   	=				{}

					# Download this model to make sure it's in the cache.
					__snake_case    :							Optional[int]   	=				BertTokenizer.from_pretrained('hf-internal-testing/tiny-random-bert')

					# Under the mock environment we get a 500 error when trying to reach the tokenizer.
					with mock.patch('requests.Session.request'			,						return_value=A_) as mock_head:
						__snake_case    :							List[Any]   	=				BertTokenizer.from_pretrained('hf-internal-testing/tiny-random-bert')
						# This check we did call the fake head request
						mock_head.assert_called()



				@require_tokenizers
				def       _lowercase  (self       :   Optional[int])					->			Dict:
					__snake_case    :							Optional[Any]   	=				mock.Mock()
					__snake_case    :							str   	=				5_00
					__snake_case    :							int   	=				{}
					__snake_case    :							Optional[int]   	=				HTTPError
					__snake_case    :							int   	=				{}

					# Download this model to make sure it's in the cache.
					__snake_case    :							List[str]   	=				GPTaTokenizerFast.from_pretrained('gpt2')

					# Under the mock environment we get a 500 error when trying to reach the tokenizer.
					with mock.patch('requests.Session.request'			,						return_value=A_) as mock_head:
						__snake_case    :							Optional[int]   	=				GPTaTokenizerFast.from_pretrained('gpt2')
						# This check we did call the fake head request
						mock_head.assert_called()



				def       _lowercase  (self       :   Any)					->			Optional[Any]:
					# This test is for deprecated behavior and can be removed in v5
					try:
						__snake_case    :							int   	=				tempfile.mktemp()
						with open(A_			,						'wb') as f:
							http_get('https://huggingface.co/albert-base-v1/resolve/main/spiece.model'			,						A_)

						__snake_case    :							Union[str, Any]   	=				AlbertTokenizer.from_pretrained(A_)
					finally:
						os.remove(A_)

					# Supporting this legacy load introduced a weird bug where the tokenizer would load local files if they are in
					# the current folder and have the right name.
					if os.path.isfile('tokenizer.json'):
						# We skip the test if the user has a `tokenizer.json` in this folder to avoid deleting it.
						return
					try:
						with open('tokenizer.json'			,						'wb') as f:
							http_get('https://huggingface.co/hf-internal-testing/tiny-random-bert/blob/main/tokenizer.json'			,						A_)
						__snake_case    :							List[str]   	=				AutoTokenizer.from_pretrained('hf-internal-testing/tiny-random-gpt2')
						# The tiny random BERT has a vocab size of 1024, tiny gpt2 as a vocab size of 1000
						self.assertEqual(tokenizer.vocab_size			,						10_00)
						# Tokenizer should depend on the remote checkpoint, not the local tokenizer.json file.

					finally:
						os.remove('tokenizer.json')






				def       _lowercase  (self       :   Dict)					->			Dict:
					__snake_case    :							List[str]   	=				AlbertTokenizer.from_pretrained('https://huggingface.co/albert-base-v1/resolve/main/spiece.model')







@is_staging_test
class 	UpperCamelCase    (					unittest.TestCase		):
				UpperCAmelCase       :   int					=	["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]", "bla", "blou"]



				@classmethod
				def       _lowercase  (cls       :   Any)					->			Tuple:
					__snake_case    :							List[str]   	=				TOKEN
					HfFolder.save_token(A_)



				@classmethod
				def       _lowercase  (cls       :   Tuple)					->			Optional[int]:
					try:
						delete_repo(token=cls._token			,						repo_id='test-tokenizer')
					except HTTPError:
						pass

					try:
						delete_repo(token=cls._token			,						repo_id='valid_org/test-tokenizer-org')
					except HTTPError:
						pass

					try:
						delete_repo(token=cls._token			,						repo_id='test-dynamic-tokenizer')
					except HTTPError:
						pass



				def       _lowercase  (self       :   Optional[int])					->			Any:
					with tempfile.TemporaryDirectory() as tmp_dir:
						__snake_case    :							List[str]   	=				os.path.join(A_			,						'vocab.txt')
						with open(A_			,						'w'			,						encoding='utf-8') as vocab_writer:
							vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]))
						__snake_case    :							List[Any]   	=				BertTokenizer(A_)

					tokenizer.push_to_hub('test-tokenizer'			,						use_auth_token=self._token)
					__snake_case    :							List[str]   	=				BertTokenizer.from_pretrained(f"{USER}/test-tokenizer")
					self.assertDictEqual(new_tokenizer.vocab			,						tokenizer.vocab)

					# Reset repo
					delete_repo(token=self._token			,						repo_id='test-tokenizer')

					# Push to hub via save_pretrained
					with tempfile.TemporaryDirectory() as tmp_dir:
						tokenizer.save_pretrained(A_			,						repo_id='test-tokenizer'			,						push_to_hub=A_			,						use_auth_token=self._token)

					__snake_case    :							Union[str, Any]   	=				BertTokenizer.from_pretrained(f"{USER}/test-tokenizer")
					self.assertDictEqual(new_tokenizer.vocab			,						tokenizer.vocab)



				def       _lowercase  (self       :   Optional[int])					->			int:
					with tempfile.TemporaryDirectory() as tmp_dir:
						__snake_case    :							List[str]   	=				os.path.join(A_			,						'vocab.txt')
						with open(A_			,						'w'			,						encoding='utf-8') as vocab_writer:
							vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]))
						__snake_case    :							Optional[Any]   	=				BertTokenizer(A_)

					tokenizer.push_to_hub('valid_org/test-tokenizer-org'			,						use_auth_token=self._token)
					__snake_case    :							Tuple   	=				BertTokenizer.from_pretrained('valid_org/test-tokenizer-org')
					self.assertDictEqual(new_tokenizer.vocab			,						tokenizer.vocab)

					# Reset repo
					delete_repo(token=self._token			,						repo_id='valid_org/test-tokenizer-org')

					# Push to hub via save_pretrained
					with tempfile.TemporaryDirectory() as tmp_dir:
						tokenizer.save_pretrained(
						    A_			,						repo_id='valid_org/test-tokenizer-org'			,						push_to_hub=A_			,						use_auth_token=self._token)

					__snake_case    :							Any   	=				BertTokenizer.from_pretrained('valid_org/test-tokenizer-org')
					self.assertDictEqual(new_tokenizer.vocab			,						tokenizer.vocab)






				@require_tokenizers
				def       _lowercase  (self       :   Union[str, Any])					->			Dict:
					CustomTokenizer.register_for_auto_class()
					with tempfile.TemporaryDirectory() as tmp_dir:
						__snake_case    :							int   	=				os.path.join(A_			,						'vocab.txt')
						with open(A_			,						'w'			,						encoding='utf-8') as vocab_writer:
							vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]))
						__snake_case    :							Tuple   	=				CustomTokenizer(A_)

					# No fast custom tokenizer
					tokenizer.push_to_hub('test-dynamic-tokenizer'			,						use_auth_token=self._token)

					__snake_case    :							Optional[Any]   	=				AutoTokenizer.from_pretrained(f"{USER}/test-dynamic-tokenizer"			,						trust_remote_code=A_)
					# Can't make an isinstance check because the new_model.config is from the CustomTokenizer class of a dynamic module
					self.assertEqual(tokenizer.__class__.__name__			,						'CustomTokenizer')

					# Fast and slow custom tokenizer
					CustomTokenizerFast.register_for_auto_class()
					with tempfile.TemporaryDirectory() as tmp_dir:
						__snake_case    :							Union[str, Any]   	=				os.path.join(A_			,						'vocab.txt')
						with open(A_			,						'w'			,						encoding='utf-8') as vocab_writer:
							vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]))

						__snake_case    :							List[str]   	=				BertTokenizerFast.from_pretrained(A_)
						bert_tokenizer.save_pretrained(A_)
						__snake_case    :							Dict   	=				CustomTokenizerFast.from_pretrained(A_)

					tokenizer.push_to_hub('test-dynamic-tokenizer'			,						use_auth_token=self._token)

					__snake_case    :							Any   	=				AutoTokenizer.from_pretrained(f"{USER}/test-dynamic-tokenizer"			,						trust_remote_code=A_)
					# Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
					self.assertEqual(tokenizer.__class__.__name__			,						'CustomTokenizerFast')
					__snake_case    :							Union[str, Any]   	=				AutoTokenizer.from_pretrained(
					    f"{USER}/test-dynamic-tokenizer"			,						use_fast=A_			,						trust_remote_code=A_)
					# Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
					self.assertEqual(tokenizer.__class__.__name__			,						'CustomTokenizer')







class 	UpperCamelCase    (					unittest.TestCase		):



				def       _lowercase  (self       :   Dict)					->			Optional[int]:
					__snake_case    :							str   	=				Trie()
					trie.add('Hello 友達')
					self.assertEqual(trie.data			,						{'H': {'e': {'l': {'l': {'o': {' ': {'友': {'達': {'': 1}}}}}}}}})
					trie.add('Hello')
					trie.data
					self.assertEqual(trie.data			,						{'H': {'e': {'l': {'l': {'o': {'': 1, ' ': {'友': {'達': {'': 1}}}}}}}}})



				def       _lowercase  (self       :   int)					->			str:
					__snake_case    :							Any   	=				Trie()
					self.assertEqual(trie.split('[CLS] This is a extra_id_100')			,						['[CLS] This is a extra_id_100'])
					trie.add('[CLS]')
					trie.add('extra_id_1')
					trie.add('extra_id_100')
					self.assertEqual(trie.split('[CLS] This is a extra_id_100')			,						['[CLS]', ' This is a ', 'extra_id_100'])



				def       _lowercase  (self       :   Union[str, Any])					->			List[Any]:
					__snake_case    :							Optional[int]   	=				Trie()
					trie.add('A')
					self.assertEqual(trie.split('ABC')			,						['A', 'BC'])
					self.assertEqual(trie.split('BCA')			,						['BC', 'A'])



				def       _lowercase  (self       :   Any)					->			Tuple:
					__snake_case    :							Dict   	=				Trie()
					trie.add('TOKEN]')
					trie.add('[SPECIAL_TOKEN]')
					self.assertEqual(trie.split('This is something [SPECIAL_TOKEN]')			,						['This is something ', '[SPECIAL_TOKEN]'])



				def       _lowercase  (self       :   Dict)					->			Dict:
					__snake_case    :							Any   	=				Trie()
					trie.add('A')
					trie.add('P')
					trie.add('[SPECIAL_TOKEN]')
					self.assertEqual(trie.split('This is something [SPECIAL_TOKEN]')			,						['This is something ', '[SPECIAL_TOKEN]'])



				def       _lowercase  (self       :   int)					->			Optional[Any]:
					__snake_case    :							List[str]   	=				Trie()
					trie.add('AB')
					trie.add('B')
					trie.add('C')
					self.assertEqual(trie.split('ABC')			,						['AB', 'C'])



				def       _lowercase  (self       :   Dict)					->			Tuple:
					__snake_case    :							Optional[Any]   	=				Trie()
					trie.add('ABC')
					trie.add('B')
					trie.add('CD')
					self.assertEqual(trie.split('ABCD')			,						['ABC', 'D'])






				def       _lowercase  (self       :   Optional[int])					->			int:
					__snake_case    :							int   	=				Trie()
					__snake_case    :							Optional[Any]   	=				trie.cut_text('ABC'			,						[0, 0, 2, 1, 2, 3])
					self.assertEqual(A_			,						['AB', 'C'])






"""simple docstring"""







from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_tf_available,
    is_torch_available,
)


_a  :  Optional[Any]=				{
    "configuration_wav2vec2": ["WAV_2_VEC_2_PRETRAINED_CONFIG_ARCHIVE_MAP", "Wav2Vec2Config"],
    "feature_extraction_wav2vec2": ["Wav2Vec2FeatureExtractor"],
    "processing_wav2vec2": ["Wav2Vec2Processor"],
    "tokenization_wav2vec2": ["Wav2Vec2CTCTokenizer", "Wav2Vec2Tokenizer"],
}


try:
	if not is_torch_available():
		raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
	pass
else:
	_a  :  Optional[Any]=				[
	    "WAV_2_VEC_2_PRETRAINED_MODEL_ARCHIVE_LIST",
	    "Wav2Vec2ForAudioFrameClassification",
	    "Wav2Vec2ForCTC",
	    "Wav2Vec2ForMaskedLM",
	    "Wav2Vec2ForPreTraining",
	    "Wav2Vec2ForSequenceClassification",
	    "Wav2Vec2ForXVector",
	    "Wav2Vec2Model",
	    "Wav2Vec2PreTrainedModel",
	]

try:
	if not is_tf_available():
		raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
	pass
else:
	_a  :  List[str]=				[
	    "TF_WAV_2_VEC_2_PRETRAINED_MODEL_ARCHIVE_LIST",
	    "TFWav2Vec2ForCTC",
	    "TFWav2Vec2Model",
	    "TFWav2Vec2PreTrainedModel",
	    "TFWav2Vec2ForSequenceClassification",
	]

try:
	if not is_flax_available():
		raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
	pass
else:
	_a  :  List[Any]=				[
	    "FlaxWav2Vec2ForCTC",
	    "FlaxWav2Vec2ForPreTraining",
	    "FlaxWav2Vec2Model",
	    "FlaxWav2Vec2PreTrainedModel",
	]


if TYPE_CHECKING:
	from .configuration_wavaveca import WAV_2_VEC_2_PRETRAINED_CONFIG_ARCHIVE_MAP, WavaVecaConfig
	from .feature_extraction_wavaveca import WavaVecaFeatureExtractor
	from .processing_wavaveca import WavaVecaProcessor
	from .tokenization_wavaveca import WavaVecaCTCTokenizer, WavaVecaTokenizer

	try:
		if not is_torch_available():
			raise OptionalDependencyNotAvailable()
	except OptionalDependencyNotAvailable:
		pass
	else:
		from .modeling_wavaveca import (
		    WAV_2_VEC_2_PRETRAINED_MODEL_ARCHIVE_LIST,
		    WavaVecaForAudioFrameClassification,
		    WavaVecaForCTC,
		    WavaVecaForMaskedLM,
		    WavaVecaForPreTraining,
		    WavaVecaForSequenceClassification,
		    WavaVecaForXVector,
		    WavaVecaModel,
		    WavaVecaPreTrainedModel,
		)

	try:
		if not is_tf_available():
			raise OptionalDependencyNotAvailable()
	except OptionalDependencyNotAvailable:
		pass
	else:
		from .modeling_tf_wavaveca import (
		    TF_WAV_2_VEC_2_PRETRAINED_MODEL_ARCHIVE_LIST,
		    TFWavaVecaForCTC,
		    TFWavaVecaForSequenceClassification,
		    TFWavaVecaModel,
		    TFWavaVecaPreTrainedModel,
		)

	try:
		if not is_flax_available():
			raise OptionalDependencyNotAvailable()
	except OptionalDependencyNotAvailable:
		pass
	else:
		from .modeling_tf_wavaveca import (
		    FlaxWavaVecaForCTC,
		    FlaxWavaVecaForPreTraining,
		    FlaxWavaVecaModel,
		    FlaxWavaVecaPreTrainedModel,
		)


else:
	import sys

	_a  :  Tuple=				_LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)





