
'''simple docstring'''
from google.protobuf import descriptor as _descriptor
from google.protobuf import descriptor_pool as _descriptor_pool
from google.protobuf import symbol_database as _symbol_database
from google.protobuf.internal import builder as _builder


# @@protoc_insertion_point(imports)

lowerCAmelCase					:    Dict           = _symbol_database.Default()


lowerCAmelCase					:    Union[str, Any]           = _descriptor_pool.Default().AddSerializedFile(
    B'\n\x19sentencepiece_model.proto\x12\rsentencepiece"\x80\x0c\n\x0bTrainerSpec\x12\r\n\x05input\x18\x01 \x03(\t\x12\x14\n\x0cinput_format\x18\x07 \x01(\t\x12\x14\n\x0cmodel_prefix\x18\x02 \x01(\t\x12\x41\n\nmodel_type\x18\x03 \x01(\x0e\x32$.sentencepiece.TrainerSpec.ModelType:\x07UNIGRAM\x12\x18\n\nvocab_size\x18\x04 \x01(\x05:\x04\x38\x30\x30\x30\x12\x17\n\x0f\x61\x63\x63\x65pt_language\x18\x05 \x03(\t\x12 \n\x15self_test_sample_size\x18\x06 \x01(\x05:\x01\x30\x12*\n\x1b\x65nable_differential_privacy\x18\x32 \x01(\x08:\x05\x66\x61lse\x12+\n differential_privacy_noise_level\x18\x33 \x01(\x02:\x01\x30\x12\x32\n\'differential_privacy_clipping_threshold\x18\x34 \x01(\x04:\x01\x30\x12"\n\x12\x63haracter_coverage\x18\n \x01(\x02:\x06\x30.9995\x12\x1e\n\x13input_sentence_size\x18\x0b \x01(\x04:\x01\x30\x12$\n\x16shuffle_input_sentence\x18\x13 \x01(\x08:\x04true\x12 \n\x14mining_sentence_size\x18\x0c \x01(\x05\x42\x02\x18\x01\x12"\n\x16training_sentence_size\x18\r \x01(\x05\x42\x02\x18\x01\x12(\n\x17seed_sentencepiece_size\x18\x0e \x01(\x05:\x07\x31\x30\x30\x30\x30\x30\x30\x12\x1e\n\x10shrinking_factor\x18\x0f \x01(\x02:\x04\x30.75\x12!\n\x13max_sentence_length\x18\x12 \x01(\x05:\x04\x34\x31\x39\x32\x12\x17\n\x0bnum_threads\x18\x10 \x01(\x05:\x02\x31\x36\x12\x1d\n\x12num_sub_iterations\x18\x11 \x01(\x05:\x01\x32\x12$\n\x18max_sentencepiece_length\x18\x14 \x01(\x05:\x02\x31\x36\x12%\n\x17split_by_unicode_script\x18\x15 \x01(\x08:\x04true\x12\x1d\n\x0fsplit_by_number\x18\x17 \x01(\x08:\x04true\x12!\n\x13split_by_whitespace\x18\x16 \x01(\x08:\x04true\x12)\n\x1atreat_whitespace_as_suffix\x18\x18 \x01(\x08:\x05\x66\x61lse\x12+\n\x1c\x61llow_whitespace_only_pieces\x18\x1a \x01(\x08:\x05\x66\x61lse\x12\x1b\n\x0csplit_digits\x18\x19 \x01(\x08:\x05\x66\x61lse\x12#\n\x19pretokenization_delimiter\x18\x35 \x01(\t:\x00\x12\x17\n\x0f\x63ontrol_symbols\x18\x1e \x03(\t\x12\x1c\n\x14user_defined_symbols\x18\x1f \x03(\t\x12\x16\n\x0erequired_chars\x18$ \x01(\t\x12\x1c\n\rbyte_fallback\x18# \x01(\x08:\x05\x66\x61lse\x12+\n\x1dvocabulary_output_piece_score\x18  \x01(\x08:\x04true\x12\x1e\n\x10hard_vocab_limit\x18! \x01(\x08:\x04true\x12\x1c\n\ruse_all_vocab\x18" \x01(\x08:\x05\x66\x61lse\x12\x11\n\x06unk_id\x18( \x01(\x05:\x01\x30\x12\x11\n\x06\x62os_id\x18) \x01(\x05:\x01\x31\x12\x11\n\x06\x65os_id\x18* \x01(\x05:\x01\x32\x12\x12\n\x06pad_id\x18+ \x01(\x05:\x02-1\x12\x18\n\tunk_piece\x18- \x01(\t:\x05<unk>\x12\x16\n\tbos_piece\x18. \x01(\t:\x03<s>\x12\x17\n\teos_piece\x18/ \x01(\t:\x04</s>\x12\x18\n\tpad_piece\x18\x30 \x01(\t:\x05<pad>\x12\x1a\n\x0bunk_surface\x18, \x01(\t:\x05 \xe2\x81\x87 \x12+\n\x1ctrain_extremely_large_corpus\x18\x31 \x01(\x08:\x05\x66\x61lse"5\n\tModelType\x12\x0b\n\x07UNIGRAM\x10\x01\x12\x07\n\x03\x42PE\x10\x02\x12\x08\n\x04WORD\x10\x03\x12\x08\n\x04\x43HAR\x10\x04*\t\x08\xc8\x01\x10\x80\x80\x80\x80\x02"\xd1\x01\n\x0eNormalizerSpec\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x1c\n\x14precompiled_charsmap\x18\x02 \x01(\x0c\x12\x1e\n\x10\x61\x64\x64_dummy_prefix\x18\x03 \x01(\x08:\x04true\x12&\n\x18remove_extra_whitespaces\x18\x04 \x01(\x08:\x04true\x12 \n\x12\x65scape_whitespaces\x18\x05 \x01(\x08:\x04true\x12\x1e\n\x16normalization_rule_tsv\x18\x06 \x01(\t*\t\x08\xc8\x01\x10\x80\x80\x80\x80\x02"y\n\x0cSelfTestData\x12\x33\n\x07samples\x18\x01 \x03(\x0b\x32".sentencepiece.SelfTestData.Sample\x1a)\n\x06Sample\x12\r\n\x05input\x18\x01 \x01(\t\x12\x10\n\x08\x65xpected\x18\x02 \x01(\t*\t\x08\xc8\x01\x10\x80\x80\x80\x80\x02"\xfe\x03\n\nModelProto\x12\x37\n\x06pieces\x18\x01 \x03(\x0b\x32\'.sentencepiece.ModelProto.SentencePiece\x12\x30\n\x0ctrainer_spec\x18\x02 \x01(\x0b\x32\x1a.sentencepiece.TrainerSpec\x12\x36\n\x0fnormalizer_spec\x18\x03 \x01(\x0b\x32\x1d.sentencepiece.NormalizerSpec\x12\x33\n\x0eself_test_data\x18\x04 \x01(\x0b\x32\x1b.sentencepiece.SelfTestData\x12\x38\n\x11\x64\x65normalizer_spec\x18\x05 \x01(\x0b\x32\x1d.sentencepiece.NormalizerSpec\x1a\xd2\x01\n\rSentencePiece\x12\r\n\x05piece\x18\x01 \x01(\t\x12\r\n\x05score\x18\x02 \x01(\x02\x12\x42\n\x04type\x18\x03 \x01(\x0e\x32,.sentencepiece.ModelProto.SentencePiece.Type:\x06NORMAL"T\n\x04Type\x12\n\n\x06NORMAL\x10\x01\x12\x0b\n\x07UNKNOWN\x10\x02\x12\x0b\n\x07\x43ONTROL\x10\x03\x12\x10\n\x0cUSER_DEFINED\x10\x04\x12\x08\n\x04\x42YTE\x10\x06\x12\n\n\x06UNUSED\x10\x05*\t\x08\xc8\x01\x10\x80\x80\x80\x80\x02*\t\x08\xc8\x01\x10\x80\x80\x80\x80\x02\x42\x02H\x03'
)

lowerCAmelCase					:    Optional[Any]           = globals()
_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'sentencepiece_model_pb2', _globals)
if _descriptor._USE_C_DESCRIPTORS is False:
						lowerCAmelCase					:    Optional[int]           = None
						lowerCAmelCase					:    str           = B'H\003'
						# (generated by protobuf compiler, but `_TRAINERSPEC` is not defined)
						# _TRAINERSPEC.fields_by_name["mining_sentence_size"]._options = None
						# _TRAINERSPEC.fields_by_name["mining_sentence_size"]._serialized_options = b"\030\001"
						# _TRAINERSPEC.fields_by_name["training_sentence_size"]._options = None
						# _TRAINERSPEC.fields_by_name["training_sentence_size"]._serialized_options = b"\030\001"
						lowerCAmelCase					:    Optional[int]           = 45
						lowerCAmelCase					:    Tuple           = 15_81
						lowerCAmelCase					:    Tuple           = 15_17
						lowerCAmelCase					:    Tuple           = 15_70
						lowerCAmelCase					:    Union[str, Any]           = 15_84
						lowerCAmelCase					:    Optional[int]           = 17_93
						lowerCAmelCase					:    int           = 17_95
						lowerCAmelCase					:    Dict           = 19_16
						lowerCAmelCase					:    List[Any]           = 18_64
						lowerCAmelCase					:    Any           = 19_05
						lowerCAmelCase					:    Any           = 19_19
						lowerCAmelCase					:    str           = 24_29
						lowerCAmelCase					:    str           = 22_08
						lowerCAmelCase					:    Any           = 24_18
						lowerCAmelCase					:    Dict           = 23_23
						lowerCAmelCase					:    Optional[int]           = 24_07
# @@protoc_insertion_point(module_scope)









'''simple docstring'''

import json
import os
import sys
import tempfile
import unittest
from pathlib import Path
from shutil import copyfile

from huggingface_hub import HfFolder, Repository, create_repo, delete_repo
from requests.exceptions import HTTPError

import transformers
from transformers import (
    CONFIG_MAPPING,
    FEATURE_EXTRACTOR_MAPPING,
    PROCESSOR_MAPPING,
    TOKENIZER_MAPPING,
    AutoConfig,
    AutoFeatureExtractor,
    AutoProcessor,
    AutoTokenizer,
    BertTokenizer,
    ProcessorMixin,
    WavaVecaConfig,
    WavaVecaFeatureExtractor,
    WavaVecaProcessor,
)
from transformers.testing_utils import TOKEN, USER, get_tests_dir, is_staging_test
from transformers.tokenization_utils import TOKENIZER_CONFIG_FILE
from transformers.utils import FEATURE_EXTRACTOR_NAME, is_tokenizers_available


sys.path.append(str(Path(__file__).parent.parent.parent.parent / '''utils'''))

from test_module.custom_configuration import CustomConfig  # noqa E402
from test_module.custom_feature_extraction import CustomFeatureExtractor  # noqa E402
from test_module.custom_processing import CustomProcessor  # noqa E402
from test_module.custom_tokenization import CustomTokenizer  # noqa E402


SCREAMING_SNAKE_CASE__     :						Optional[int]	=  get_tests_dir('''fixtures/dummy_feature_extractor_config.json''')
SCREAMING_SNAKE_CASE__     :						Any	=  get_tests_dir('''fixtures/vocab.json''')
SCREAMING_SNAKE_CASE__     :						Optional[Any]	=  get_tests_dir('''fixtures''')

class 							a__( unittest.TestCase     ):
			a_	: Union[str, Any]									=      ['''[UNK]''', '''[CLS]''', '''[SEP]''', '''[PAD]''', '''[MASK]''', '''bla''', '''blou''']






			def 						_lowercase  (     self						)							->							Dict:
					snake_case__	     =0






			def 						_lowercase  (     self						)							->							List[Any]:
					snake_case__	     =AutoProcessor.from_pretrained('facebook/wav2vec2-base-960h'						)
					self.assertIsInstance(_UpperCAmelCase					,   _UpperCAmelCase						)






			def 						_lowercase  (     self						)							->							Dict:
					with tempfile.TemporaryDirectory() as tmpdirname:
							snake_case__	     =WavaVecaConfig()
							snake_case__	     =AutoProcessor.from_pretrained('facebook/wav2vec2-base-960h'						)

							# save in new folder
							model_config.save_pretrained(_UpperCAmelCase						)
							processor.save_pretrained(_UpperCAmelCase						)

							snake_case__	     =AutoProcessor.from_pretrained(_UpperCAmelCase						)

					self.assertIsInstance(_UpperCAmelCase					,   _UpperCAmelCase						)






			def 						_lowercase  (     self						)							->							Tuple:
					with tempfile.TemporaryDirectory() as tmpdirname:
							# copy relevant files
							copyfile(_UpperCAmelCase					,   os.path.join(_UpperCAmelCase					,   _UpperCAmelCase						)						)
							copyfile(_UpperCAmelCase					,   os.path.join(_UpperCAmelCase					,   'vocab.json'						)						)

							snake_case__	     =AutoProcessor.from_pretrained(_UpperCAmelCase						)

					self.assertIsInstance(_UpperCAmelCase					,   _UpperCAmelCase						)






			def 						_lowercase  (     self						)							->							Dict:
					with tempfile.TemporaryDirectory() as tmpdirname:
							snake_case__	     =WavaVecaFeatureExtractor()
							snake_case__	     =AutoTokenizer.from_pretrained('facebook/wav2vec2-base-960h'						)

							snake_case__	     =WavaVecaProcessor(_UpperCAmelCase					,   _UpperCAmelCase						)

							# save in new folder
							processor.save_pretrained(_UpperCAmelCase						)

							# drop `processor_class` in tokenizer
							with open(os.path.join(_UpperCAmelCase					,   _UpperCAmelCase						)					,   'r'						) as f:
									snake_case__	     =json.load(_UpperCAmelCase						)
									config_dict.pop('processor_class'						)

							with open(os.path.join(_UpperCAmelCase					,   _UpperCAmelCase						)					,   'w'						) as f:
									f.write(json.dumps(_UpperCAmelCase						)						)

							snake_case__	     =AutoProcessor.from_pretrained(_UpperCAmelCase						)

					self.assertIsInstance(_UpperCAmelCase					,   _UpperCAmelCase						)






			def 						_lowercase  (     self						)							->							Optional[int]:
					with tempfile.TemporaryDirectory() as tmpdirname:
							snake_case__	     =WavaVecaFeatureExtractor()
							snake_case__	     =AutoTokenizer.from_pretrained('facebook/wav2vec2-base-960h'						)

							snake_case__	     =WavaVecaProcessor(_UpperCAmelCase					,   _UpperCAmelCase						)

							# save in new folder
							processor.save_pretrained(_UpperCAmelCase						)

							# drop `processor_class` in feature extractor
							with open(os.path.join(_UpperCAmelCase					,   _UpperCAmelCase						)					,   'r'						) as f:
									snake_case__	     =json.load(_UpperCAmelCase						)
									config_dict.pop('processor_class'						)

							with open(os.path.join(_UpperCAmelCase					,   _UpperCAmelCase						)					,   'w'						) as f:
									f.write(json.dumps(_UpperCAmelCase						)						)

							snake_case__	     =AutoProcessor.from_pretrained(_UpperCAmelCase						)

					self.assertIsInstance(_UpperCAmelCase					,   _UpperCAmelCase						)






			def 						_lowercase  (     self						)							->							List[Any]:
					with tempfile.TemporaryDirectory() as tmpdirname:
							snake_case__	     =WavaVecaConfig(processor_class='Wav2Vec2Processor'						)
							model_config.save_pretrained(_UpperCAmelCase						)
							# copy relevant files
							copyfile(_UpperCAmelCase					,   os.path.join(_UpperCAmelCase					,   'vocab.json'						)						)
							# create emtpy sample processor
							with open(os.path.join(_UpperCAmelCase					,   _UpperCAmelCase						)					,   'w'						) as f:
									f.write('{}'						)

							snake_case__	     =AutoProcessor.from_pretrained(_UpperCAmelCase						)

					self.assertIsInstance(_UpperCAmelCase					,   _UpperCAmelCase						)






			def 						_lowercase  (     self						)							->							int:
					# If remote code is not set, we will time out when asking whether to load the model.
					with self.assertRaises(_UpperCAmelCase						):
							snake_case__	     =AutoProcessor.from_pretrained('hf-internal-testing/test_dynamic_processor'						)
					# If remote code is disabled, we can't load this config.
					with self.assertRaises(_UpperCAmelCase						):
							snake_case__	     =AutoProcessor.from_pretrained(
							    'hf-internal-testing/test_dynamic_processor'					,   trust_remote_code=_UpperCAmelCase						)

					snake_case__	     =AutoProcessor.from_pretrained('hf-internal-testing/test_dynamic_processor'					,   trust_remote_code=_UpperCAmelCase						)
					self.assertTrue(processor.special_attribute_present						)
					self.assertEqual(processor.__class__.__name__					,   'NewProcessor'						)

					snake_case__	     =processor.feature_extractor
					self.assertTrue(feature_extractor.special_attribute_present						)
					self.assertEqual(feature_extractor.__class__.__name__					,   'NewFeatureExtractor'						)

					snake_case__	     =processor.tokenizer
					self.assertTrue(tokenizer.special_attribute_present						)
					if is_tokenizers_available():
							self.assertEqual(tokenizer.__class__.__name__					,   'NewTokenizerFast'						)

							# Test we can also load the slow version
							snake_case__	     =AutoProcessor.from_pretrained(
							    'hf-internal-testing/test_dynamic_processor'					,   trust_remote_code=_UpperCAmelCase					,   use_fast=_UpperCAmelCase						)
							snake_case__	     =new_processor.tokenizer
							self.assertTrue(new_tokenizer.special_attribute_present						)
							self.assertEqual(new_tokenizer.__class__.__name__					,   'NewTokenizer'						)
					else:
							self.assertEqual(tokenizer.__class__.__name__					,   'NewTokenizer'						)






			def 						_lowercase  (     self						)							->							Optional[Any]:
					try:
							AutoConfig.register('custom'					,   _UpperCAmelCase						)
							AutoFeatureExtractor.register(_UpperCAmelCase					,   _UpperCAmelCase						)
							AutoTokenizer.register(_UpperCAmelCase					,   slow_tokenizer_class=_UpperCAmelCase						)
							AutoProcessor.register(_UpperCAmelCase					,   _UpperCAmelCase						)
							# Trying to register something existing in the Transformers library will raise an error
							with self.assertRaises(_UpperCAmelCase						):
									AutoProcessor.register(_UpperCAmelCase					,   _UpperCAmelCase						)

							# Now that the config is registered, it can be used as any other config with the auto-API
							snake_case__	     =CustomFeatureExtractor.from_pretrained(_UpperCAmelCase						)

							with tempfile.TemporaryDirectory() as tmp_dir:
									snake_case__	     =os.path.join(_UpperCAmelCase					,   'vocab.txt'						)
									with open(_UpperCAmelCase					,   'w'					,   encoding='utf-8'						) as vocab_writer:
											vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]						)						)
									snake_case__	     =CustomTokenizer(_UpperCAmelCase						)

							snake_case__	     =CustomProcessor(_UpperCAmelCase					,   _UpperCAmelCase						)

							with tempfile.TemporaryDirectory() as tmp_dir:
									processor.save_pretrained(_UpperCAmelCase						)
									snake_case__	     =AutoProcessor.from_pretrained(_UpperCAmelCase						)
									self.assertIsInstance(_UpperCAmelCase					,   _UpperCAmelCase						)

					finally:
							if "custom" in CONFIG_MAPPING._extra_content:
									del CONFIG_MAPPING._extra_content["custom"]
							if CustomConfig in FEATURE_EXTRACTOR_MAPPING._extra_content:
									del FEATURE_EXTRACTOR_MAPPING._extra_content[CustomConfig]
							if CustomConfig in TOKENIZER_MAPPING._extra_content:
									del TOKENIZER_MAPPING._extra_content[CustomConfig]
							if CustomConfig in PROCESSOR_MAPPING._extra_content:
									del PROCESSOR_MAPPING._extra_content[CustomConfig]






			def 						_lowercase  (     self						)							->							Union[str, Any]:

					class 							a__( snake_case__     ):
								a_	: Optional[int]									=      False





					class 							a__( snake_case__     ):
								a_	: List[Any]									=      False





					class 							a__( snake_case__     ):
								a_	: int									=      '''AutoFeatureExtractor'''
								a_	: int									=      '''AutoTokenizer'''
								a_	: List[str]									=      False

					try:
							AutoConfig.register('custom'					,   _UpperCAmelCase						)
							AutoFeatureExtractor.register(_UpperCAmelCase					,   _UpperCAmelCase						)
							AutoTokenizer.register(_UpperCAmelCase					,   slow_tokenizer_class=_UpperCAmelCase						)
							AutoProcessor.register(_UpperCAmelCase					,   _UpperCAmelCase						)
							# If remote code is not set, the default is to use local classes.
							snake_case__	     =AutoProcessor.from_pretrained('hf-internal-testing/test_dynamic_processor'						)
							self.assertEqual(processor.__class__.__name__					,   'NewProcessor'						)
							self.assertFalse(processor.special_attribute_present						)
							self.assertFalse(processor.feature_extractor.special_attribute_present						)
							self.assertFalse(processor.tokenizer.special_attribute_present						)

							# If remote code is disabled, we load the local ones.
							snake_case__	     =AutoProcessor.from_pretrained(
							    'hf-internal-testing/test_dynamic_processor'					,   trust_remote_code=_UpperCAmelCase						)
							self.assertEqual(processor.__class__.__name__					,   'NewProcessor'						)
							self.assertFalse(processor.special_attribute_present						)
							self.assertFalse(processor.feature_extractor.special_attribute_present						)
							self.assertFalse(processor.tokenizer.special_attribute_present						)

							# If remote is enabled, we load from the Hub.
							snake_case__	     =AutoProcessor.from_pretrained(
							    'hf-internal-testing/test_dynamic_processor'					,   trust_remote_code=_UpperCAmelCase						)
							self.assertEqual(processor.__class__.__name__					,   'NewProcessor'						)
							self.assertTrue(processor.special_attribute_present						)
							self.assertTrue(processor.feature_extractor.special_attribute_present						)
							self.assertTrue(processor.tokenizer.special_attribute_present						)

					finally:
							if "custom" in CONFIG_MAPPING._extra_content:
									del CONFIG_MAPPING._extra_content["custom"]
							if CustomConfig in FEATURE_EXTRACTOR_MAPPING._extra_content:
									del FEATURE_EXTRACTOR_MAPPING._extra_content[CustomConfig]
							if CustomConfig in TOKENIZER_MAPPING._extra_content:
									del TOKENIZER_MAPPING._extra_content[CustomConfig]
							if CustomConfig in PROCESSOR_MAPPING._extra_content:
									del PROCESSOR_MAPPING._extra_content[CustomConfig]






			def 						_lowercase  (     self						)							->							Union[str, Any]:
					snake_case__	     =AutoProcessor.from_pretrained('hf-internal-testing/tiny-random-bert'						)
					self.assertEqual(processor.__class__.__name__					,   'BertTokenizerFast'						)






			def 						_lowercase  (     self						)							->							List[Any]:
					snake_case__	     =AutoProcessor.from_pretrained('hf-internal-testing/tiny-random-convnext'						)
					self.assertEqual(processor.__class__.__name__					,   'ConvNextImageProcessor'						)





@is_staging_test
class 							a__( unittest.TestCase     ):
			a_	: Dict									=      ['''[UNK]''', '''[CLS]''', '''[SEP]''', '''[PAD]''', '''[MASK]''', '''bla''', '''blou''']






			@classmethod
			def 						_lowercase  (     cls						)							->							str:
					snake_case__	     =TOKEN
					HfFolder.save_token(_UpperCAmelCase						)






			@classmethod
			def 						_lowercase  (     cls						)							->							int:
					try:
							delete_repo(token=cls._token					,   repo_id='test-processor'						)
					except HTTPError:
							pass

					try:
							delete_repo(token=cls._token					,   repo_id='valid_org/test-processor-org'						)
					except HTTPError:
							pass

					try:
							delete_repo(token=cls._token					,   repo_id='test-dynamic-processor'						)
					except HTTPError:
							pass






			def 						_lowercase  (     self						)							->							int:
					snake_case__	     =WavaVecaProcessor.from_pretrained(_UpperCAmelCase						)
					with tempfile.TemporaryDirectory() as tmp_dir:
							processor.save_pretrained(
							    os.path.join(_UpperCAmelCase					,   'test-processor'						)					,   push_to_hub=_UpperCAmelCase					,   use_auth_token=self._token						)

							snake_case__	     =WavaVecaProcessor.from_pretrained(f"""{USER}/test-processor"""						)
							for k, v in processor.feature_extractor.__dict__.items():
									self.assertEqual(_UpperCAmelCase					,   getattr(new_processor.feature_extractor					,   _UpperCAmelCase						)						)
							self.assertDictEqual(new_processor.tokenizer.get_vocab()					,   processor.tokenizer.get_vocab()						)






			def 						_lowercase  (     self						)							->							List[str]:
					snake_case__	     =WavaVecaProcessor.from_pretrained(_UpperCAmelCase						)

					with tempfile.TemporaryDirectory() as tmp_dir:
							processor.save_pretrained(
							    os.path.join(_UpperCAmelCase					,   'test-processor-org'						)					,   push_to_hub=_UpperCAmelCase					,   use_auth_token=self._token					,   organization='valid_org'					,   )

							snake_case__	     =WavaVecaProcessor.from_pretrained('valid_org/test-processor-org'						)
							for k, v in processor.feature_extractor.__dict__.items():
									self.assertEqual(_UpperCAmelCase					,   getattr(new_processor.feature_extractor					,   _UpperCAmelCase						)						)
							self.assertDictEqual(new_processor.tokenizer.get_vocab()					,   processor.tokenizer.get_vocab()						)






			def 						_lowercase  (     self						)							->							Any:
					CustomFeatureExtractor.register_for_auto_class()
					CustomTokenizer.register_for_auto_class()
					CustomProcessor.register_for_auto_class()

					snake_case__	     =CustomFeatureExtractor.from_pretrained(_UpperCAmelCase						)

					with tempfile.TemporaryDirectory() as tmp_dir:
							snake_case__	     =os.path.join(_UpperCAmelCase					,   'vocab.txt'						)
							with open(_UpperCAmelCase					,   'w'					,   encoding='utf-8'						) as vocab_writer:
									vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]						)						)
							snake_case__	     =CustomTokenizer(_UpperCAmelCase						)

					snake_case__	     =CustomProcessor(_UpperCAmelCase					,   _UpperCAmelCase						)

					with tempfile.TemporaryDirectory() as tmp_dir:
							create_repo(f"""{USER}/test-dynamic-processor"""					,   token=self._token						)
							snake_case__	     =Repository(_UpperCAmelCase					,   clone_from=f"""{USER}/test-dynamic-processor"""					,   token=self._token						)
							processor.save_pretrained(_UpperCAmelCase						)

							# This has added the proper auto_map field to the feature extractor config
							self.assertDictEqual(
							    processor.feature_extractor.auto_map					,   {
							        'AutoFeatureExtractor': 'custom_feature_extraction.CustomFeatureExtractor',
							        'AutoProcessor': 'custom_processing.CustomProcessor',
							    }					,   )

							# This has added the proper auto_map field to the tokenizer config
							with open(os.path.join(_UpperCAmelCase					,   'tokenizer_config.json'						)						) as f:
									snake_case__	     =json.load(_UpperCAmelCase						)
							self.assertDictEqual(
							    tokenizer_config['auto_map']					,   {
							        'AutoTokenizer': ['custom_tokenization.CustomTokenizer', None],
							        'AutoProcessor': 'custom_processing.CustomProcessor',
							    }					,   )

							# The code has been copied from fixtures
							self.assertTrue(os.path.isfile(os.path.join(_UpperCAmelCase					,   'custom_feature_extraction.py'						)						)						)
							self.assertTrue(os.path.isfile(os.path.join(_UpperCAmelCase					,   'custom_tokenization.py'						)						)						)
							self.assertTrue(os.path.isfile(os.path.join(_UpperCAmelCase					,   'custom_processing.py'						)						)						)

							repo.push_to_hub()

					snake_case__	     =AutoProcessor.from_pretrained(f"""{USER}/test-dynamic-processor"""					,   trust_remote_code=_UpperCAmelCase						)
					# Can't make an isinstance check because the new_processor is from the CustomProcessor class of a dynamic module
					self.assertEqual(new_processor.__class__.__name__					,   'CustomProcessor'						)




