"""simple docstring"""



from collections import OrderedDict

from ...utils import logging
from .auto_factory import _BaseAutoModelClass, _LazyAutoMapping, auto_class_update
from .configuration_auto import CONFIG_MAPPING_NAMES


__magic_name__     					=       logging.get_logger(__name__)


__magic_name__     					=       OrderedDict(
    [
        # Base model mapping
        ("albert", "FlaxAlbertModel"),
        ("bart", "FlaxBartModel"),
        ("beit", "FlaxBeitModel"),
        ("bert", "FlaxBertModel"),
        ("big_bird", "FlaxBigBirdModel"),
        ("blenderbot", "FlaxBlenderbotModel"),
        ("blenderbot-small", "FlaxBlenderbotSmallModel"),
        ("clip", "FlaxCLIPModel"),
        ("distilbert", "FlaxDistilBertModel"),
        ("electra", "FlaxElectraModel"),
        ("gpt-sw3", "FlaxGPT2Model"),
        ("gpt2", "FlaxGPT2Model"),
        ("gpt_neo", "FlaxGPTNeoModel"),
        ("gptj", "FlaxGPTJModel"),
        ("longt5", "FlaxLongT5Model"),
        ("marian", "FlaxMarianModel"),
        ("mbart", "FlaxMBartModel"),
        ("mt5", "FlaxMT5Model"),
        ("opt", "FlaxOPTModel"),
        ("pegasus", "FlaxPegasusModel"),
        ("regnet", "FlaxRegNetModel"),
        ("resnet", "FlaxResNetModel"),
        ("roberta", "FlaxRobertaModel"),
        ("roberta-prelayernorm", "FlaxRobertaPreLayerNormModel"),
        ("roformer", "FlaxRoFormerModel"),
        ("t5", "FlaxT5Model"),
        ("vision-text-dual-encoder", "FlaxVisionTextDualEncoderModel"),
        ("vit", "FlaxViTModel"),
        ("wav2vec2", "FlaxWav2Vec2Model"),
        ("whisper", "FlaxWhisperModel"),
        ("xglm", "FlaxXGLMModel"),
        ("xlm-roberta", "FlaxXLMRobertaModel"),
    ]
)

__magic_name__     					=       OrderedDict(
    [
        # Model for pre-training mapping
        ("albert", "FlaxAlbertForPreTraining"),
        ("bart", "FlaxBartForConditionalGeneration"),
        ("bert", "FlaxBertForPreTraining"),
        ("big_bird", "FlaxBigBirdForPreTraining"),
        ("electra", "FlaxElectraForPreTraining"),
        ("longt5", "FlaxLongT5ForConditionalGeneration"),
        ("mbart", "FlaxMBartForConditionalGeneration"),
        ("mt5", "FlaxMT5ForConditionalGeneration"),
        ("roberta", "FlaxRobertaForMaskedLM"),
        ("roberta-prelayernorm", "FlaxRobertaPreLayerNormForMaskedLM"),
        ("roformer", "FlaxRoFormerForMaskedLM"),
        ("t5", "FlaxT5ForConditionalGeneration"),
        ("wav2vec2", "FlaxWav2Vec2ForPreTraining"),
        ("whisper", "FlaxWhisperForConditionalGeneration"),
        ("xlm-roberta", "FlaxXLMRobertaForMaskedLM"),
    ]
)

__magic_name__     					=       OrderedDict(
    [
        # Model for Masked LM mapping
        ("albert", "FlaxAlbertForMaskedLM"),
        ("bart", "FlaxBartForConditionalGeneration"),
        ("bert", "FlaxBertForMaskedLM"),
        ("big_bird", "FlaxBigBirdForMaskedLM"),
        ("distilbert", "FlaxDistilBertForMaskedLM"),
        ("electra", "FlaxElectraForMaskedLM"),
        ("mbart", "FlaxMBartForConditionalGeneration"),
        ("roberta", "FlaxRobertaForMaskedLM"),
        ("roberta-prelayernorm", "FlaxRobertaPreLayerNormForMaskedLM"),
        ("roformer", "FlaxRoFormerForMaskedLM"),
        ("xlm-roberta", "FlaxXLMRobertaForMaskedLM"),
    ]
)

__magic_name__     					=       OrderedDict(
    [
        # Model for Seq2Seq Causal LM mapping
        ("bart", "FlaxBartForConditionalGeneration"),
        ("blenderbot", "FlaxBlenderbotForConditionalGeneration"),
        ("blenderbot-small", "FlaxBlenderbotSmallForConditionalGeneration"),
        ("encoder-decoder", "FlaxEncoderDecoderModel"),
        ("longt5", "FlaxLongT5ForConditionalGeneration"),
        ("marian", "FlaxMarianMTModel"),
        ("mbart", "FlaxMBartForConditionalGeneration"),
        ("mt5", "FlaxMT5ForConditionalGeneration"),
        ("pegasus", "FlaxPegasusForConditionalGeneration"),
        ("t5", "FlaxT5ForConditionalGeneration"),
    ]
)

__magic_name__     					=       OrderedDict(
    [
        # Model for Image-classsification
        ("beit", "FlaxBeitForImageClassification"),
        ("regnet", "FlaxRegNetForImageClassification"),
        ("resnet", "FlaxResNetForImageClassification"),
        ("vit", "FlaxViTForImageClassification"),
    ]
)

__magic_name__     					=       OrderedDict(
    [
        ("vision-encoder-decoder", "FlaxVisionEncoderDecoderModel"),
    ]
)

__magic_name__     					=       OrderedDict(
    [
        # Model for Causal LM mapping
        ("bart", "FlaxBartForCausalLM"),
        ("bert", "FlaxBertForCausalLM"),
        ("big_bird", "FlaxBigBirdForCausalLM"),
        ("electra", "FlaxElectraForCausalLM"),
        ("gpt-sw3", "FlaxGPT2LMHeadModel"),
        ("gpt2", "FlaxGPT2LMHeadModel"),
        ("gpt_neo", "FlaxGPTNeoForCausalLM"),
        ("gptj", "FlaxGPTJForCausalLM"),
        ("opt", "FlaxOPTForCausalLM"),
        ("roberta", "FlaxRobertaForCausalLM"),
        ("roberta-prelayernorm", "FlaxRobertaPreLayerNormForCausalLM"),
        ("xglm", "FlaxXGLMForCausalLM"),
        ("xlm-roberta", "FlaxXLMRobertaForCausalLM"),
    ]
)

__magic_name__     					=       OrderedDict(
    [
        # Model for Sequence Classification mapping
        ("albert", "FlaxAlbertForSequenceClassification"),
        ("bart", "FlaxBartForSequenceClassification"),
        ("bert", "FlaxBertForSequenceClassification"),
        ("big_bird", "FlaxBigBirdForSequenceClassification"),
        ("distilbert", "FlaxDistilBertForSequenceClassification"),
        ("electra", "FlaxElectraForSequenceClassification"),
        ("mbart", "FlaxMBartForSequenceClassification"),
        ("roberta", "FlaxRobertaForSequenceClassification"),
        ("roberta-prelayernorm", "FlaxRobertaPreLayerNormForSequenceClassification"),
        ("roformer", "FlaxRoFormerForSequenceClassification"),
        ("xlm-roberta", "FlaxXLMRobertaForSequenceClassification"),
    ]
)

__magic_name__     					=       OrderedDict(
    [
        # Model for Question Answering mapping
        ("albert", "FlaxAlbertForQuestionAnswering"),
        ("bart", "FlaxBartForQuestionAnswering"),
        ("bert", "FlaxBertForQuestionAnswering"),
        ("big_bird", "FlaxBigBirdForQuestionAnswering"),
        ("distilbert", "FlaxDistilBertForQuestionAnswering"),
        ("electra", "FlaxElectraForQuestionAnswering"),
        ("mbart", "FlaxMBartForQuestionAnswering"),
        ("roberta", "FlaxRobertaForQuestionAnswering"),
        ("roberta-prelayernorm", "FlaxRobertaPreLayerNormForQuestionAnswering"),
        ("roformer", "FlaxRoFormerForQuestionAnswering"),
        ("xlm-roberta", "FlaxXLMRobertaForQuestionAnswering"),
    ]
)

__magic_name__     					=       OrderedDict(
    [
        # Model for Token Classification mapping
        ("albert", "FlaxAlbertForTokenClassification"),
        ("bert", "FlaxBertForTokenClassification"),
        ("big_bird", "FlaxBigBirdForTokenClassification"),
        ("distilbert", "FlaxDistilBertForTokenClassification"),
        ("electra", "FlaxElectraForTokenClassification"),
        ("roberta", "FlaxRobertaForTokenClassification"),
        ("roberta-prelayernorm", "FlaxRobertaPreLayerNormForTokenClassification"),
        ("roformer", "FlaxRoFormerForTokenClassification"),
        ("xlm-roberta", "FlaxXLMRobertaForTokenClassification"),
    ]
)

__magic_name__     					=       OrderedDict(
    [
        # Model for Multiple Choice mapping
        ("albert", "FlaxAlbertForMultipleChoice"),
        ("bert", "FlaxBertForMultipleChoice"),
        ("big_bird", "FlaxBigBirdForMultipleChoice"),
        ("distilbert", "FlaxDistilBertForMultipleChoice"),
        ("electra", "FlaxElectraForMultipleChoice"),
        ("roberta", "FlaxRobertaForMultipleChoice"),
        ("roberta-prelayernorm", "FlaxRobertaPreLayerNormForMultipleChoice"),
        ("roformer", "FlaxRoFormerForMultipleChoice"),
        ("xlm-roberta", "FlaxXLMRobertaForMultipleChoice"),
    ]
)

__magic_name__     					=       OrderedDict(
    [
        ("bert", "FlaxBertForNextSentencePrediction"),
    ]
)

__magic_name__     					=       OrderedDict(
    [
        ("speech-encoder-decoder", "FlaxSpeechEncoderDecoderModel"),
        ("whisper", "FlaxWhisperForConditionalGeneration"),
    ]
)

__magic_name__     					=       OrderedDict(
    [
        ("whisper", "FlaxWhisperForAudioClassification"),
    ]
)

__magic_name__     					=       _LazyAutoMapping(CONFIG_MAPPING_NAMES, FLAX_MODEL_MAPPING_NAMES)
__magic_name__     					=       _LazyAutoMapping(CONFIG_MAPPING_NAMES, FLAX_MODEL_FOR_PRETRAINING_MAPPING_NAMES)
__magic_name__     					=       _LazyAutoMapping(CONFIG_MAPPING_NAMES, FLAX_MODEL_FOR_MASKED_LM_MAPPING_NAMES)
__magic_name__     					=       _LazyAutoMapping(
    CONFIG_MAPPING_NAMES, FLAX_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING_NAMES
)
__magic_name__     					=       _LazyAutoMapping(
    CONFIG_MAPPING_NAMES, FLAX_MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING_NAMES
)
__magic_name__     					=       _LazyAutoMapping(CONFIG_MAPPING_NAMES, FLAX_MODEL_FOR_VISION_2_SEQ_MAPPING_NAMES)
__magic_name__     					=       _LazyAutoMapping(CONFIG_MAPPING_NAMES, FLAX_MODEL_FOR_CAUSAL_LM_MAPPING_NAMES)
__magic_name__     					=       _LazyAutoMapping(
    CONFIG_MAPPING_NAMES, FLAX_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING_NAMES
)
__magic_name__     					=       _LazyAutoMapping(
    CONFIG_MAPPING_NAMES, FLAX_MODEL_FOR_QUESTION_ANSWERING_MAPPING_NAMES
)
__magic_name__     					=       _LazyAutoMapping(
    CONFIG_MAPPING_NAMES, FLAX_MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING_NAMES
)
__magic_name__     					=       _LazyAutoMapping(
    CONFIG_MAPPING_NAMES, FLAX_MODEL_FOR_MULTIPLE_CHOICE_MAPPING_NAMES
)
__magic_name__     					=       _LazyAutoMapping(
    CONFIG_MAPPING_NAMES, FLAX_MODEL_FOR_NEXT_SENTENCE_PREDICTION_MAPPING_NAMES
)
__magic_name__     					=       _LazyAutoMapping(
    CONFIG_MAPPING_NAMES, FLAX_MODEL_FOR_SPEECH_SEQ_2_SEQ_MAPPING_NAMES
)
__magic_name__     					=       _LazyAutoMapping(
    CONFIG_MAPPING_NAMES, FLAX_MODEL_FOR_AUDIO_CLASSIFICATION_MAPPING_NAMES
)

class SCREAMING_SNAKE_CASE  (     _BaseAutoModelClass						):







 """simple docstring"""


 a_							:						str	      =FLAX_MODEL_MAPPING
__magic_name__     					=       auto_class_update(FlaxAutoModel)

class SCREAMING_SNAKE_CASE  (     _BaseAutoModelClass						):







 """simple docstring"""


 a_							:						Optional[int]	      =FLAX_MODEL_FOR_PRETRAINING_MAPPING
__magic_name__     					=       auto_class_update(FlaxAutoModelForPreTraining, head_doc="pretraining")

class SCREAMING_SNAKE_CASE  (     _BaseAutoModelClass						):







 """simple docstring"""


 a_							:						str	      =FLAX_MODEL_FOR_CAUSAL_LM_MAPPING
__magic_name__     					=       auto_class_update(FlaxAutoModelForCausalLM, head_doc="causal language modeling")

class SCREAMING_SNAKE_CASE  (     _BaseAutoModelClass						):







 """simple docstring"""


 a_							:						str	      =FLAX_MODEL_FOR_MASKED_LM_MAPPING
__magic_name__     					=       auto_class_update(FlaxAutoModelForMaskedLM, head_doc="masked language modeling")

class SCREAMING_SNAKE_CASE  (     _BaseAutoModelClass						):







 """simple docstring"""


 a_							:						Optional[Any]	      =FLAX_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING
__magic_name__     					=       auto_class_update(
    FlaxAutoModelForSeqaSeqLM, head_doc="sequence-to-sequence language modeling", checkpoint_for_example="t5-base"
)

class SCREAMING_SNAKE_CASE  (     _BaseAutoModelClass						):







 """simple docstring"""


 a_							:						Any	      =FLAX_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING
__magic_name__     					=       auto_class_update(
    FlaxAutoModelForSequenceClassification, head_doc="sequence classification"
)

class SCREAMING_SNAKE_CASE  (     _BaseAutoModelClass						):







 """simple docstring"""


 a_							:						Optional[int]	      =FLAX_MODEL_FOR_QUESTION_ANSWERING_MAPPING
__magic_name__     					=       auto_class_update(FlaxAutoModelForQuestionAnswering, head_doc="question answering")

class SCREAMING_SNAKE_CASE  (     _BaseAutoModelClass						):







 """simple docstring"""


 a_							:						Optional[int]	      =FLAX_MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING
__magic_name__     					=       auto_class_update(
    FlaxAutoModelForTokenClassification, head_doc="token classification"
)

class SCREAMING_SNAKE_CASE  (     _BaseAutoModelClass						):







 """simple docstring"""


 a_							:						Optional[Any]	      =FLAX_MODEL_FOR_MULTIPLE_CHOICE_MAPPING
__magic_name__     					=       auto_class_update(FlaxAutoModelForMultipleChoice, head_doc="multiple choice")

class SCREAMING_SNAKE_CASE  (     _BaseAutoModelClass						):







 """simple docstring"""


 a_							:						List[str]	      =FLAX_MODEL_FOR_NEXT_SENTENCE_PREDICTION_MAPPING
__magic_name__     					=       auto_class_update(
    FlaxAutoModelForNextSentencePrediction, head_doc="next sentence prediction"
)

class SCREAMING_SNAKE_CASE  (     _BaseAutoModelClass						):







 """simple docstring"""


 a_							:						int	      =FLAX_MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING
__magic_name__     					=       auto_class_update(
    FlaxAutoModelForImageClassification, head_doc="image classification"
)

class SCREAMING_SNAKE_CASE  (     _BaseAutoModelClass						):







 """simple docstring"""


 a_							:						Optional[Any]	      =FLAX_MODEL_FOR_VISION_2_SEQ_MAPPING
__magic_name__     					=       auto_class_update(FlaxAutoModelForVisionaSeq, head_doc="vision-to-text modeling")

class SCREAMING_SNAKE_CASE  (     _BaseAutoModelClass						):







 """simple docstring"""


 a_							:						List[str]	      =FLAX_MODEL_FOR_SPEECH_SEQ_2_SEQ_MAPPING
__magic_name__     					=       auto_class_update(
    FlaxAutoModelForSpeechSeqaSeq, head_doc="sequence-to-sequence speech-to-text modeling"
)





import json
import os
import sys
import tempfile
import unittest
from pathlib import Path
from shutil import copyfile

from huggingface_hub import HfFolder, Repository, create_repo, delete_repo
from requests.exceptions import HTTPError

import transformers
from transformers import (
    CONFIG_MAPPING,
    FEATURE_EXTRACTOR_MAPPING,
    PROCESSOR_MAPPING,
    TOKENIZER_MAPPING,
    AutoConfig,
    AutoFeatureExtractor,
    AutoProcessor,
    AutoTokenizer,
    BertTokenizer,
    ProcessorMixin,
    WavaVecaConfig,
    WavaVecaFeatureExtractor,
    WavaVecaProcessor,
)
from transformers.testing_utils import TOKEN, USER, get_tests_dir, is_staging_test
from transformers.tokenization_utils import TOKENIZER_CONFIG_FILE
from transformers.utils import FEATURE_EXTRACTOR_NAME, is_tokenizers_available


sys.path.append(str(Path(__file__).parent.parent.parent.parent / """utils"""))

from test_module.custom_configuration import CustomConfig  # noqa E402
from test_module.custom_feature_extraction import CustomFeatureExtractor  # noqa E402
from test_module.custom_processing import CustomProcessor  # noqa E402
from test_module.custom_tokenization import CustomTokenizer  # noqa E402


_lowerCAmelCase  :      List[str]			 =get_tests_dir("""fixtures/dummy_feature_extractor_config.json""")
_lowerCAmelCase  :      List[str]			 =get_tests_dir("""fixtures/vocab.json""")
_lowerCAmelCase  :      Dict			 =get_tests_dir("""fixtures""")

class __UpperCamelCase     (				unittest.TestCase		):






					'''simple docstring'''



					__magic_name__        =   ["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]", "bla", "blou"]





					def    _UpperCAmelCase    (				self						):
						UpperCAmelCase__:	Union[str, Any]									= 0





					def    _UpperCAmelCase    (				self						):
						UpperCAmelCase__:	Optional[int]									= AutoProcessor.from_pretrained("facebook/wav2vec2-base-960h"						)
						self.assertIsInstance(lowerCamelCase__				,      lowerCamelCase__						)





					def    _UpperCAmelCase    (				self						):
						with tempfile.TemporaryDirectory() as tmpdirname:
							UpperCAmelCase__:	Any									= WavaVecaConfig()
							UpperCAmelCase__:	Tuple									= AutoProcessor.from_pretrained("facebook/wav2vec2-base-960h"						)

							# save in new folder
							model_config.save_pretrained(lowerCamelCase__						)
							processor.save_pretrained(lowerCamelCase__						)

							UpperCAmelCase__:	int									= AutoProcessor.from_pretrained(lowerCamelCase__						)

						self.assertIsInstance(lowerCamelCase__				,      lowerCamelCase__						)





					def    _UpperCAmelCase    (				self						):
						with tempfile.TemporaryDirectory() as tmpdirname:
							# copy relevant files
							copyfile(lowerCamelCase__				,      os.path.join(lowerCamelCase__				,      lowerCamelCase__						)						)
							copyfile(lowerCamelCase__				,      os.path.join(lowerCamelCase__				,      "vocab.json"						)						)

							UpperCAmelCase__:	Tuple									= AutoProcessor.from_pretrained(lowerCamelCase__						)

						self.assertIsInstance(lowerCamelCase__				,      lowerCamelCase__						)





					def    _UpperCAmelCase    (				self						):
						with tempfile.TemporaryDirectory() as tmpdirname:
							UpperCAmelCase__:	Optional[Any]									= WavaVecaFeatureExtractor()
							UpperCAmelCase__:	List[Any]									= AutoTokenizer.from_pretrained("facebook/wav2vec2-base-960h"						)

							UpperCAmelCase__:	Dict									= WavaVecaProcessor(lowerCamelCase__				,      lowerCamelCase__						)

							# save in new folder
							processor.save_pretrained(lowerCamelCase__						)

							# drop `processor_class` in tokenizer
							with open(os.path.join(lowerCamelCase__				,      lowerCamelCase__						)				,      "r"						) as f:
								UpperCAmelCase__:	Optional[int]									= json.load(lowerCamelCase__						)
								config_dict.pop("processor_class"						)

							with open(os.path.join(lowerCamelCase__				,      lowerCamelCase__						)				,      "w"						) as f:
								f.write(json.dumps(lowerCamelCase__						)						)

							UpperCAmelCase__:	Any									= AutoProcessor.from_pretrained(lowerCamelCase__						)

						self.assertIsInstance(lowerCamelCase__				,      lowerCamelCase__						)





					def    _UpperCAmelCase    (				self						):
						with tempfile.TemporaryDirectory() as tmpdirname:
							UpperCAmelCase__:	str									= WavaVecaFeatureExtractor()
							UpperCAmelCase__:	Tuple									= AutoTokenizer.from_pretrained("facebook/wav2vec2-base-960h"						)

							UpperCAmelCase__:	List[Any]									= WavaVecaProcessor(lowerCamelCase__				,      lowerCamelCase__						)

							# save in new folder
							processor.save_pretrained(lowerCamelCase__						)

							# drop `processor_class` in feature extractor
							with open(os.path.join(lowerCamelCase__				,      lowerCamelCase__						)				,      "r"						) as f:
								UpperCAmelCase__:	str									= json.load(lowerCamelCase__						)
								config_dict.pop("processor_class"						)

							with open(os.path.join(lowerCamelCase__				,      lowerCamelCase__						)				,      "w"						) as f:
								f.write(json.dumps(lowerCamelCase__						)						)

							UpperCAmelCase__:	Tuple									= AutoProcessor.from_pretrained(lowerCamelCase__						)

						self.assertIsInstance(lowerCamelCase__				,      lowerCamelCase__						)





					def    _UpperCAmelCase    (				self						):
						with tempfile.TemporaryDirectory() as tmpdirname:
							UpperCAmelCase__:	Union[str, Any]									= WavaVecaConfig(processor_class="Wav2Vec2Processor"						)
							model_config.save_pretrained(lowerCamelCase__						)
							# copy relevant files
							copyfile(lowerCamelCase__				,      os.path.join(lowerCamelCase__				,      "vocab.json"						)						)
							# create emtpy sample processor
							with open(os.path.join(lowerCamelCase__				,      lowerCamelCase__						)				,      "w"						) as f:
								f.write("{}"						)

							UpperCAmelCase__:	Union[str, Any]									= AutoProcessor.from_pretrained(lowerCamelCase__						)

						self.assertIsInstance(lowerCamelCase__				,      lowerCamelCase__						)





					def    _UpperCAmelCase    (				self						):
						# If remote code is not set, we will time out when asking whether to load the model.
						with self.assertRaises(lowerCamelCase__						):
							UpperCAmelCase__:	Optional[Any]									= AutoProcessor.from_pretrained("hf-internal-testing/test_dynamic_processor"						)
						# If remote code is disabled, we can't load this config.
						with self.assertRaises(lowerCamelCase__						):
							UpperCAmelCase__:	Optional[int]									= AutoProcessor.from_pretrained(
							    "hf-internal-testing/test_dynamic_processor"				,      trust_remote_code=lowerCamelCase__						)

						UpperCAmelCase__:	Union[str, Any]									= AutoProcessor.from_pretrained("hf-internal-testing/test_dynamic_processor"				,      trust_remote_code=lowerCamelCase__						)
						self.assertTrue(processor.special_attribute_present						)
						self.assertEqual(processor.__class__.__name__				,      "NewProcessor"						)

						UpperCAmelCase__:	Dict									= processor.feature_extractor
						self.assertTrue(feature_extractor.special_attribute_present						)
						self.assertEqual(feature_extractor.__class__.__name__				,      "NewFeatureExtractor"						)

						UpperCAmelCase__:	Any									= processor.tokenizer
						self.assertTrue(tokenizer.special_attribute_present						)
						if is_tokenizers_available():
							self.assertEqual(tokenizer.__class__.__name__				,      "NewTokenizerFast"						)

							# Test we can also load the slow version
							UpperCAmelCase__:	Dict									= AutoProcessor.from_pretrained(
							    "hf-internal-testing/test_dynamic_processor"				,      trust_remote_code=lowerCamelCase__				,      use_fast=lowerCamelCase__						)
							UpperCAmelCase__:	str									= new_processor.tokenizer
							self.assertTrue(new_tokenizer.special_attribute_present						)
							self.assertEqual(new_tokenizer.__class__.__name__				,      "NewTokenizer"						)
						else:
							self.assertEqual(tokenizer.__class__.__name__				,      "NewTokenizer"						)





					def    _UpperCAmelCase    (				self						):
						try:
							AutoConfig.register("custom"				,      lowerCamelCase__						)
							AutoFeatureExtractor.register(lowerCamelCase__				,      lowerCamelCase__						)
							AutoTokenizer.register(lowerCamelCase__				,      slow_tokenizer_class=lowerCamelCase__						)
							AutoProcessor.register(lowerCamelCase__				,      lowerCamelCase__						)
							# Trying to register something existing in the Transformers library will raise an error
							with self.assertRaises(lowerCamelCase__						):
								AutoProcessor.register(lowerCamelCase__				,      lowerCamelCase__						)

							# Now that the config is registered, it can be used as any other config with the auto-API
							UpperCAmelCase__:	Dict									= CustomFeatureExtractor.from_pretrained(lowerCamelCase__						)

							with tempfile.TemporaryDirectory() as tmp_dir:
								UpperCAmelCase__:	Tuple									= os.path.join(lowerCamelCase__				,      "vocab.txt"						)
								with open(lowerCamelCase__				,      "w"				,      encoding="utf-8"						) as vocab_writer:
									vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]						)						)
								UpperCAmelCase__:	str									= CustomTokenizer(lowerCamelCase__						)

							UpperCAmelCase__:	Optional[int]									= CustomProcessor(lowerCamelCase__				,      lowerCamelCase__						)

							with tempfile.TemporaryDirectory() as tmp_dir:
								processor.save_pretrained(lowerCamelCase__						)
								UpperCAmelCase__:	Tuple									= AutoProcessor.from_pretrained(lowerCamelCase__						)
								self.assertIsInstance(lowerCamelCase__				,      lowerCamelCase__						)

						finally:
							if "custom" in CONFIG_MAPPING._extra_content:
								del CONFIG_MAPPING._extra_content["custom"]
							if CustomConfig in FEATURE_EXTRACTOR_MAPPING._extra_content:
								del FEATURE_EXTRACTOR_MAPPING._extra_content[CustomConfig]
							if CustomConfig in TOKENIZER_MAPPING._extra_content:
								del TOKENIZER_MAPPING._extra_content[CustomConfig]
							if CustomConfig in PROCESSOR_MAPPING._extra_content:
								del PROCESSOR_MAPPING._extra_content[CustomConfig]





					def    _UpperCAmelCase    (				self						):

						class __UpperCamelCase     (				_a		):






											'''simple docstring'''



											__magic_name__        =   False

						class __UpperCamelCase     (				_a		):






											'''simple docstring'''



											__magic_name__        =   False

						class __UpperCamelCase     (				_a		):






											'''simple docstring'''



											__magic_name__        =   "AutoFeatureExtractor"
											__magic_name__        =   "AutoTokenizer"
											__magic_name__        =   False

						try:
							AutoConfig.register("custom"				,      lowerCamelCase__						)
							AutoFeatureExtractor.register(lowerCamelCase__				,      lowerCamelCase__						)
							AutoTokenizer.register(lowerCamelCase__				,      slow_tokenizer_class=lowerCamelCase__						)
							AutoProcessor.register(lowerCamelCase__				,      lowerCamelCase__						)
							# If remote code is not set, the default is to use local classes.
							UpperCAmelCase__:	Any									= AutoProcessor.from_pretrained("hf-internal-testing/test_dynamic_processor"						)
							self.assertEqual(processor.__class__.__name__				,      "NewProcessor"						)
							self.assertFalse(processor.special_attribute_present						)
							self.assertFalse(processor.feature_extractor.special_attribute_present						)
							self.assertFalse(processor.tokenizer.special_attribute_present						)

							# If remote code is disabled, we load the local ones.
							UpperCAmelCase__:	Union[str, Any]									= AutoProcessor.from_pretrained(
							    "hf-internal-testing/test_dynamic_processor"				,      trust_remote_code=lowerCamelCase__						)
							self.assertEqual(processor.__class__.__name__				,      "NewProcessor"						)
							self.assertFalse(processor.special_attribute_present						)
							self.assertFalse(processor.feature_extractor.special_attribute_present						)
							self.assertFalse(processor.tokenizer.special_attribute_present						)

							# If remote is enabled, we load from the Hub.
							UpperCAmelCase__:	Tuple									= AutoProcessor.from_pretrained(
							    "hf-internal-testing/test_dynamic_processor"				,      trust_remote_code=lowerCamelCase__						)
							self.assertEqual(processor.__class__.__name__				,      "NewProcessor"						)
							self.assertTrue(processor.special_attribute_present						)
							self.assertTrue(processor.feature_extractor.special_attribute_present						)
							self.assertTrue(processor.tokenizer.special_attribute_present						)

						finally:
							if "custom" in CONFIG_MAPPING._extra_content:
								del CONFIG_MAPPING._extra_content["custom"]
							if CustomConfig in FEATURE_EXTRACTOR_MAPPING._extra_content:
								del FEATURE_EXTRACTOR_MAPPING._extra_content[CustomConfig]
							if CustomConfig in TOKENIZER_MAPPING._extra_content:
								del TOKENIZER_MAPPING._extra_content[CustomConfig]
							if CustomConfig in PROCESSOR_MAPPING._extra_content:
								del PROCESSOR_MAPPING._extra_content[CustomConfig]





					def    _UpperCAmelCase    (				self						):
						UpperCAmelCase__:	Union[str, Any]									= AutoProcessor.from_pretrained("hf-internal-testing/tiny-random-bert"						)
						self.assertEqual(processor.__class__.__name__				,      "BertTokenizerFast"						)





					def    _UpperCAmelCase    (				self						):
						UpperCAmelCase__:	Tuple									= AutoProcessor.from_pretrained("hf-internal-testing/tiny-random-convnext"						)
						self.assertEqual(processor.__class__.__name__				,      "ConvNextImageProcessor"						)

@is_staging_test
class __UpperCamelCase     (				unittest.TestCase		):






					'''simple docstring'''



					__magic_name__        =   ["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]", "bla", "blou"]





					@classmethod
					def    _UpperCAmelCase    (				cls						):
						UpperCAmelCase__:	Dict									= TOKEN
						HfFolder.save_token(lowerCamelCase__						)





					@classmethod
					def    _UpperCAmelCase    (				cls						):
						try:
							delete_repo(token=cls._token				,      repo_id="test-processor"						)
						except HTTPError:
							pass

						try:
							delete_repo(token=cls._token				,      repo_id="valid_org/test-processor-org"						)
						except HTTPError:
							pass

						try:
							delete_repo(token=cls._token				,      repo_id="test-dynamic-processor"						)
						except HTTPError:
							pass





					def    _UpperCAmelCase    (				self						):
						UpperCAmelCase__:	Optional[Any]									= WavaVecaProcessor.from_pretrained(lowerCamelCase__						)
						with tempfile.TemporaryDirectory() as tmp_dir:
							processor.save_pretrained(
							    os.path.join(lowerCamelCase__				,      "test-processor"						)				,      push_to_hub=lowerCamelCase__				,      use_auth_token=self._token						)

							UpperCAmelCase__:	Union[str, Any]									= WavaVecaProcessor.from_pretrained(F"{USER}/test-processor"						)
							for k, v in processor.feature_extractor.__dict__.items():
								self.assertEqual(lowerCamelCase__				,      getattr(new_processor.feature_extractor				,      lowerCamelCase__						)						)
							self.assertDictEqual(new_processor.tokenizer.get_vocab()				,      processor.tokenizer.get_vocab()						)





					def    _UpperCAmelCase    (				self						):
						UpperCAmelCase__:	Dict									= WavaVecaProcessor.from_pretrained(lowerCamelCase__						)

						with tempfile.TemporaryDirectory() as tmp_dir:
							processor.save_pretrained(
							    os.path.join(lowerCamelCase__				,      "test-processor-org"						)				,      push_to_hub=lowerCamelCase__				,      use_auth_token=self._token				,      organization="valid_org"				,      )

							UpperCAmelCase__:	Optional[int]									= WavaVecaProcessor.from_pretrained("valid_org/test-processor-org"						)
							for k, v in processor.feature_extractor.__dict__.items():
								self.assertEqual(lowerCamelCase__				,      getattr(new_processor.feature_extractor				,      lowerCamelCase__						)						)
							self.assertDictEqual(new_processor.tokenizer.get_vocab()				,      processor.tokenizer.get_vocab()						)





					def    _UpperCAmelCase    (				self						):
						CustomFeatureExtractor.register_for_auto_class()
						CustomTokenizer.register_for_auto_class()
						CustomProcessor.register_for_auto_class()

						UpperCAmelCase__:	List[str]									= CustomFeatureExtractor.from_pretrained(lowerCamelCase__						)

						with tempfile.TemporaryDirectory() as tmp_dir:
							UpperCAmelCase__:	Optional[int]									= os.path.join(lowerCamelCase__				,      "vocab.txt"						)
							with open(lowerCamelCase__				,      "w"				,      encoding="utf-8"						) as vocab_writer:
								vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]						)						)
							UpperCAmelCase__:	Optional[Any]									= CustomTokenizer(lowerCamelCase__						)

						UpperCAmelCase__:	str									= CustomProcessor(lowerCamelCase__				,      lowerCamelCase__						)

						with tempfile.TemporaryDirectory() as tmp_dir:
							create_repo(F"{USER}/test-dynamic-processor"				,      token=self._token						)
							UpperCAmelCase__:	int									= Repository(lowerCamelCase__				,      clone_from=F"{USER}/test-dynamic-processor"				,      token=self._token						)
							processor.save_pretrained(lowerCamelCase__						)

							# This has added the proper auto_map field to the feature extractor config
							self.assertDictEqual(
							    processor.feature_extractor.auto_map				,      {
							        "AutoFeatureExtractor": "custom_feature_extraction.CustomFeatureExtractor",
							        "AutoProcessor": "custom_processing.CustomProcessor",
							    }				,      )

							# This has added the proper auto_map field to the tokenizer config
							with open(os.path.join(lowerCamelCase__				,      "tokenizer_config.json"						)						) as f:
								UpperCAmelCase__:	Tuple									= json.load(lowerCamelCase__						)
							self.assertDictEqual(
							    tokenizer_config["auto_map"]				,      {
							        "AutoTokenizer": ["custom_tokenization.CustomTokenizer", None],
							        "AutoProcessor": "custom_processing.CustomProcessor",
							    }				,      )

							# The code has been copied from fixtures
							self.assertTrue(os.path.isfile(os.path.join(lowerCamelCase__				,      "custom_feature_extraction.py"						)						)						)
							self.assertTrue(os.path.isfile(os.path.join(lowerCamelCase__				,      "custom_tokenization.py"						)						)						)
							self.assertTrue(os.path.isfile(os.path.join(lowerCamelCase__				,      "custom_processing.py"						)						)						)

							repo.push_to_hub()

						UpperCAmelCase__:	Union[str, Any]									= AutoProcessor.from_pretrained(F"{USER}/test-dynamic-processor"				,      trust_remote_code=lowerCamelCase__						)
						# Can't make an isinstance check because the new_processor is from the CustomProcessor class of a dynamic module
						self.assertEqual(new_processor.__class__.__name__				,      "CustomProcessor"						)