





'''simple docstring'''



import importlib
import inspect
import json
import os
from collections import OrderedDict

# Build the list of all feature extractors
from ...configuration_utils import PretrainedConfig
from ...dynamic_module_utils import get_class_from_dynamic_module, resolve_trust_remote_code
from ...feature_extraction_utils import FeatureExtractionMixin
from ...image_processing_utils import ImageProcessingMixin
from ...tokenization_utils import TOKENIZER_CONFIG_FILE
from ...utils import FEATURE_EXTRACTOR_NAME, get_file_from_repo, logging
from .auto_factory import _LazyAutoMapping
from .configuration_auto import (
    CONFIG_MAPPING_NAMES,
    AutoConfig,
    model_type_to_module_name,
    replace_list_option_in_docstrings,
)
from .feature_extraction_auto import AutoFeatureExtractor
from .image_processing_auto import AutoImageProcessor
from .tokenization_auto import AutoTokenizer


_SCREAMING_SNAKE_CASE    =			logging.get_logger(__name__)

_SCREAMING_SNAKE_CASE    =			OrderedDict(
    [
        ('''align''', '''AlignProcessor'''),
        ('''altclip''', '''AltCLIPProcessor'''),
        ('''bark''', '''BarkProcessor'''),
        ('''blip''', '''BlipProcessor'''),
        ('''blip-2''', '''Blip2Processor'''),
        ('''bridgetower''', '''BridgeTowerProcessor'''),
        ('''chinese_clip''', '''ChineseCLIPProcessor'''),
        ('''clap''', '''ClapProcessor'''),
        ('''clip''', '''CLIPProcessor'''),
        ('''clipseg''', '''CLIPSegProcessor'''),
        ('''flava''', '''FlavaProcessor'''),
        ('''git''', '''GitProcessor'''),
        ('''groupvit''', '''CLIPProcessor'''),
        ('''hubert''', '''Wav2Vec2Processor'''),
        ('''instructblip''', '''InstructBlipProcessor'''),
        ('''layoutlmv2''', '''LayoutLMv2Processor'''),
        ('''layoutlmv3''', '''LayoutLMv3Processor'''),
        ('''markuplm''', '''MarkupLMProcessor'''),
        ('''mctct''', '''MCTCTProcessor'''),
        ('''mgp-str''', '''MgpstrProcessor'''),
        ('''oneformer''', '''OneFormerProcessor'''),
        ('''owlvit''', '''OwlViTProcessor'''),
        ('''pix2struct''', '''Pix2StructProcessor'''),
        ('''sam''', '''SamProcessor'''),
        ('''sew''', '''Wav2Vec2Processor'''),
        ('''sew-d''', '''Wav2Vec2Processor'''),
        ('''speech_to_text''', '''Speech2TextProcessor'''),
        ('''speech_to_text_2''', '''Speech2Text2Processor'''),
        ('''speecht5''', '''SpeechT5Processor'''),
        ('''trocr''', '''TrOCRProcessor'''),
        ('''tvlt''', '''TvltProcessor'''),
        ('''unispeech''', '''Wav2Vec2Processor'''),
        ('''unispeech-sat''', '''Wav2Vec2Processor'''),
        ('''vilt''', '''ViltProcessor'''),
        ('''vision-text-dual-encoder''', '''VisionTextDualEncoderProcessor'''),
        ('''wav2vec2''', '''Wav2Vec2Processor'''),
        ('''wav2vec2-conformer''', '''Wav2Vec2Processor'''),
        ('''wavlm''', '''Wav2Vec2Processor'''),
        ('''whisper''', '''WhisperProcessor'''),
        ('''xclip''', '''XCLIPProcessor'''),
    ]
)

_SCREAMING_SNAKE_CASE    =			_LazyAutoMapping(CONFIG_MAPPING_NAMES, PROCESSOR_MAPPING_NAMES)




def       _lowerCAmelCase  ( lowerCamelCase_	:     str ):
			for module_name, processors in PROCESSOR_MAPPING_NAMES.items():
						if class_name in processors:
									__lowercase   =							model_type_to_module_name(lowerCamelCase_ )

									__lowercase   =							importlib.import_module(f".{module_name}"						,				'''transformers.models''' )
									try:
												return getattr(lowerCamelCase_						,				lowerCamelCase_ )
									except AttributeError:
												continue

			for processor in PROCESSOR_MAPPING._extra_content.values():
						if getattr(lowerCamelCase_						,				'''__name__'''						,				lowerCamelCase_ ) == class_name:
									return processor

    # We did not fine the class, but maybe it's because a dep is missing. In that case, the class will be in the main
    # init and we return the proper dummy to get an appropriate error message.
			__lowercase   =							importlib.import_module('''transformers''' )
			if hasattr(lowerCamelCase_						,				lowerCamelCase_ ):
						return getattr(lowerCamelCase_						,				lowerCamelCase_ )

			return None






class 							__lowercase				:







	'''simple docstring'''

	def __init__(self       )							->							List[Any]:







				'''simple docstring'''





				raise EnvironmentError(
				    '''AutoProcessor is designed to be instantiated '''
				    '''using the `AutoProcessor.from_pretrained(pretrained_model_name_or_path)` method.'''       )


	@classmethod
	@replace_list_option_in_docstrings(_lowerCamelCase       )
	def   _UpperCAmelCase   (cls   ,_lowerCamelCase   ,**_lowerCamelCase       )							->							Optional[Any]:







				'''simple docstring'''





				__lowercase   =							kwargs.pop('''config'''   ,_lowerCamelCase       )
				__lowercase   =							kwargs.pop('''trust_remote_code'''   ,_lowerCamelCase       )
				__lowercase   =							True

				__lowercase   =							None
				__lowercase   =							None

				# First, let's see if we have a preprocessor config.
				# Filter the kwargs for `get_file_from_repo`.
				__lowercase   =							{
				    key: kwargs[key] for key in inspect.signature(_lowerCamelCase       ).parameters.keys() if key in kwargs
				}
				# Let's start by checking whether the processor class is saved in an image processor
				__lowercase   =							get_file_from_repo(
				    _lowerCamelCase   ,_lowerCamelCase   ,**_lowerCamelCase       )
				if preprocessor_config_file is not None:
							__lowercase  ,						__lowercase   =							ImageProcessingMixin.get_image_processor_dict(_lowerCamelCase   ,**_lowerCamelCase       )
							__lowercase   =							config_dict.get('''processor_class'''   ,_lowerCamelCase       )
							if "AutoProcessor" in config_dict.get('''auto_map'''   ,{}       ):
										__lowercase   =							config_dict['''auto_map''']['''AutoProcessor''']

        # If not found, let's check whether the processor class is saved in a feature extractor config
				if preprocessor_config_file is not None and processor_class is None:
							__lowercase  ,						__lowercase   =							FeatureExtractionMixin.get_feature_extractor_dict(_lowerCamelCase   ,**_lowerCamelCase       )
							__lowercase   =							config_dict.get('''processor_class'''   ,_lowerCamelCase       )
							if "AutoProcessor" in config_dict.get('''auto_map'''   ,{}       ):
										__lowercase   =							config_dict['''auto_map''']['''AutoProcessor''']

				if processor_class is None:
							# Next, let's check whether the processor class is saved in a tokenizer
							__lowercase   =							get_file_from_repo(
							    _lowerCamelCase   ,_lowerCamelCase   ,**_lowerCamelCase       )
							if tokenizer_config_file is not None:
										with open(_lowerCamelCase   ,encoding='''utf-8'''       ) as reader:
													__lowercase   =							json.load(_lowerCamelCase       )

										__lowercase   =							config_dict.get('''processor_class'''   ,_lowerCamelCase       )
										if "AutoProcessor" in config_dict.get('''auto_map'''   ,{}       ):
													__lowercase   =							config_dict['''auto_map''']['''AutoProcessor''']

				if processor_class is None:
							# Otherwise, load config, if it can be loaded.
							if not isinstance(_lowerCamelCase   ,_lowerCamelCase       ):
										__lowercase   =							AutoConfig.from_pretrained(
										    _lowerCamelCase   ,trust_remote_code=_lowerCamelCase   ,**_lowerCamelCase       )

							# And check if the config contains the processor class.
							__lowercase   =							getattr(_lowerCamelCase   ,'''processor_class'''   ,_lowerCamelCase       )
							if hasattr(_lowerCamelCase   ,'''auto_map'''       ) and "AutoProcessor" in config.auto_map:
										__lowercase   =							config.auto_map['''AutoProcessor''']

				if processor_class is not None:
							__lowercase   =							processor_class_from_name(_lowerCamelCase       )

				__lowercase   =							processor_auto_map is not None
				__lowercase   =							processor_class is not None or type(_lowerCamelCase       ) in PROCESSOR_MAPPING
				__lowercase   =							resolve_trust_remote_code(
				    _lowerCamelCase   ,_lowerCamelCase   ,_lowerCamelCase   ,_lowerCamelCase       )

				if has_remote_code and trust_remote_code:
							__lowercase   =							get_class_from_dynamic_module(
							    _lowerCamelCase   ,_lowerCamelCase   ,**_lowerCamelCase       )
							__lowercase   =							kwargs.pop('''code_revision'''   ,_lowerCamelCase       )
							if os.path.isdir(_lowerCamelCase       ):
										processor_class.register_for_auto_class()
							return processor_class.from_pretrained(
							    _lowerCamelCase   ,trust_remote_code=_lowerCamelCase   ,**_lowerCamelCase       )
				elif processor_class is not None:
							return processor_class.from_pretrained(
							    _lowerCamelCase   ,trust_remote_code=_lowerCamelCase   ,**_lowerCamelCase       )
				# Last try: we use the PROCESSOR_MAPPING.
				elif type(_lowerCamelCase       ) in PROCESSOR_MAPPING:
							return PROCESSOR_MAPPING[type(_lowerCamelCase       )].from_pretrained(_lowerCamelCase   ,**_lowerCamelCase       )

				# At this stage, there doesn't seem to be a `Processor` class available for this model, so let's try a
				# tokenizer.
				try:
							return AutoTokenizer.from_pretrained(
							    _lowerCamelCase   ,trust_remote_code=_lowerCamelCase   ,**_lowerCamelCase       )
				except Exception:
							try:
										return AutoImageProcessor.from_pretrained(
										    _lowerCamelCase   ,trust_remote_code=_lowerCamelCase   ,**_lowerCamelCase       )
							except Exception:
										pass

							try:
										return AutoFeatureExtractor.from_pretrained(
										    _lowerCamelCase   ,trust_remote_code=_lowerCamelCase   ,**_lowerCamelCase       )
							except Exception:
										pass

				raise ValueError(
				    f"Unrecognized processing class in {pretrained_model_name_or_path}. Can't instantiate a processor, a "
				    '''tokenizer, an image processor or a feature extractor for this model. Make sure the repository contains'''
				    '''the files of at least one of those processing classes.'''       )


	@staticmethod
	def   _UpperCAmelCase   (_lowerCamelCase   ,_lowerCamelCase       )							->							int:







				'''simple docstring'''





				PROCESSOR_MAPPING.register(_lowerCamelCase   ,_lowerCamelCase       )












import itertools
import os
import subprocess
from os.path import dirname

from parameterized import parameterized

from tests.trainer.test_trainer import TrainerIntegrationCommon  # noqa
from transformers import is_torch_available
from transformers.testing_utils import (
    TestCasePlus,
    execute_subprocess_async,
    get_gpu_count,
    get_tests_dir,
    require_deepspeed,
    require_torch_gpu,
    slow,
)
from transformers.trainer_utils import set_seed


if is_torch_available():
  from tests.trainer.test_trainer import (  # noqa
      RegressionModelConfig,
      RegressionPreTrainedModel,
      get_regression_trainer,
  )


set_seed(42)

_snake_case				= get_tests_dir("""fixtures""")
_snake_case				= os.path.join(dirname(get_tests_dir()))
_snake_case				= dirname(os.path.abspath(__file__))

# default torch.distributed port
_snake_case				= """10999"""

_snake_case				= """t5-small"""

# *** Working Models ***
_snake_case				= """hf-internal-testing/tiny-albert"""
_snake_case				= """sshleifer/bart-tiny-random"""
_snake_case				= """hf-internal-testing/tiny-bert"""
_snake_case				= """hf-internal-testing/tiny-random-bigbird_pegasus"""
_snake_case				= """hf-internal-testing/tiny-random-big_bird"""
_snake_case				= """hf-internal-testing/tiny-random-blenderbot"""
_snake_case				= """bigscience/bigscience-small-testing"""
_snake_case				= """hf-internal-testing/tiny-random-deberta"""
_snake_case				= """hf-internal-testing/tiny-random-deberta-v2"""
_snake_case				= """sshleifer/tiny-distilbert-base-cased"""
_snake_case				= """hf-internal-testing/tiny-electra"""
_snake_case				= """hf-internal-testing/tiny-random-flaubert"""
_snake_case				= """stas/tiny-wmt19-en-de"""
_snake_case				= """hf-internal-testing/tiny-random-funnel"""
_snake_case				= """sshleifer/tiny-gpt2"""
_snake_case				= """hf-internal-testing/tiny-random-gptj"""
_snake_case				= """hf-internal-testing/tiny-random-gpt_neo"""
_snake_case				= """hf-internal-testing/tiny-layoutlm"""
_snake_case				= """hf-internal-testing/tiny-random-led"""
_snake_case				= """hf-internal-testing/tiny-random-longformer"""
_snake_case				= """stas/tiny-m2m_100"""  # hf tiny model is unsuitable
_snake_case				= """sshleifer/tiny-marian-en-de"""
_snake_case				= """sshleifer/tiny-mbart"""
_snake_case				= """hf-internal-testing/tiny-random-mobilebert"""
_snake_case				= """hf-internal-testing/tiny-random-mpnet"""
_snake_case				= """stas/pegasus-cnn_dailymail-tiny-random"""
_snake_case				= """hf-internal-testing/tiny-random-prophetnet"""
_snake_case				= """sshleifer/tiny-distilroberta-base"""
_snake_case				= """hf-internal-testing/tiny-random-squeezebert"""
_snake_case				= """patrickvonplaten/t5-tiny-random"""
_snake_case				= """hf-internal-testing/tiny-random-t5-v1.1"""
_snake_case				= """hf-internal-testing/tiny-random-vit"""
_snake_case				= """hf-internal-testing/tiny-xlm-roberta"""
_snake_case				= """sshleifer/tiny-xlnet-base-cased"""


# *** To Fix ***


# *** tiny model issues ***
# missing model files:
_snake_case				= """hf-internal-testing/tiny-random-mt5"""
_snake_case				= """hf-internal-testing/tiny-random-camembert"""
_snake_case				= """hf-internal-testing/tiny-random-openai-gpt"""

# missing tokenizer files
_snake_case				= """hf-internal-testing/tiny-random-convbert"""
_snake_case				= """hf-internal-testing/tiny-random-layoutlmv2"""
_snake_case				= """hf-internal-testing/tiny-random-hubert"""

# issues with tokenizer
_snake_case				= """hf-internal-testing/tiny-random-ctrl"""
_snake_case				= """hf-internal-testing/tiny-random-transfo-xl"""  # same as ctrl

# other issues with tiny models
_snake_case				= """hf-internal-testing/tiny-random-ibert"""  # multiple issues with either mlm/qa/clas
_snake_case				= """hf-internal-testing/tiny-random-reformer"""  # multiple issues with either mlm/qa/clas

# *** Lacking official examples to test with ***
# or not working with examples
_snake_case				= """hf-internal-testing/tiny-random-dpr"""
# - "dpr"  examples/research_projects/rag-end2end-retriever/
_snake_case				= """hf-internal-testing/tiny-random-rag"""
# - "rag" research_projects
_snake_case				= """"""
# - "luke" Entities classes - no plan to make such example
_snake_case				= """hf-internal-testing/tiny-random-lxmert"""
# - "lxmert" doesn't work with run_qa.py
_snake_case				= """hf-internal-testing/tiny-random-clip"""
# - "clip" nothing under pytorch examples - XXX: Suraj is working on adding some - check by end of Sep
_snake_case				= """hf-internal-testing/tiny-random-speech_to_text"""
# - "speech_to_text", nothing under pytorch examples


# *** Reactive mode ***
# models with low usage, unstable API, things about to change - do nothing about the following until someone runs into a problem
_snake_case				= """hf-internal-testing/tiny-random-tapas"""


def _A   (    __magic_name__=False       ):
   # 1. explicitly set --num_nodes=1 just in case these tests end up run on a multi-node setup
   # - it won't be able to handle that
   # 2. for now testing with just 2 gpus max (since some quality tests may give different
   # results with mode gpus because we use very little data)
   lowercase__    =  min(2  ,					get_gpu_count()       ) if distributed else 1
   lowercase__    =  os.environ.get("DS_TEST_PORT"  ,					__magic_name__       )
   return f'''deepspeed --num_nodes 1 --num_gpus {num_gpus} --master_port {master_port}'''.split()


def _A   (    ):
   lowercase__    =  f'''{FIXTURE_DIRECTORY}/tests_samples'''
   lowercase__    =  f'''{data_dir_samples}/wmt_en_ro'''
   lowercase__    =  f'''{data_dir_samples}/xsum'''
   lowercase__    =  "\n        --do_train\n        --max_train_samples 4\n        --per_device_train_batch_size 2\n        --num_train_epochs 1\n        --fp16\n        --report_to none\n        --overwrite_output_dir\n        ".split()

   # try to cover as many models as possible once (it's enough to run on one task per model)
   # but need a tiny model for each
   #
   # should have "{model_type.upper()}_TINY" corresponding vars defined, e.g., T5_TINY, etc.
   lowercase__    =  {
       "trans": [
           "bart",
           "fsmt",
           "m2m_100",
           "marian",
           "mbart",
           "t5",
           "t5_v1",
           # "mt5", missing model files
       ],
       "sum": [
           "pegasus",
       ],
       "clm": [
           "big_bird",
           "bigbird_pegasus",
           "blenderbot",
           "bloom",
           "gpt2",
           "gpt_neo",
           "gptj",
           "xlm-roberta",
           "prophetnet",
           # "camembert", missing model files
       ],
       "mlm": [
           "albert",
           "deberta",
           "deberta-v2",
           "distilbert",
           "electra",
           "flaubert",
           "funnel",
           "layoutlm",
           # "reformer", # multiple issues with either mlm/qa/clas
       ],
       "qa": [
           "led",
           "longformer",
           "mobilebert",
           "mpnet",
           "roberta",
           "squeezebert",
           # "convbert", # missing tokenizer files
           # "layoutlmv2", missing model files
       ],
       "clas": [
           "bert",
           "xlnet",
           # "hubert", # missing tokenizer files
           # "ibert", # multiple issues with either mlm/qa/clas
           # "transfo-xl", # tokenizer issues as ctrl
           # "ctrl", # tokenizer issues
           # "openai-gpt", missing model files
           # "tapas", multiple issues
       ],
       "img_clas": [
           "vit",
       ],
   }

   lowercase__    =  f'''{ROOT_DIRECTORY}/examples/pytorch'''

   lowercase__    =  {
       "trans": f'''
        {scripts_dir}/translation/run_translation.py
        --train_file {data_dir_wmt}/train.json
        --source_lang en
        --target_lang ro
        ''',
       "sum": f'''
        {scripts_dir}/summarization/run_summarization.py
        --train_file {data_dir_xsum}/sample.json
        --max_source_length 12
        --max_target_length 12
        --lang en
        ''',
       "clm": f'''
        {scripts_dir}/language-modeling/run_clm.py
        --train_file {FIXTURE_DIRECTORY}/sample_text.txt
        --block_size 8
        ''',
       "mlm": f'''
        {scripts_dir}/language-modeling/run_mlm.py
        --train_file {FIXTURE_DIRECTORY}/sample_text.txt
        ''',
       "qa": f'''
        {scripts_dir}/question-answering/run_qa.py
        --train_file {data_dir_samples}/SQUAD/sample.json
        ''',
       "clas": f'''
        {scripts_dir}/text-classification/run_glue.py
        --train_file {data_dir_samples}/MRPC/train.csv
        --max_seq_length 12
        --task_name MRPC
        ''',
       "img_clas": f'''
        {scripts_dir}/image-classification/run_image_classification.py
            --dataset_name hf-internal-testing/cats_vs_dogs_sample
            --remove_unused_columns False
            --max_steps 10
            --image_processor_name {DS_TESTS_DIRECTORY}/vit_feature_extractor.json
        ''',
   }

   lowercase__    =  get_launcher(distributed=__magic_name__       )

   lowercase__    =  {}
   for task, args in tasks.items():
      lowercase__    =  args.split()
      for model in tasksamodels[task]:
         lowercase__    =  globals()[f'''{model.upper().replace('-'  ,					'_'       )}_TINY''']
         lowercase__    =  f'''--model_name_or_path {model_name}'''.split()
         lowercase__    =  launcher + args + args_model + args_main

         # # generation special case
         # if task == "gen":
         #     launcher = f"deepspeed --num_nodes 1 --num_gpus 1".split()
         #     args_model += f"--model_type {model}".split()
         #     cmds[f"{task}_{model}"] = launcher + args + args_model
         # else:

   return cmds


_snake_case				= make_task_cmds()

_snake_case				= """zero2"""
_snake_case				= """zero3"""

_snake_case				= [ZEROa, ZEROa]


def _A   (    __magic_name__  ,					__magic_name__  ,					__magic_name__       ):
   # customize the test name generator function as we want both params to appear in the sub-test
   # name, as by default it shows only the first param
   lowercase__    =  parameterized.to_safe_name("_".join(str(__magic_name__       ) for x in param.args       )       )
   return f'''{func.__name__}_{param_based_name}'''


# Cartesian-product of zero stages with models to test
_snake_case				= list(itertools.product(stages, task_cmds.keys()))







@slow
@require_deepspeed
@require_torch_gpu
class 					lowerCAmelCase							(   lowercase_					):







  def   UpperCAmelCase     (			self  :Tuple    ,							_lowercase  :int    ,							_lowercase  :Any					):






     '''simple docstring'''





     if task not in task_cmds:
        raise ValueError(f'''don\'t know of task {task}, have {task_cmds.keys()}'''					)

     lowercase__    =  task_cmds[task]
     lowercase__    =  f'''--deepspeed {self.test_file_dir_str}/ds_config_{stage}.json'''.split()

     lowercase__    =  self.get_auto_remove_tmp_dir()
     lowercase__    =  f'''--output_dir {output_dir}'''.split()

     cmd += args_ds + args_out

     return cmd, output_dir







  @parameterized.expand(_lowercase    ,							name_func=_lowercase					)
  def   UpperCAmelCase     (			self  :str    ,							_lowercase  :Dict    ,							_lowercase  :Tuple					):






     '''simple docstring'''





     lowercase__   ,							lowercase__    =  self.get_task_cmd(_lowercase    ,							_lowercase					)

     # 1. generate the checkpoint
     cmd += "--save_steps 1".split()
     # keep for quick debug
     # print(" ".join([f"\nPYTHONPATH={self.src_dir_str}"] + cmd)); die
     execute_subprocess_async(_lowercase    ,							env=self.get_env()					)

     # 2. test that the fp32 weights get reconsolidated
     lowercase__    =  f'''{output_dir}/checkpoint-1'''
     lowercase__    =  f'''{chkpt_dir}/out.bin'''
     lowercase__    =  f'''{chkpt_dir}/zero_to_fp32.py {chkpt_dir} {recovered_model_path}'''
     # keep for quick debug
     # print(" ".join([f"\nPYTHONPATH={self.src_dir_str}"] +cmd)); die
     subprocess.check_call(_lowercase    ,							shell=_lowercase					)
     assert os.path.exists(_lowercase					), f'''{recovered_model_path} was not found'''

     # possibly could also test that the resulting saved model is usable but given that we use
     # random models we won't know if it's any good






