




import itertools
import os
import subprocess
from os.path import dirname

from parameterized import parameterized

from tests.trainer.test_trainer import TrainerIntegrationCommon  # noqa
from transformers import is_torch_available
from transformers.testing_utils import (
    TestCasePlus,
    execute_subprocess_async,
    get_gpu_count,
    get_tests_dir,
    require_deepspeed,
    require_torch_gpu,
    slow,
)
from transformers.trainer_utils import set_seed


if is_torch_available():
							from tests.trainer.test_trainer import (  # noqa
							    RegressionModelConfig,
							    RegressionPreTrainedModel,
							    get_regression_trainer,
							)


set_seed(4_2)

UpperCamelCase__              =						get_tests_dir('fixtures')
UpperCamelCase__              =						os.path.join(dirname(get_tests_dir()))
UpperCamelCase__              =						dirname(os.path.abspath(__file__))

# default torch.distributed port
UpperCamelCase__              =						'10999'

UpperCamelCase__              =						't5-small'

# *** Working Models ***
UpperCamelCase__              =						'hf-internal-testing/tiny-albert'
UpperCamelCase__              =						'sshleifer/bart-tiny-random'
UpperCamelCase__              =						'hf-internal-testing/tiny-bert'
UpperCamelCase__              =						'hf-internal-testing/tiny-random-bigbird_pegasus'
UpperCamelCase__              =						'hf-internal-testing/tiny-random-big_bird'
UpperCamelCase__              =						'hf-internal-testing/tiny-random-blenderbot'
UpperCamelCase__              =						'bigscience/bigscience-small-testing'
UpperCamelCase__              =						'hf-internal-testing/tiny-random-deberta'
UpperCamelCase__              =						'hf-internal-testing/tiny-random-deberta-v2'
UpperCamelCase__              =						'sshleifer/tiny-distilbert-base-cased'
UpperCamelCase__              =						'hf-internal-testing/tiny-electra'
UpperCamelCase__              =						'hf-internal-testing/tiny-random-flaubert'
UpperCamelCase__              =						'stas/tiny-wmt19-en-de'
UpperCamelCase__              =						'hf-internal-testing/tiny-random-funnel'
UpperCamelCase__              =						'sshleifer/tiny-gpt2'
UpperCamelCase__              =						'hf-internal-testing/tiny-random-gptj'
UpperCamelCase__              =						'hf-internal-testing/tiny-random-gpt_neo'
UpperCamelCase__              =						'hf-internal-testing/tiny-layoutlm'
UpperCamelCase__              =						'hf-internal-testing/tiny-random-led'
UpperCamelCase__              =						'hf-internal-testing/tiny-random-longformer'
UpperCamelCase__              =						'stas/tiny-m2m_100'  # hf tiny model is unsuitable
UpperCamelCase__              =						'sshleifer/tiny-marian-en-de'
UpperCamelCase__              =						'sshleifer/tiny-mbart'
UpperCamelCase__              =						'hf-internal-testing/tiny-random-mobilebert'
UpperCamelCase__              =						'hf-internal-testing/tiny-random-mpnet'
UpperCamelCase__              =						'stas/pegasus-cnn_dailymail-tiny-random'
UpperCamelCase__              =						'hf-internal-testing/tiny-random-prophetnet'
UpperCamelCase__              =						'sshleifer/tiny-distilroberta-base'
UpperCamelCase__              =						'hf-internal-testing/tiny-random-squeezebert'
UpperCamelCase__              =						'patrickvonplaten/t5-tiny-random'
UpperCamelCase__              =						'hf-internal-testing/tiny-random-t5-v1.1'
UpperCamelCase__              =						'hf-internal-testing/tiny-random-vit'
UpperCamelCase__              =						'hf-internal-testing/tiny-xlm-roberta'
UpperCamelCase__              =						'sshleifer/tiny-xlnet-base-cased'


# *** To Fix ***


# *** tiny model issues ***
# missing model files:
UpperCamelCase__              =						'hf-internal-testing/tiny-random-mt5'
UpperCamelCase__              =						'hf-internal-testing/tiny-random-camembert'
UpperCamelCase__              =						'hf-internal-testing/tiny-random-openai-gpt'

# missing tokenizer files
UpperCamelCase__              =						'hf-internal-testing/tiny-random-convbert'
UpperCamelCase__              =						'hf-internal-testing/tiny-random-layoutlmv2'
UpperCamelCase__              =						'hf-internal-testing/tiny-random-hubert'

# issues with tokenizer
UpperCamelCase__              =						'hf-internal-testing/tiny-random-ctrl'
UpperCamelCase__              =						'hf-internal-testing/tiny-random-transfo-xl'  # same as ctrl

# other issues with tiny models
UpperCamelCase__              =						'hf-internal-testing/tiny-random-ibert'  # multiple issues with either mlm/qa/clas
UpperCamelCase__              =						'hf-internal-testing/tiny-random-reformer'  # multiple issues with either mlm/qa/clas

# *** Lacking official examples to test with ***
# or not working with examples
UpperCamelCase__              =						'hf-internal-testing/tiny-random-dpr'
# - "dpr"  examples/research_projects/rag-end2end-retriever/
UpperCamelCase__              =						'hf-internal-testing/tiny-random-rag'
# - "rag" research_projects
UpperCamelCase__              =						''
# - "luke" Entities classes - no plan to make such example
UpperCamelCase__              =						'hf-internal-testing/tiny-random-lxmert'
# - "lxmert" doesn't work with run_qa.py
UpperCamelCase__              =						'hf-internal-testing/tiny-random-clip'
# - "clip" nothing under pytorch examples - XXX: Suraj is working on adding some - check by end of Sep
UpperCamelCase__              =						'hf-internal-testing/tiny-random-speech_to_text'
# - "speech_to_text", nothing under pytorch examples


# *** Reactive mode ***
# models with low usage, unstable API, things about to change - do nothing about the following until someone runs into a problem
UpperCamelCase__              =						'hf-internal-testing/tiny-random-tapas'






def 					snake_case					(							A=False			)							->							List[str]:




	'''simple docstring'''


	lowerCamelCase_							:		List[Any]							   =							min(2						,   get_gpu_count()			) if distributed else 1
	lowerCamelCase_							:		Union[str, Any]							   =							os.environ.get('DS_TEST_PORT'						,   A			)
	return f"deepspeed --num_nodes 1 --num_gpus {num_gpus} --master_port {master_port}".split()






def 					snake_case					(							)							->							Optional[int]:




	'''simple docstring'''


	lowerCamelCase_							:		List[str]							   =							f"{FIXTURE_DIRECTORY}/tests_samples"
	lowerCamelCase_							:		str							   =							f"{data_dir_samples}/wmt_en_ro"
	lowerCamelCase_							:		Any							   =							f"{data_dir_samples}/xsum"
	lowerCamelCase_							:		Optional[int]							   =							'\n        --do_train\n        --max_train_samples 4\n        --per_device_train_batch_size 2\n        --num_train_epochs 1\n        --fp16\n        --report_to none\n        --overwrite_output_dir\n        '.split()

	# try to cover as many models as possible once (it's enough to run on one task per model)
	# but need a tiny model for each
	#
	# should have "{model_type.upper()}_TINY" corresponding vars defined, e.g., T5_TINY, etc.
	lowerCamelCase_							:		List[str]							   =							{
	    'trans': [
	        'bart',
	        'fsmt',
	        'm2m_100',
	        'marian',
	        'mbart',
	        't5',
	        't5_v1',
	        # "mt5", missing model files
	    ],
	    'sum': [
	        'pegasus',
	    ],
	    'clm': [
	        'big_bird',
	        'bigbird_pegasus',
	        'blenderbot',
	        'bloom',
	        'gpt2',
	        'gpt_neo',
	        'gptj',
	        'xlm-roberta',
	        'prophetnet',
	        # "camembert", missing model files
	    ],
	    'mlm': [
	        'albert',
	        'deberta',
	        'deberta-v2',
	        'distilbert',
	        'electra',
	        'flaubert',
	        'funnel',
	        'layoutlm',
	        # "reformer", # multiple issues with either mlm/qa/clas
	    ],
	    'qa': [
	        'led',
	        'longformer',
	        'mobilebert',
	        'mpnet',
	        'roberta',
	        'squeezebert',
	        # "convbert", # missing tokenizer files
	        # "layoutlmv2", missing model files
	    ],
	    'clas': [
	        'bert',
	        'xlnet',
	        # "hubert", # missing tokenizer files
	        # "ibert", # multiple issues with either mlm/qa/clas
	        # "transfo-xl", # tokenizer issues as ctrl
	        # "ctrl", # tokenizer issues
	        # "openai-gpt", missing model files
	        # "tapas", multiple issues
	    ],
	    'img_clas': [
	        'vit',
	    ],
	}

	lowerCamelCase_							:		Optional[Any]							   =							f"{ROOT_DIRECTORY}/examples/pytorch"

	lowerCamelCase_							:		List[str]							   =							{
	    'trans': f"\n        {scripts_dir}/translation/run_translation.py\n        --train_file {data_dir_wmt}/train.json\n        --source_lang en\n        --target_lang ro\n        ",
	    'sum': f"\n        {scripts_dir}/summarization/run_summarization.py\n        --train_file {data_dir_xsum}/sample.json\n        --max_source_length 12\n        --max_target_length 12\n        --lang en\n        ",
	    'clm': f"\n        {scripts_dir}/language-modeling/run_clm.py\n        --train_file {FIXTURE_DIRECTORY}/sample_text.txt\n        --block_size 8\n        ",
	    'mlm': f"\n        {scripts_dir}/language-modeling/run_mlm.py\n        --train_file {FIXTURE_DIRECTORY}/sample_text.txt\n        ",
	    'qa': f"\n        {scripts_dir}/question-answering/run_qa.py\n        --train_file {data_dir_samples}/SQUAD/sample.json\n        ",
	    'clas': f"\n        {scripts_dir}/text-classification/run_glue.py\n        --train_file {data_dir_samples}/MRPC/train.csv\n        --max_seq_length 12\n        --task_name MRPC\n        ",
	    'img_clas': f"\n        {scripts_dir}/image-classification/run_image_classification.py\n            --dataset_name hf-internal-testing/cats_vs_dogs_sample\n            --remove_unused_columns False\n            --max_steps 10\n            --image_processor_name {DS_TESTS_DIRECTORY}/vit_feature_extractor.json\n        ",
	}

	lowerCamelCase_							:		List[str]							   =							get_launcher(distributed=A			)

	lowerCamelCase_							:		Tuple							   =							{}
	for task, args in tasks.items():
		lowerCamelCase_							:		str							   =							args.split()
		for model in tasksamodels[task]:
			lowerCamelCase_							:		str							   =							globals()[f"{model.upper().replace('-'						,   '_'			)}_TINY"]
			lowerCamelCase_							:		Dict							   =							f"--model_name_or_path {model_name}".split()
			lowerCamelCase_							:		Optional[Any]							   =							launcher + args + args_model + args_main

			# # generation special case
			# if task == "gen":
			#     launcher = f"deepspeed --num_nodes 1 --num_gpus 1".split()
			#     args_model += f"--model_type {model}".split()
			#     cmds[f"{task}_{model}"] = launcher + args + args_model
			# else:

	return cmds


UpperCamelCase__              =						make_task_cmds()

UpperCamelCase__              =						'zero2'
UpperCamelCase__              =						'zero3'

UpperCamelCase__              =						[ZEROa, ZEROa]






def 					snake_case					(							A						,   A						,   A			)							->							List[Any]:




	'''simple docstring'''


	lowerCamelCase_							:		Any							   =							parameterized.to_safe_name('_'.join(str(A			) for x in param.args			)			)
	return f"{func.__name__}_{param_based_name}"


# Cartesian-product of zero stages with models to test
UpperCamelCase__              =						list(itertools.product(stages, task_cmds.keys()))



@slow
@require_deepspeed
@require_torch_gpu
class 				__SCREAMING_SNAKE_CASE							(		lowerCamelCase  ):

					def 	lowercase_							(				self ,       __lowercase ,       __lowercase					):







						'''simple docstring'''




						if task not in task_cmds:
							raise ValueError(f"don't know of task {task}, have {task_cmds.keys()}"					)

						lowerCamelCase_							:		Union[str, Any]							   =							task_cmds[task]
						lowerCamelCase_							:		Union[str, Any]							   =							f"--deepspeed {self.test_file_dir_str}/ds_config_{stage}.json".split()

						lowerCamelCase_							:		List[Any]							   =							self.get_auto_remove_tmp_dir()
						lowerCamelCase_							:		Tuple							   =							f"--output_dir {output_dir}".split()

						cmd += args_ds + args_out

						return cmd, output_dir





					@parameterized.expand(__lowercase ,       name_func=__lowercase					)
					def 	lowercase_							(				self ,       __lowercase ,       __lowercase					):







						'''simple docstring'''




						lowerCamelCase_   ,			lowerCamelCase_							:		int							   =							self.get_task_cmd(__lowercase ,       __lowercase					)

						# 1. generate the checkpoint
						cmd += "--save_steps 1".split()
						# keep for quick debug
						# print(" ".join([f"\nPYTHONPATH={self.src_dir_str}"] + cmd)); die
						execute_subprocess_async(__lowercase ,       env=self.get_env()					)

						# 2. test that the fp32 weights get reconsolidated
						lowerCamelCase_							:		Tuple							   =							f"{output_dir}/checkpoint-1"
						lowerCamelCase_							:		Optional[int]							   =							f"{chkpt_dir}/out.bin"
						lowerCamelCase_							:		Dict							   =							f"{chkpt_dir}/zero_to_fp32.py {chkpt_dir} {recovered_model_path}"
						# keep for quick debug
						# print(" ".join([f"\nPYTHONPATH={self.src_dir_str}"] +cmd)); die
						subprocess.check_call(__lowercase ,       shell=__lowercase					)
						assert os.path.exists(__lowercase					), f"{recovered_model_path} was not found"

						# possibly could also test that the resulting saved model is usable but given that we use
						# random models we won't know if it's any good






'''simple docstring'''



from sklearn.metrics import accuracy_score

import datasets


UpperCAmelCase_				:     List[Any]     			=						'''
Accuracy is the proportion of correct predictions among the total number of cases processed. It can be computed with:
Accuracy = (TP + TN) / (TP + TN + FP + FN)
 Where:
TP: True positive
TN: True negative
FP: False positive
FN: False negative
'''


UpperCAmelCase_				:     List[Any]     			=						'''
Args:
    predictions (`list` of `int`): Predicted labels.
    references (`list` of `int`): Ground truth labels.
    normalize (`boolean`): If set to False, returns the number of correctly classified samples. Otherwise, returns the fraction of correctly classified samples. Defaults to True.
    sample_weight (`list` of `float`): Sample weights Defaults to None.

Returns:
    accuracy (`float` or `int`): Accuracy score. Minimum possible value is 0. Maximum possible value is 1.0, or the number of examples input, if `normalize` is set to `True`.. A higher score means higher accuracy.

Examples:

    Example 1-A simple example
        >>> accuracy_metric = datasets.load_metric("accuracy")
        >>> results = accuracy_metric.compute(references=[0, 1, 2, 0, 1, 2], predictions=[0, 1, 1, 2, 1, 0])
        >>> print(results)
        {\'accuracy\': 0.5}

    Example 2-The same as Example 1, except with `normalize` set to `False`.
        >>> accuracy_metric = datasets.load_metric("accuracy")
        >>> results = accuracy_metric.compute(references=[0, 1, 2, 0, 1, 2], predictions=[0, 1, 1, 2, 1, 0], normalize=False)
        >>> print(results)
        {\'accuracy\': 3.0}

    Example 3-The same as Example 1, except with `sample_weight` set.
        >>> accuracy_metric = datasets.load_metric("accuracy")
        >>> results = accuracy_metric.compute(references=[0, 1, 2, 0, 1, 2], predictions=[0, 1, 1, 2, 1, 0], sample_weight=[0.5, 2, 0.7, 0.5, 9, 0.4])
        >>> print(results)
        {\'accuracy\': 0.8778625954198473}
'''


UpperCAmelCase_				:     Union[str, Any]     			=						'''
@article{scikit-learn,
  title={Scikit-learn: Machine Learning in {P}ython},
  author={Pedregosa, F. and Varoquaux, G. and Gramfort, A. and Michel, V.
         and Thirion, B. and Grisel, O. and Blondel, M. and Prettenhofer, P.
         and Weiss, R. and Dubourg, V. and Vanderplas, J. and Passos, A. and
         Cournapeau, D. and Brucher, M. and Perrot, M. and Duchesnay, E.},
  journal={Journal of Machine Learning Research},
  volume={12},
  pages={2825--2830},
  year={2011}
}
'''





@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION ,      _KWARGS_DESCRIPTION)
class 			lowerCAmelCase      (       datasets.Metric):


       def 					lowerCAmelCase					(      self     )					->     List[str]:



            '''simple docstring'''




            return datasets.MetricInfo(
                description=_DESCRIPTION			,     citation=_CITATION			,     inputs_description=_KWARGS_DESCRIPTION			,     features=datasets.Features(
                    {
                        '''predictions''': datasets.Sequence(datasets.Value('''int32'''     )     ),
                        '''references''': datasets.Sequence(datasets.Value('''int32'''     )     ),
                    }
                    if self.config_name == '''multilabel'''
                    else {
                        '''predictions''': datasets.Value('''int32'''     ),
                        '''references''': datasets.Value('''int32'''     ),
                    }     )			,     reference_urls=['''https://scikit-learn.org/stable/modules/generated/sklearn.metrics.accuracy_score.html''']			,     )




       def 					lowerCAmelCase					(      self			,     __SCREAMING_SNAKE_CASE			,     __SCREAMING_SNAKE_CASE			,     __SCREAMING_SNAKE_CASE=True			,     __SCREAMING_SNAKE_CASE=None     )					->     Any:



            '''simple docstring'''




            return {
                "accuracy": float(
                    accuracy_score(__SCREAMING_SNAKE_CASE			,     __SCREAMING_SNAKE_CASE			,     normalize=__SCREAMING_SNAKE_CASE			,     sample_weight=__SCREAMING_SNAKE_CASE     )     )
            }






