



import argparse
import json
import logging
import os
import shutil
import sys
import tempfile
import unittest
from unittest import mock

import torch
from accelerate.utils import write_basic_config

from transformers.testing_utils import TestCasePlus, get_gpu_count, run_command, slow, torch_device
from transformers.utils import is_apex_available


logging.basicConfig(level=logging.DEBUG)

__snake_case       :	Tuple            =logging.getLogger()






def 			lowerCAmelCase__	(					):





				'''simple docstring'''

				lowerCAmelCase__  :       Union[str, Any]				=  argparse.ArgumentParser()
				parser.add_argument('''-f''')
				lowerCAmelCase__  :       Optional[Any]				=  parser.parse_args()
				return args.f






def 			lowerCAmelCase__	(					lowerCamelCase_	:	List[Any]):





				'''simple docstring'''

				lowerCAmelCase__  :       List[Any]				=  {}
				lowerCAmelCase__  :       Tuple				=  os.path.join(lowerCamelCase_ ,'''all_results.json''')
				if os.path.exists(lowerCamelCase_):
								with open(lowerCamelCase_ ,'''r''') as f:
												lowerCAmelCase__  :       Any				=  json.load(lowerCamelCase_)
				else:
								raise ValueError(f"""can't find {path}""")
				return results






def 			lowerCAmelCase__	(					):





				'''simple docstring'''

				lowerCAmelCase__  :       List[str]				=  torch.cuda.is_available() and torch_device == '''cuda'''
				return is_using_cuda and is_apex_available()


__snake_case       :	Optional[Any]            =logging.StreamHandler(sys.stdout)
logger.addHandler(stream_handler)



class  lowerCamelCase__    (							lowerCamelCase__):




		'''simple docstring'''



		@classmethod
		def 	lowerCAmelCase__   (cls				)  ->    Optional[int]:

						"""simple docstring"""






						lowerCAmelCase__  :       Tuple				=  tempfile.mkdtemp()
						lowerCAmelCase__  :       str				=  os.path.join(cls.tmpdir	,'''default_config.yml'''				)
						write_basic_config(save_location=cls.configPath				)
						lowerCAmelCase__  :       Optional[Any]				=  ['''accelerate''', '''launch''', '''--config_file''', cls.configPath]



		@classmethod
		def 	lowerCAmelCase__   (cls				)  ->    Dict:

						"""simple docstring"""






						shutil.rmtree(cls.tmpdir				)



		@mock.patch.dict(os.environ	,{'''WANDB_MODE''': '''offline'''}				)
		def 	lowerCAmelCase__   (self				)  ->    Any:

						"""simple docstring"""






						lowerCAmelCase__  :       List[str]				=  self.get_auto_remove_tmp_dir()
						lowerCAmelCase__  :       List[Any]				=  f"""
            {self.examples_dir}/pytorch/text-classification/run_glue_no_trainer.py
            --model_name_or_path distilbert-base-uncased
            --output_dir {tmp_dir}
            --train_file ./tests/fixtures/tests_samples/MRPC/train.csv
            --validation_file ./tests/fixtures/tests_samples/MRPC/dev.csv
            --per_device_train_batch_size=2
            --per_device_eval_batch_size=1
            --learning_rate=1e-4
            --seed=42
            --checkpointing_steps epoch
            --with_tracking
        """.split()

						if is_cuda_and_apex_available():
										testargs.append('''--fp16'''				)

						run_command(self._launch_args + testargs				)
						lowerCAmelCase__  :       List[Any]				=  get_results(__lowerCamelCase				)
						self.assertGreaterEqual(result['''eval_accuracy''']	,0.75				)
						self.assertTrue(os.path.exists(os.path.join(__lowerCamelCase	,'''epoch_0'''				)				)				)
						self.assertTrue(os.path.exists(os.path.join(__lowerCamelCase	,'''glue_no_trainer'''				)				)				)



		@mock.patch.dict(os.environ	,{'''WANDB_MODE''': '''offline'''}				)
		def 	lowerCAmelCase__   (self				)  ->    Optional[int]:

						"""simple docstring"""






						lowerCAmelCase__  :       Optional[Any]				=  self.get_auto_remove_tmp_dir()
						lowerCAmelCase__  :       Optional[int]				=  f"""
            {self.examples_dir}/pytorch/language-modeling/run_clm_no_trainer.py
            --model_name_or_path distilgpt2
            --train_file ./tests/fixtures/sample_text.txt
            --validation_file ./tests/fixtures/sample_text.txt
            --block_size 128
            --per_device_train_batch_size 5
            --per_device_eval_batch_size 5
            --num_train_epochs 2
            --output_dir {tmp_dir}
            --checkpointing_steps epoch
            --with_tracking
        """.split()

						if torch.cuda.device_count() > 1:
										# Skipping because there are not enough batches to train the model + would need a drop_last to work.
										return

						run_command(self._launch_args + testargs				)
						lowerCAmelCase__  :       Optional[Any]				=  get_results(__lowerCamelCase				)
						self.assertLess(result['''perplexity''']	,1_00				)
						self.assertTrue(os.path.exists(os.path.join(__lowerCamelCase	,'''epoch_0'''				)				)				)
						self.assertTrue(os.path.exists(os.path.join(__lowerCamelCase	,'''clm_no_trainer'''				)				)				)



		@mock.patch.dict(os.environ	,{'''WANDB_MODE''': '''offline'''}				)
		def 	lowerCAmelCase__   (self				)  ->    int:

						"""simple docstring"""






						lowerCAmelCase__  :       int				=  self.get_auto_remove_tmp_dir()
						lowerCAmelCase__  :       str				=  f"""
            {self.examples_dir}/pytorch/language-modeling/run_mlm_no_trainer.py
            --model_name_or_path distilroberta-base
            --train_file ./tests/fixtures/sample_text.txt
            --validation_file ./tests/fixtures/sample_text.txt
            --output_dir {tmp_dir}
            --num_train_epochs=1
            --checkpointing_steps epoch
            --with_tracking
        """.split()

						run_command(self._launch_args + testargs				)
						lowerCAmelCase__  :       List[Any]				=  get_results(__lowerCamelCase				)
						self.assertLess(result['''perplexity''']	,42				)
						self.assertTrue(os.path.exists(os.path.join(__lowerCamelCase	,'''epoch_0'''				)				)				)
						self.assertTrue(os.path.exists(os.path.join(__lowerCamelCase	,'''mlm_no_trainer'''				)				)				)



		@mock.patch.dict(os.environ	,{'''WANDB_MODE''': '''offline'''}				)
		def 	lowerCAmelCase__   (self				)  ->    List[str]:

						"""simple docstring"""






						lowerCAmelCase__  :       Tuple				=  7 if get_gpu_count() > 1 else 2

						lowerCAmelCase__  :       Optional[Any]				=  self.get_auto_remove_tmp_dir()
						lowerCAmelCase__  :       int				=  f"""
            {self.examples_dir}/pytorch/token-classification/run_ner_no_trainer.py
            --model_name_or_path bert-base-uncased
            --train_file tests/fixtures/tests_samples/conll/sample.json
            --validation_file tests/fixtures/tests_samples/conll/sample.json
            --output_dir {tmp_dir}
            --learning_rate=2e-4
            --per_device_train_batch_size=2
            --per_device_eval_batch_size=2
            --num_train_epochs={epochs}
            --seed 7
            --checkpointing_steps epoch
            --with_tracking
        """.split()

						run_command(self._launch_args + testargs				)
						lowerCAmelCase__  :       List[str]				=  get_results(__lowerCamelCase				)
						self.assertGreaterEqual(result['''eval_accuracy''']	,0.75				)
						self.assertLess(result['''train_loss''']	,0.5				)
						self.assertTrue(os.path.exists(os.path.join(__lowerCamelCase	,'''epoch_0'''				)				)				)
						self.assertTrue(os.path.exists(os.path.join(__lowerCamelCase	,'''ner_no_trainer'''				)				)				)



		@unittest.skip(reason='''Fix me @muellerzr'''				)
		@mock.patch.dict(os.environ	,{'''WANDB_MODE''': '''offline'''}				)
		def 	lowerCAmelCase__   (self				)  ->    Optional[Any]:

						"""simple docstring"""






						lowerCAmelCase__  :       Optional[int]				=  self.get_auto_remove_tmp_dir()
						lowerCAmelCase__  :       Dict				=  f"""
            {self.examples_dir}/pytorch/question-answering/run_qa_no_trainer.py
            --model_name_or_path bert-base-uncased
            --version_2_with_negative
            --train_file tests/fixtures/tests_samples/SQUAD/sample.json
            --validation_file tests/fixtures/tests_samples/SQUAD/sample.json
            --output_dir {tmp_dir}
            --seed=42
            --max_train_steps=10
            --num_warmup_steps=2
            --learning_rate=2e-4
            --per_device_train_batch_size=2
            --per_device_eval_batch_size=1
            --checkpointing_steps epoch
            --with_tracking
        """.split()

						run_command(self._launch_args + testargs				)
						lowerCAmelCase__  :       Optional[int]				=  get_results(__lowerCamelCase				)
						# Because we use --version_2_with_negative the testing script uses SQuAD v2 metrics.
						self.assertGreaterEqual(result['''eval_f1''']	,28				)
						self.assertGreaterEqual(result['''eval_exact''']	,28				)
						self.assertTrue(os.path.exists(os.path.join(__lowerCamelCase	,'''epoch_0'''				)				)				)
						self.assertTrue(os.path.exists(os.path.join(__lowerCamelCase	,'''qa_no_trainer'''				)				)				)



		@mock.patch.dict(os.environ	,{'''WANDB_MODE''': '''offline'''}				)
		def 	lowerCAmelCase__   (self				)  ->    str:

						"""simple docstring"""






						lowerCAmelCase__  :       str				=  self.get_auto_remove_tmp_dir()
						lowerCAmelCase__  :       str				=  f"""
            {self.examples_dir}/pytorch/multiple-choice/run_swag_no_trainer.py
            --model_name_or_path bert-base-uncased
            --train_file tests/fixtures/tests_samples/swag/sample.json
            --validation_file tests/fixtures/tests_samples/swag/sample.json
            --output_dir {tmp_dir}
            --max_train_steps=20
            --num_warmup_steps=2
            --learning_rate=2e-4
            --per_device_train_batch_size=2
            --per_device_eval_batch_size=1
            --with_tracking
        """.split()

						run_command(self._launch_args + testargs				)
						lowerCAmelCase__  :       Optional[Any]				=  get_results(__lowerCamelCase				)
						self.assertGreaterEqual(result['''eval_accuracy''']	,0.8				)
						self.assertTrue(os.path.exists(os.path.join(__lowerCamelCase	,'''swag_no_trainer'''				)				)				)



		@slow
		@mock.patch.dict(os.environ	,{'''WANDB_MODE''': '''offline'''}				)
		def 	lowerCAmelCase__   (self				)  ->    List[str]:

						"""simple docstring"""






						lowerCAmelCase__  :       Any				=  self.get_auto_remove_tmp_dir()
						lowerCAmelCase__  :       Union[str, Any]				=  f"""
            {self.examples_dir}/pytorch/summarization/run_summarization_no_trainer.py
            --model_name_or_path t5-small
            --train_file tests/fixtures/tests_samples/xsum/sample.json
            --validation_file tests/fixtures/tests_samples/xsum/sample.json
            --output_dir {tmp_dir}
            --max_train_steps=50
            --num_warmup_steps=8
            --learning_rate=2e-4
            --per_device_train_batch_size=2
            --per_device_eval_batch_size=1
            --checkpointing_steps epoch
            --with_tracking
        """.split()

						run_command(self._launch_args + testargs				)
						lowerCAmelCase__  :       Optional[Any]				=  get_results(__lowerCamelCase				)
						self.assertGreaterEqual(result['''eval_rouge1''']	,10				)
						self.assertGreaterEqual(result['''eval_rouge2''']	,2				)
						self.assertGreaterEqual(result['''eval_rougeL''']	,7				)
						self.assertGreaterEqual(result['''eval_rougeLsum''']	,7				)
						self.assertTrue(os.path.exists(os.path.join(__lowerCamelCase	,'''epoch_0'''				)				)				)
						self.assertTrue(os.path.exists(os.path.join(__lowerCamelCase	,'''summarization_no_trainer'''				)				)				)



		@slow
		@mock.patch.dict(os.environ	,{'''WANDB_MODE''': '''offline'''}				)
		def 	lowerCAmelCase__   (self				)  ->    Optional[int]:

						"""simple docstring"""






						lowerCAmelCase__  :       Any				=  self.get_auto_remove_tmp_dir()
						lowerCAmelCase__  :       Any				=  f"""
            {self.examples_dir}/pytorch/translation/run_translation_no_trainer.py
            --model_name_or_path sshleifer/student_marian_en_ro_6_1
            --source_lang en
            --target_lang ro
            --train_file tests/fixtures/tests_samples/wmt16/sample.json
            --validation_file tests/fixtures/tests_samples/wmt16/sample.json
            --output_dir {tmp_dir}
            --max_train_steps=50
            --num_warmup_steps=8
            --num_beams=6
            --learning_rate=3e-3
            --per_device_train_batch_size=2
            --per_device_eval_batch_size=1
            --source_lang en_XX
            --target_lang ro_RO
            --checkpointing_steps epoch
            --with_tracking
        """.split()

						run_command(self._launch_args + testargs				)
						lowerCAmelCase__  :       Tuple				=  get_results(__lowerCamelCase				)
						self.assertGreaterEqual(result['''eval_bleu''']	,30				)
						self.assertTrue(os.path.exists(os.path.join(__lowerCamelCase	,'''epoch_0'''				)				)				)
						self.assertTrue(os.path.exists(os.path.join(__lowerCamelCase	,'''translation_no_trainer'''				)				)				)



		@slow
		def 	lowerCAmelCase__   (self				)  ->    Dict:

						"""simple docstring"""






						lowerCAmelCase__  :       Dict				=  logging.StreamHandler(sys.stdout				)
						logger.addHandler(__lowerCamelCase				)

						lowerCAmelCase__  :       List[Any]				=  self.get_auto_remove_tmp_dir()
						lowerCAmelCase__  :       Optional[int]				=  f"""
            {self.examples_dir}/pytorch/semantic-segmentation/run_semantic_segmentation_no_trainer.py
            --dataset_name huggingface/semantic-segmentation-test-sample
            --output_dir {tmp_dir}
            --max_train_steps=10
            --num_warmup_steps=2
            --learning_rate=2e-4
            --per_device_train_batch_size=2
            --per_device_eval_batch_size=1
            --checkpointing_steps epoch
        """.split()

						run_command(self._launch_args + testargs				)
						lowerCAmelCase__  :       str				=  get_results(__lowerCamelCase				)
						self.assertGreaterEqual(result['''eval_overall_accuracy''']	,0.10				)



		@mock.patch.dict(os.environ	,{'''WANDB_MODE''': '''offline'''}				)
		def 	lowerCAmelCase__   (self				)  ->    int:

						"""simple docstring"""






						lowerCAmelCase__  :       Tuple				=  self.get_auto_remove_tmp_dir()
						lowerCAmelCase__  :       Optional[int]				=  f"""
            {self.examples_dir}/pytorch/image-classification/run_image_classification_no_trainer.py
            --model_name_or_path google/vit-base-patch16-224-in21k
            --dataset_name hf-internal-testing/cats_vs_dogs_sample
            --learning_rate 1e-4
            --per_device_train_batch_size 2
            --per_device_eval_batch_size 1
            --max_train_steps 2
            --train_val_split 0.1
            --seed 42
            --output_dir {tmp_dir}
            --with_tracking
            --checkpointing_steps 1
        """.split()

						if is_cuda_and_apex_available():
										testargs.append('''--fp16'''				)

						run_command(self._launch_args + testargs				)
						lowerCAmelCase__  :       List[str]				=  get_results(__lowerCamelCase				)
						# The base model scores a 25%
						self.assertGreaterEqual(result['''eval_accuracy''']	,0.6				)
						self.assertTrue(os.path.exists(os.path.join(__lowerCamelCase	,'''step_1'''				)				)				)
						self.assertTrue(os.path.exists(os.path.join(__lowerCamelCase	,'''image_classification_no_trainer'''				)				)				)






from scipy.stats import pearsonr, spearmanr
from sklearn.metrics import fa_score, matthews_corrcoef

import datasets


__snake_case       :	str            ='\\n@inproceedings{wang2019glue,\n  title={{GLUE}: A Multi-Task Benchmark and Analysis Platform for Natural Language Understanding},\n  author={Wang, Alex and Singh, Amanpreet and Michael, Julian and Hill, Felix and Levy, Omer and Bowman, Samuel R.},\n  note={In the Proceedings of ICLR.},\n  year={2019}\n}\n'

__snake_case       :	Tuple            ='\\nGLUE, the General Language Understanding Evaluation benchmark\n(https://gluebenchmark.com/) is a collection of resources for training,\nevaluating, and analyzing natural language understanding systems.\n'

__snake_case       :	int            ='\nCompute GLUE evaluation metric associated to each GLUE dataset.\nArgs:\n    predictions: list of predictions to score.\n        Each translation should be tokenized into a list of tokens.\n    references: list of lists of references for each translation.\n        Each reference should be tokenized into a list of tokens.\nReturns: depending on the GLUE subset, one or several of:\n    "accuracy": Accuracy\n    "f1": F1 score\n    "pearson": Pearson Correlation\n    "spearmanr": Spearman Correlation\n    "matthews_correlation": Matthew Correlation\nExamples:\n\n    >>> glue_metric = datasets.load_metric(\'glue\', \'sst2\')  # \'sst2\' or any of ["mnli", "mnli_mismatched", "mnli_matched", "qnli", "rte", "wnli", "hans"]\n    >>> references = [0, 1]\n    >>> predictions = [0, 1]\n    >>> results = glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {\'accuracy\': 1.0}\n\n    >>> glue_metric = datasets.load_metric(\'glue\', \'mrpc\')  # \'mrpc\' or \'qqp\'\n    >>> references = [0, 1]\n    >>> predictions = [0, 1]\n    >>> results = glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {\'accuracy\': 1.0, \'f1\': 1.0}\n\n    >>> glue_metric = datasets.load_metric(\'glue\', \'stsb\')\n    >>> references = [0., 1., 2., 3., 4., 5.]\n    >>> predictions = [0., 1., 2., 3., 4., 5.]\n    >>> results = glue_metric.compute(predictions=predictions, references=references)\n    >>> print({"pearson": round(results["pearson"], 2), "spearmanr": round(results["spearmanr"], 2)})\n    {\'pearson\': 1.0, \'spearmanr\': 1.0}\n\n    >>> glue_metric = datasets.load_metric(\'glue\', \'cola\')\n    >>> references = [0, 1]\n    >>> predictions = [0, 1]\n    >>> results = glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {\'matthews_correlation\': 1.0}\n'






def 			lowerCAmelCase__	(					lowerCamelCase_	:	List[str] ,lowerCamelCase_	:	Dict):





				'''simple docstring'''

				return float((preds == labels).mean())






def 			lowerCAmelCase__	(					lowerCamelCase_	:	List[Any] ,lowerCamelCase_	:	Optional[int]):





				'''simple docstring'''

				lowerCAmelCase__  :       int				=  simple_accuracy(lowerCamelCase_ ,lowerCamelCase_)
				lowerCAmelCase__  :       Any				=  float(fa_score(y_true=lowerCamelCase_ ,y_pred=lowerCamelCase_))
				return {
				    "accuracy": acc,
				    "f1": fa,
				}






def 			lowerCAmelCase__	(					lowerCamelCase_	:	List[Any] ,lowerCamelCase_	:	Optional[Any]):





				'''simple docstring'''

				lowerCAmelCase__  :       Union[str, Any]				=  float(pearsonr(lowerCamelCase_ ,lowerCamelCase_)[0])
				lowerCAmelCase__  :       int				=  float(spearmanr(lowerCamelCase_ ,lowerCamelCase_)[0])
				return {
				    "pearson": pearson_corr,
				    "spearmanr": spearman_corr,
				}



@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION   ,       _KWARGS_DESCRIPTION)
class  lowerCamelCase__    (							datasets.Metric):




		'''simple docstring'''



		def 	lowerCAmelCase__   (self				)  ->    Union[str, Any]:

						"""simple docstring"""






						if self.config_name not in [
						    "sst2",
						    "mnli",
						    "mnli_mismatched",
						    "mnli_matched",
						    "cola",
						    "stsb",
						    "mrpc",
						    "qqp",
						    "qnli",
						    "rte",
						    "wnli",
						    "hans",
						]:
										raise KeyError(
										    '''You should supply a configuration name selected in '''
										    '''["sst2", "mnli", "mnli_mismatched", "mnli_matched", '''
										    '''"cola", "stsb", "mrpc", "qqp", "qnli", "rte", "wnli", "hans"]'''				)
						return datasets.MetricInfo(
						    description=_DESCRIPTION	,citation=_CITATION	,inputs_description=_KWARGS_DESCRIPTION	,features=datasets.Features(
						        {
						            '''predictions''': datasets.Value('''int64''' if self.config_name != '''stsb''' else '''float32'''				),
						            '''references''': datasets.Value('''int64''' if self.config_name != '''stsb''' else '''float32'''				),
						        }				)	,codebase_urls=[]	,reference_urls=[]	,format='''numpy'''	,)



		def 	lowerCAmelCase__   (self	,__lowerCamelCase	,__lowerCamelCase				)  ->    Tuple:

						"""simple docstring"""






						if self.config_name == "cola":
										return {"matthews_correlation": matthews_corrcoef(__lowerCamelCase	,__lowerCamelCase				)}
						elif self.config_name == "stsb":
										return pearson_and_spearman(__lowerCamelCase	,__lowerCamelCase				)
						elif self.config_name in ["mrpc", "qqp"]:
										return acc_and_fa(__lowerCamelCase	,__lowerCamelCase				)
						elif self.config_name in ["sst2", "mnli", "mnli_mismatched", "mnli_matched", "qnli", "rte", "wnli", "hans"]:
										return {"accuracy": simple_accuracy(__lowerCamelCase	,__lowerCamelCase				)}
						else:
										raise KeyError(
										    '''You should supply a configuration name selected in '''
										    '''["sst2", "mnli", "mnli_mismatched", "mnli_matched", '''
										    '''"cola", "stsb", "mrpc", "qqp", "qnli", "rte", "wnli", "hans"]'''				)

