




import argparse
import json
import logging
import os
import shutil
import sys
import tempfile
import unittest
from unittest import mock

import torch
from accelerate.utils import write_basic_config

from transformers.testing_utils import TestCasePlus, get_gpu_count, run_command, slow, torch_device
from transformers.utils import is_apex_available


logging.basicConfig(level=logging.DEBUG)

__SCREAMING_SNAKE_CASE 						=logging.getLogger()

def 			a   ():
							SCREAMING_SNAKE_CASE_   	=    argparse.ArgumentParser()
							parser.add_argument('''-f'''     )
							SCREAMING_SNAKE_CASE_   	=    parser.parse_args()
							return args.f

def 			a   (_lowerCAmelCase     ):
							SCREAMING_SNAKE_CASE_   	=    {}
							SCREAMING_SNAKE_CASE_   	=    os.path.join(_lowerCAmelCase  ,    '''all_results.json'''     )
							if os.path.exists(_lowerCAmelCase     ):
														with open(_lowerCAmelCase  ,    '''r'''     ) as f:
																					SCREAMING_SNAKE_CASE_   	=    json.load(_lowerCAmelCase     )
							else:
														raise ValueError(F"can't find {path}"     )
							return results

def 			a   ():
							SCREAMING_SNAKE_CASE_   	=    torch.cuda.is_available() and torch_device == '''cuda'''
							return is_using_cuda and is_apex_available()


__SCREAMING_SNAKE_CASE 						=logging.StreamHandler(sys.stdout)
logger.addHandler(stream_handler)






class  __magic_name__						(  __UpperCAmelCase):
				'''simple docstring'''







				@classmethod
				def     _A							(							cls:   str		):
											# Write Accelerate config, will pick up on CPU, GPU, and multi-GPU
											SCREAMING_SNAKE_CASE_   	=    tempfile.mkdtemp()
											SCREAMING_SNAKE_CASE_   	=    os.path.join(cls.tmpdir		,					'''default_config.yml'''		)
											write_basic_config(save_location=cls.configPath		)
											SCREAMING_SNAKE_CASE_   	=    ['''accelerate''', '''launch''', '''--config_file''', cls.configPath]


				@classmethod
				def     _A							(							cls:   str		):
											shutil.rmtree(cls.tmpdir		)


				@mock.patch.dict(os.environ		,					{'''WANDB_MODE''': '''offline'''}		)
				def     _A							(							self:   Union[str, Any]		):
											SCREAMING_SNAKE_CASE_   	=    self.get_auto_remove_tmp_dir()
											SCREAMING_SNAKE_CASE_   	=    f"\n            {self.examples_dir}/pytorch/text-classification/run_glue_no_trainer.py\n            --model_name_or_path distilbert-base-uncased\n            --output_dir {tmp_dir}\n            --train_file ./tests/fixtures/tests_samples/MRPC/train.csv\n            --validation_file ./tests/fixtures/tests_samples/MRPC/dev.csv\n            --per_device_train_batch_size=2\n            --per_device_eval_batch_size=1\n            --learning_rate=1e-4\n            --seed=42\n            --checkpointing_steps epoch\n            --with_tracking\n        ".split()

											if is_cuda_and_apex_available():
																		testargs.append('''--fp16'''		)

											run_command(self._launch_args + testargs		)
											SCREAMING_SNAKE_CASE_   	=    get_results(_lowerCamelCase		)
											self.assertGreaterEqual(result['''eval_accuracy''']		,					0.75		)
											self.assertTrue(os.path.exists(os.path.join(_lowerCamelCase		,					'''epoch_0'''		)		)		)
											self.assertTrue(os.path.exists(os.path.join(_lowerCamelCase		,					'''glue_no_trainer'''		)		)		)


				@mock.patch.dict(os.environ		,					{'''WANDB_MODE''': '''offline'''}		)
				def     _A							(							self:   Optional[int]		):
											SCREAMING_SNAKE_CASE_   	=    self.get_auto_remove_tmp_dir()
											SCREAMING_SNAKE_CASE_   	=    f"\n            {self.examples_dir}/pytorch/language-modeling/run_clm_no_trainer.py\n            --model_name_or_path distilgpt2\n            --train_file ./tests/fixtures/sample_text.txt\n            --validation_file ./tests/fixtures/sample_text.txt\n            --block_size 128\n            --per_device_train_batch_size 5\n            --per_device_eval_batch_size 5\n            --num_train_epochs 2\n            --output_dir {tmp_dir}\n            --checkpointing_steps epoch\n            --with_tracking\n        ".split()

											if torch.cuda.device_count() > 1:
																		# Skipping because there are not enough batches to train the model + would need a drop_last to work.
																		return

											run_command(self._launch_args + testargs		)
											SCREAMING_SNAKE_CASE_   	=    get_results(_lowerCamelCase		)
											self.assertLess(result['''perplexity''']		,					1_00		)
											self.assertTrue(os.path.exists(os.path.join(_lowerCamelCase		,					'''epoch_0'''		)		)		)
											self.assertTrue(os.path.exists(os.path.join(_lowerCamelCase		,					'''clm_no_trainer'''		)		)		)


				@mock.patch.dict(os.environ		,					{'''WANDB_MODE''': '''offline'''}		)
				def     _A							(							self:   Dict		):
											SCREAMING_SNAKE_CASE_   	=    self.get_auto_remove_tmp_dir()
											SCREAMING_SNAKE_CASE_   	=    f"\n            {self.examples_dir}/pytorch/language-modeling/run_mlm_no_trainer.py\n            --model_name_or_path distilroberta-base\n            --train_file ./tests/fixtures/sample_text.txt\n            --validation_file ./tests/fixtures/sample_text.txt\n            --output_dir {tmp_dir}\n            --num_train_epochs=1\n            --checkpointing_steps epoch\n            --with_tracking\n        ".split()

											run_command(self._launch_args + testargs		)
											SCREAMING_SNAKE_CASE_   	=    get_results(_lowerCamelCase		)
											self.assertLess(result['''perplexity''']		,					42		)
											self.assertTrue(os.path.exists(os.path.join(_lowerCamelCase		,					'''epoch_0'''		)		)		)
											self.assertTrue(os.path.exists(os.path.join(_lowerCamelCase		,					'''mlm_no_trainer'''		)		)		)


				@mock.patch.dict(os.environ		,					{'''WANDB_MODE''': '''offline'''}		)
				def     _A							(							self:   Optional[Any]		):
											# with so little data distributed training needs more epochs to get the score on par with 0/1 gpu
											SCREAMING_SNAKE_CASE_   	=    7 if get_gpu_count() > 1 else 2

											SCREAMING_SNAKE_CASE_   	=    self.get_auto_remove_tmp_dir()
											SCREAMING_SNAKE_CASE_   	=    f"\n            {self.examples_dir}/pytorch/token-classification/run_ner_no_trainer.py\n            --model_name_or_path bert-base-uncased\n            --train_file tests/fixtures/tests_samples/conll/sample.json\n            --validation_file tests/fixtures/tests_samples/conll/sample.json\n            --output_dir {tmp_dir}\n            --learning_rate=2e-4\n            --per_device_train_batch_size=2\n            --per_device_eval_batch_size=2\n            --num_train_epochs={epochs}\n            --seed 7\n            --checkpointing_steps epoch\n            --with_tracking\n        ".split()

											run_command(self._launch_args + testargs		)
											SCREAMING_SNAKE_CASE_   	=    get_results(_lowerCamelCase		)
											self.assertGreaterEqual(result['''eval_accuracy''']		,					0.75		)
											self.assertLess(result['''train_loss''']		,					0.5		)
											self.assertTrue(os.path.exists(os.path.join(_lowerCamelCase		,					'''epoch_0'''		)		)		)
											self.assertTrue(os.path.exists(os.path.join(_lowerCamelCase		,					'''ner_no_trainer'''		)		)		)


				@unittest.skip(reason='''Fix me @muellerzr'''		)
				@mock.patch.dict(os.environ		,					{'''WANDB_MODE''': '''offline'''}		)
				def     _A							(							self:   str		):
											SCREAMING_SNAKE_CASE_   	=    self.get_auto_remove_tmp_dir()
											SCREAMING_SNAKE_CASE_   	=    f"\n            {self.examples_dir}/pytorch/question-answering/run_qa_no_trainer.py\n            --model_name_or_path bert-base-uncased\n            --version_2_with_negative\n            --train_file tests/fixtures/tests_samples/SQUAD/sample.json\n            --validation_file tests/fixtures/tests_samples/SQUAD/sample.json\n            --output_dir {tmp_dir}\n            --seed=42\n            --max_train_steps=10\n            --num_warmup_steps=2\n            --learning_rate=2e-4\n            --per_device_train_batch_size=2\n            --per_device_eval_batch_size=1\n            --checkpointing_steps epoch\n            --with_tracking\n        ".split()

											run_command(self._launch_args + testargs		)
											SCREAMING_SNAKE_CASE_   	=    get_results(_lowerCamelCase		)
											# Because we use --version_2_with_negative the testing script uses SQuAD v2 metrics.
											self.assertGreaterEqual(result['''eval_f1''']		,					28		)
											self.assertGreaterEqual(result['''eval_exact''']		,					28		)
											self.assertTrue(os.path.exists(os.path.join(_lowerCamelCase		,					'''epoch_0'''		)		)		)
											self.assertTrue(os.path.exists(os.path.join(_lowerCamelCase		,					'''qa_no_trainer'''		)		)		)


				@mock.patch.dict(os.environ		,					{'''WANDB_MODE''': '''offline'''}		)
				def     _A							(							self:   str		):
											SCREAMING_SNAKE_CASE_   	=    self.get_auto_remove_tmp_dir()
											SCREAMING_SNAKE_CASE_   	=    f"\n            {self.examples_dir}/pytorch/multiple-choice/run_swag_no_trainer.py\n            --model_name_or_path bert-base-uncased\n            --train_file tests/fixtures/tests_samples/swag/sample.json\n            --validation_file tests/fixtures/tests_samples/swag/sample.json\n            --output_dir {tmp_dir}\n            --max_train_steps=20\n            --num_warmup_steps=2\n            --learning_rate=2e-4\n            --per_device_train_batch_size=2\n            --per_device_eval_batch_size=1\n            --with_tracking\n        ".split()

											run_command(self._launch_args + testargs		)
											SCREAMING_SNAKE_CASE_   	=    get_results(_lowerCamelCase		)
											self.assertGreaterEqual(result['''eval_accuracy''']		,					0.8		)
											self.assertTrue(os.path.exists(os.path.join(_lowerCamelCase		,					'''swag_no_trainer'''		)		)		)


				@slow
				@mock.patch.dict(os.environ		,					{'''WANDB_MODE''': '''offline'''}		)
				def     _A							(							self:   Optional[Any]		):
											SCREAMING_SNAKE_CASE_   	=    self.get_auto_remove_tmp_dir()
											SCREAMING_SNAKE_CASE_   	=    f"\n            {self.examples_dir}/pytorch/summarization/run_summarization_no_trainer.py\n            --model_name_or_path t5-small\n            --train_file tests/fixtures/tests_samples/xsum/sample.json\n            --validation_file tests/fixtures/tests_samples/xsum/sample.json\n            --output_dir {tmp_dir}\n            --max_train_steps=50\n            --num_warmup_steps=8\n            --learning_rate=2e-4\n            --per_device_train_batch_size=2\n            --per_device_eval_batch_size=1\n            --checkpointing_steps epoch\n            --with_tracking\n        ".split()

											run_command(self._launch_args + testargs		)
											SCREAMING_SNAKE_CASE_   	=    get_results(_lowerCamelCase		)
											self.assertGreaterEqual(result['''eval_rouge1''']		,					10		)
											self.assertGreaterEqual(result['''eval_rouge2''']		,					2		)
											self.assertGreaterEqual(result['''eval_rougeL''']		,					7		)
											self.assertGreaterEqual(result['''eval_rougeLsum''']		,					7		)
											self.assertTrue(os.path.exists(os.path.join(_lowerCamelCase		,					'''epoch_0'''		)		)		)
											self.assertTrue(os.path.exists(os.path.join(_lowerCamelCase		,					'''summarization_no_trainer'''		)		)		)


				@slow
				@mock.patch.dict(os.environ		,					{'''WANDB_MODE''': '''offline'''}		)
				def     _A							(							self:   Any		):
											SCREAMING_SNAKE_CASE_   	=    self.get_auto_remove_tmp_dir()
											SCREAMING_SNAKE_CASE_   	=    f"\n            {self.examples_dir}/pytorch/translation/run_translation_no_trainer.py\n            --model_name_or_path sshleifer/student_marian_en_ro_6_1\n            --source_lang en\n            --target_lang ro\n            --train_file tests/fixtures/tests_samples/wmt16/sample.json\n            --validation_file tests/fixtures/tests_samples/wmt16/sample.json\n            --output_dir {tmp_dir}\n            --max_train_steps=50\n            --num_warmup_steps=8\n            --num_beams=6\n            --learning_rate=3e-3\n            --per_device_train_batch_size=2\n            --per_device_eval_batch_size=1\n            --source_lang en_XX\n            --target_lang ro_RO\n            --checkpointing_steps epoch\n            --with_tracking\n        ".split()

											run_command(self._launch_args + testargs		)
											SCREAMING_SNAKE_CASE_   	=    get_results(_lowerCamelCase		)
											self.assertGreaterEqual(result['''eval_bleu''']		,					30		)
											self.assertTrue(os.path.exists(os.path.join(_lowerCamelCase		,					'''epoch_0'''		)		)		)
											self.assertTrue(os.path.exists(os.path.join(_lowerCamelCase		,					'''translation_no_trainer'''		)		)		)


				@slow
				def     _A							(							self:   Tuple		):
											SCREAMING_SNAKE_CASE_   	=    logging.StreamHandler(sys.stdout		)
											logger.addHandler(_lowerCamelCase		)

											SCREAMING_SNAKE_CASE_   	=    self.get_auto_remove_tmp_dir()
											SCREAMING_SNAKE_CASE_   	=    f"\n            {self.examples_dir}/pytorch/semantic-segmentation/run_semantic_segmentation_no_trainer.py\n            --dataset_name huggingface/semantic-segmentation-test-sample\n            --output_dir {tmp_dir}\n            --max_train_steps=10\n            --num_warmup_steps=2\n            --learning_rate=2e-4\n            --per_device_train_batch_size=2\n            --per_device_eval_batch_size=1\n            --checkpointing_steps epoch\n        ".split()

											run_command(self._launch_args + testargs		)
											SCREAMING_SNAKE_CASE_   	=    get_results(_lowerCamelCase		)
											self.assertGreaterEqual(result['''eval_overall_accuracy''']		,					0.10		)





				@mock.patch.dict(os.environ		,					{'''WANDB_MODE''': '''offline'''}		)
				def     _A							(							self:   Union[str, Any]		):
											SCREAMING_SNAKE_CASE_   	=    self.get_auto_remove_tmp_dir()
											SCREAMING_SNAKE_CASE_   	=    f"\n            {self.examples_dir}/pytorch/image-classification/run_image_classification_no_trainer.py\n            --model_name_or_path google/vit-base-patch16-224-in21k\n            --dataset_name hf-internal-testing/cats_vs_dogs_sample\n            --learning_rate 1e-4\n            --per_device_train_batch_size 2\n            --per_device_eval_batch_size 1\n            --max_train_steps 2\n            --train_val_split 0.1\n            --seed 42\n            --output_dir {tmp_dir}\n            --with_tracking\n            --checkpointing_steps 1\n        ".split()

											if is_cuda_and_apex_available():
																		testargs.append('''--fp16'''		)

											run_command(self._launch_args + testargs		)
											SCREAMING_SNAKE_CASE_   	=    get_results(_lowerCamelCase		)
											# The base model scores a 25%
											self.assertGreaterEqual(result['''eval_accuracy''']		,					0.6		)
											self.assertTrue(os.path.exists(os.path.join(_lowerCamelCase		,					'''step_1'''		)		)		)
											self.assertTrue(os.path.exists(os.path.join(_lowerCamelCase		,					'''image_classification_no_trainer'''		)		)		)








import math
import time
from typing import Dict, List, Optional

from torch.utils.data import Dataset

from transformers import SeqaSeqTrainer, is_torch_tpu_available
from transformers.trainer_utils import PredictionOutput, speed_metrics


if is_torch_tpu_available(check_device=False):
			import torch_xla.core.xla_model as xm
			import torch_xla.debug.metrics as met






class  __magic_name__						(  __UpperCAmelCase):
				'''simple docstring'''







				def __init__(							self:   List[Any]		,					*_lowerCamelCase:   Tuple		,					_lowerCamelCase:   Optional[Any]=None		,					_lowerCamelCase:   Dict=None		,					**_lowerCamelCase:   Dict		):
											super().__init__(*_lowerCamelCase		,					**_lowerCamelCase		)
											SCREAMING_SNAKE_CASE_   	=    eval_examples
											SCREAMING_SNAKE_CASE_   	=    post_process_function


				def     _A							(							self:   Tuple		,					_lowerCamelCase:   Optional[Dataset] = None		,					_lowerCamelCase:   List[str]=None		,					_lowerCamelCase:   Optional[List[str]] = None		,					_lowerCamelCase:   str = "eval"		,					**_lowerCamelCase:   Union[str, Any]		,					):
											SCREAMING_SNAKE_CASE_   	=    gen_kwargs.copy()
											SCREAMING_SNAKE_CASE_   	=    (
											    gen_kwargs['''max_length'''] if gen_kwargs.get('''max_length'''		) is not None else self.args.generation_max_length
											)
											SCREAMING_SNAKE_CASE_   	=    (
											    gen_kwargs['''num_beams'''] if gen_kwargs.get('''num_beams'''		) is not None else self.args.generation_num_beams
											)
											SCREAMING_SNAKE_CASE_   	=    gen_kwargs

											SCREAMING_SNAKE_CASE_   	=    self.eval_dataset if eval_dataset is None else eval_dataset
											SCREAMING_SNAKE_CASE_   	=    self.get_eval_dataloader(_lowerCamelCase		)
											SCREAMING_SNAKE_CASE_   	=    self.eval_examples if eval_examples is None else eval_examples

											# Temporarily disable metric computation, we will do it in the loop here.
											SCREAMING_SNAKE_CASE_   	=    self.compute_metrics
											SCREAMING_SNAKE_CASE_   	=    None
											SCREAMING_SNAKE_CASE_   	=    time.time()
											SCREAMING_SNAKE_CASE_   	=    self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop
											try:
																		SCREAMING_SNAKE_CASE_   	=    eval_loop(
																		    _lowerCamelCase		,					description='''Evaluation'''		,					prediction_loss_only=True if compute_metrics is None else None		,					ignore_keys=_lowerCamelCase		,					metric_key_prefix=_lowerCamelCase		,					)
											finally:
																		SCREAMING_SNAKE_CASE_   	=    compute_metrics
											SCREAMING_SNAKE_CASE_   	=    self.args.eval_batch_size * self.args.world_size
											if f"{metric_key_prefix}_jit_compilation_time" in output.metrics:
																		start_time += output.metrics[f"{metric_key_prefix}_jit_compilation_time"]
											output.metrics.update(
											    speed_metrics(
											        _lowerCamelCase		,					_lowerCamelCase		,					num_samples=output.num_samples		,					num_steps=math.ceil(output.num_samples / total_batch_size		)		,					)		)

											if self.post_process_function is not None and self.compute_metrics is not None and self.args.should_save:
																		# Only the main node write the results by default
																		SCREAMING_SNAKE_CASE_   	=    self.post_process_function(_lowerCamelCase		,					_lowerCamelCase		,					_lowerCamelCase		)
																		SCREAMING_SNAKE_CASE_   	=    self.compute_metrics(_lowerCamelCase		)

																		# Prefix all keys with metric_key_prefix + '_'
																		for key in list(metrics.keys()		):
																									if not key.startswith(f"{metric_key_prefix}_"		):
																																SCREAMING_SNAKE_CASE_   	=    metrics.pop(_lowerCamelCase		)

																		metrics.update(output.metrics		)
											else:
																		SCREAMING_SNAKE_CASE_   	=    output.metrics

											if self.args.should_log:
																		# Only the main node log the results by default
																		self.log(_lowerCamelCase		)

											if self.args.tpu_metrics_debug or self.args.debug:
																		# tpu-comment: Logging debug metrics for PyTorch/XLA (compile, execute times, ops, etc.)
																		xm.master_print(met.metrics_report()		)

											SCREAMING_SNAKE_CASE_   	=    self.callback_handler.on_evaluate(self.args		,					self.state		,					self.control		,					_lowerCamelCase		)
											return metrics





				def     _A							(							self:   List[str]		,					_lowerCamelCase:   int		,					_lowerCamelCase:   List[str]		,					_lowerCamelCase:   Optional[int]=None		,					_lowerCamelCase:   str = "test"		,					**_lowerCamelCase:   str		):
											SCREAMING_SNAKE_CASE_   	=    gen_kwargs.copy()

											SCREAMING_SNAKE_CASE_   	=    self.get_test_dataloader(_lowerCamelCase		)

											# Temporarily disable metric computation, we will do it in the loop here.
											SCREAMING_SNAKE_CASE_   	=    self.compute_metrics
											SCREAMING_SNAKE_CASE_   	=    None
											SCREAMING_SNAKE_CASE_   	=    time.time()
											SCREAMING_SNAKE_CASE_   	=    self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop
											try:
																		SCREAMING_SNAKE_CASE_   	=    eval_loop(
																		    _lowerCamelCase		,					description='''Prediction'''		,					prediction_loss_only=True if compute_metrics is None else None		,					ignore_keys=_lowerCamelCase		,					metric_key_prefix=_lowerCamelCase		,					)
											finally:
																		SCREAMING_SNAKE_CASE_   	=    compute_metrics

											SCREAMING_SNAKE_CASE_   	=    self.args.eval_batch_size * self.args.world_size
											if f"{metric_key_prefix}_jit_compilation_time" in output.metrics:
																		start_time += output.metrics[f"{metric_key_prefix}_jit_compilation_time"]
											output.metrics.update(
											    speed_metrics(
											        _lowerCamelCase		,					_lowerCamelCase		,					num_samples=output.num_samples		,					num_steps=math.ceil(output.num_samples / total_batch_size		)		,					)		)
											if self.post_process_function is None or self.compute_metrics is None:
																		return output

											SCREAMING_SNAKE_CASE_   	=    self.post_process_function(_lowerCamelCase		,					_lowerCamelCase		,					_lowerCamelCase		,					'''predict'''		)
											SCREAMING_SNAKE_CASE_   	=    self.compute_metrics(_lowerCamelCase		)

											# Prefix all keys with metric_key_prefix + '_'
											for key in list(metrics.keys()		):
																		if not key.startswith(f"{metric_key_prefix}_"		):
																									SCREAMING_SNAKE_CASE_   	=    metrics.pop(_lowerCamelCase		)
											metrics.update(output.metrics		)
											return PredictionOutput(predictions=predictions.predictions		,					label_ids=predictions.label_ids		,					metrics=_lowerCamelCase		)


