




import argparse
import json
import logging
import os
import shutil
import sys
import tempfile
import unittest
from unittest import mock

import torch
from accelerate.utils import write_basic_config

from transformers.testing_utils import TestCasePlus, get_gpu_count, run_command, slow, torch_device
from transformers.utils import is_apex_available


logging.basicConfig(level=logging.DEBUG)

UpperCAmelCase__				     = logging.getLogger()







def 		_A(       )    -> Dict:
							'''simple docstring'''


							__lowercase      =    argparse.ArgumentParser()
							parser.add_argument('''-f'''				)
							__lowercase      =    parser.parse_args()
							return args.f







def 		_A(       UpperCamelCase__							:			Optional[int]				)    -> int:
							'''simple docstring'''


							__lowercase      =    {}
							__lowercase      =    os.path.join(UpperCamelCase__				,       '''all_results.json'''				)
							if os.path.exists(UpperCamelCase__				):
														with open(UpperCamelCase__				,       '''r'''				) as f:
																					__lowercase      =    json.load(UpperCamelCase__				)
							else:
														raise ValueError(F'can\'t find {path}'				)
							return results







def 		_A(       )    -> List[Any]:
							'''simple docstring'''


							__lowercase      =    torch.cuda.is_available() and torch_device == '''cuda'''
							return is_using_cuda and is_apex_available()


UpperCAmelCase__				     = logging.StreamHandler(sys.stdout)
logger.addHandler(stream_handler)







class a (     __SCREAMING_SNAKE_CASE  ):
						"""simple docstring"""

						@classmethod
						def     UpperCAmelCase_      ( cls :     List[Any]				)    ->						Optional[Any]:
													"""simple docstring"""


													__lowercase      =    tempfile.mkdtemp()
													__lowercase      =    os.path.join(cls.tmpdir				,							'''default_config.yml'''				)
													write_basic_config(save_location=cls.configPath				)
													__lowercase      =    ['''accelerate''', '''launch''', '''--config_file''', cls.configPath]






						@classmethod
						def     UpperCAmelCase_      ( cls :     Dict				)    ->						int:
													"""simple docstring"""


													shutil.rmtree(cls.tmpdir				)






						@mock.patch.dict(os.environ				,							{'''WANDB_MODE''': '''offline'''}				)
						def     UpperCAmelCase_      ( self :     Tuple				)    ->						Union[str, Any]:
													"""simple docstring"""


													__lowercase      =    self.get_auto_remove_tmp_dir()
													__lowercase      =    f'\n            {self.examples_dir}/pytorch/text-classification/run_glue_no_trainer.py\n            --model_name_or_path distilbert-base-uncased\n            --output_dir {tmp_dir}\n            --train_file ./tests/fixtures/tests_samples/MRPC/train.csv\n            --validation_file ./tests/fixtures/tests_samples/MRPC/dev.csv\n            --per_device_train_batch_size=2\n            --per_device_eval_batch_size=1\n            --learning_rate=1e-4\n            --seed=42\n            --checkpointing_steps epoch\n            --with_tracking\n        '.split()

													if is_cuda_and_apex_available():
																				testargs.append('''--fp16'''				)

													run_command(self._launch_args + testargs				)
													__lowercase      =    get_results(lowerCamelCase__				)
													self.assertGreaterEqual(result['''eval_accuracy''']				,							0.7_5				)
													self.assertTrue(os.path.exists(os.path.join(lowerCamelCase__				,							'''epoch_0'''				)				)				)
													self.assertTrue(os.path.exists(os.path.join(lowerCamelCase__				,							'''glue_no_trainer'''				)				)				)






						@mock.patch.dict(os.environ				,							{'''WANDB_MODE''': '''offline'''}				)
						def     UpperCAmelCase_      ( self :     List[Any]				)    ->						List[str]:
													"""simple docstring"""


													__lowercase      =    self.get_auto_remove_tmp_dir()
													__lowercase      =    f'\n            {self.examples_dir}/pytorch/language-modeling/run_clm_no_trainer.py\n            --model_name_or_path distilgpt2\n            --train_file ./tests/fixtures/sample_text.txt\n            --validation_file ./tests/fixtures/sample_text.txt\n            --block_size 128\n            --per_device_train_batch_size 5\n            --per_device_eval_batch_size 5\n            --num_train_epochs 2\n            --output_dir {tmp_dir}\n            --checkpointing_steps epoch\n            --with_tracking\n        '.split()

													if torch.cuda.device_count() > 1:
																				# Skipping because there are not enough batches to train the model + would need a drop_last to work.
																				return

													run_command(self._launch_args + testargs				)
													__lowercase      =    get_results(lowerCamelCase__				)
													self.assertLess(result['''perplexity''']				,							100				)
													self.assertTrue(os.path.exists(os.path.join(lowerCamelCase__				,							'''epoch_0'''				)				)				)
													self.assertTrue(os.path.exists(os.path.join(lowerCamelCase__				,							'''clm_no_trainer'''				)				)				)






						@mock.patch.dict(os.environ				,							{'''WANDB_MODE''': '''offline'''}				)
						def     UpperCAmelCase_      ( self :     Any				)    ->						Any:
													"""simple docstring"""


													__lowercase      =    self.get_auto_remove_tmp_dir()
													__lowercase      =    f'\n            {self.examples_dir}/pytorch/language-modeling/run_mlm_no_trainer.py\n            --model_name_or_path distilroberta-base\n            --train_file ./tests/fixtures/sample_text.txt\n            --validation_file ./tests/fixtures/sample_text.txt\n            --output_dir {tmp_dir}\n            --num_train_epochs=1\n            --checkpointing_steps epoch\n            --with_tracking\n        '.split()

													run_command(self._launch_args + testargs				)
													__lowercase      =    get_results(lowerCamelCase__				)
													self.assertLess(result['''perplexity''']				,							42				)
													self.assertTrue(os.path.exists(os.path.join(lowerCamelCase__				,							'''epoch_0'''				)				)				)
													self.assertTrue(os.path.exists(os.path.join(lowerCamelCase__				,							'''mlm_no_trainer'''				)				)				)






						@mock.patch.dict(os.environ				,							{'''WANDB_MODE''': '''offline'''}				)
						def     UpperCAmelCase_      ( self :     str				)    ->						Optional[Any]:
													"""simple docstring"""


													__lowercase      =    7 if get_gpu_count() > 1 else 2

													__lowercase      =    self.get_auto_remove_tmp_dir()
													__lowercase      =    f'\n            {self.examples_dir}/pytorch/token-classification/run_ner_no_trainer.py\n            --model_name_or_path bert-base-uncased\n            --train_file tests/fixtures/tests_samples/conll/sample.json\n            --validation_file tests/fixtures/tests_samples/conll/sample.json\n            --output_dir {tmp_dir}\n            --learning_rate=2e-4\n            --per_device_train_batch_size=2\n            --per_device_eval_batch_size=2\n            --num_train_epochs={epochs}\n            --seed 7\n            --checkpointing_steps epoch\n            --with_tracking\n        '.split()

													run_command(self._launch_args + testargs				)
													__lowercase      =    get_results(lowerCamelCase__				)
													self.assertGreaterEqual(result['''eval_accuracy''']				,							0.7_5				)
													self.assertLess(result['''train_loss''']				,							0.5				)
													self.assertTrue(os.path.exists(os.path.join(lowerCamelCase__				,							'''epoch_0'''				)				)				)
													self.assertTrue(os.path.exists(os.path.join(lowerCamelCase__				,							'''ner_no_trainer'''				)				)				)






						@unittest.skip(reason='''Fix me @muellerzr'''				)
						@mock.patch.dict(os.environ				,							{'''WANDB_MODE''': '''offline'''}				)
						def     UpperCAmelCase_      ( self :     str				)    ->						int:
													"""simple docstring"""


													__lowercase      =    self.get_auto_remove_tmp_dir()
													__lowercase      =    f'\n            {self.examples_dir}/pytorch/question-answering/run_qa_no_trainer.py\n            --model_name_or_path bert-base-uncased\n            --version_2_with_negative\n            --train_file tests/fixtures/tests_samples/SQUAD/sample.json\n            --validation_file tests/fixtures/tests_samples/SQUAD/sample.json\n            --output_dir {tmp_dir}\n            --seed=42\n            --max_train_steps=10\n            --num_warmup_steps=2\n            --learning_rate=2e-4\n            --per_device_train_batch_size=2\n            --per_device_eval_batch_size=1\n            --checkpointing_steps epoch\n            --with_tracking\n        '.split()

													run_command(self._launch_args + testargs				)
													__lowercase      =    get_results(lowerCamelCase__				)
													# Because we use --version_2_with_negative the testing script uses SQuAD v2 metrics.
													self.assertGreaterEqual(result['''eval_f1''']				,							28				)
													self.assertGreaterEqual(result['''eval_exact''']				,							28				)
													self.assertTrue(os.path.exists(os.path.join(lowerCamelCase__				,							'''epoch_0'''				)				)				)
													self.assertTrue(os.path.exists(os.path.join(lowerCamelCase__				,							'''qa_no_trainer'''				)				)				)






						@mock.patch.dict(os.environ				,							{'''WANDB_MODE''': '''offline'''}				)
						def     UpperCAmelCase_      ( self :     Optional[Any]				)    ->						List[Any]:
													"""simple docstring"""


													__lowercase      =    self.get_auto_remove_tmp_dir()
													__lowercase      =    f'\n            {self.examples_dir}/pytorch/multiple-choice/run_swag_no_trainer.py\n            --model_name_or_path bert-base-uncased\n            --train_file tests/fixtures/tests_samples/swag/sample.json\n            --validation_file tests/fixtures/tests_samples/swag/sample.json\n            --output_dir {tmp_dir}\n            --max_train_steps=20\n            --num_warmup_steps=2\n            --learning_rate=2e-4\n            --per_device_train_batch_size=2\n            --per_device_eval_batch_size=1\n            --with_tracking\n        '.split()

													run_command(self._launch_args + testargs				)
													__lowercase      =    get_results(lowerCamelCase__				)
													self.assertGreaterEqual(result['''eval_accuracy''']				,							0.8				)
													self.assertTrue(os.path.exists(os.path.join(lowerCamelCase__				,							'''swag_no_trainer'''				)				)				)






						@slow
						@mock.patch.dict(os.environ				,							{'''WANDB_MODE''': '''offline'''}				)
						def     UpperCAmelCase_      ( self :     Optional[int]				)    ->						Tuple:
													"""simple docstring"""


													__lowercase      =    self.get_auto_remove_tmp_dir()
													__lowercase      =    f'\n            {self.examples_dir}/pytorch/summarization/run_summarization_no_trainer.py\n            --model_name_or_path t5-small\n            --train_file tests/fixtures/tests_samples/xsum/sample.json\n            --validation_file tests/fixtures/tests_samples/xsum/sample.json\n            --output_dir {tmp_dir}\n            --max_train_steps=50\n            --num_warmup_steps=8\n            --learning_rate=2e-4\n            --per_device_train_batch_size=2\n            --per_device_eval_batch_size=1\n            --checkpointing_steps epoch\n            --with_tracking\n        '.split()

													run_command(self._launch_args + testargs				)
													__lowercase      =    get_results(lowerCamelCase__				)
													self.assertGreaterEqual(result['''eval_rouge1''']				,							10				)
													self.assertGreaterEqual(result['''eval_rouge2''']				,							2				)
													self.assertGreaterEqual(result['''eval_rougeL''']				,							7				)
													self.assertGreaterEqual(result['''eval_rougeLsum''']				,							7				)
													self.assertTrue(os.path.exists(os.path.join(lowerCamelCase__				,							'''epoch_0'''				)				)				)
													self.assertTrue(os.path.exists(os.path.join(lowerCamelCase__				,							'''summarization_no_trainer'''				)				)				)






						@slow
						@mock.patch.dict(os.environ				,							{'''WANDB_MODE''': '''offline'''}				)
						def     UpperCAmelCase_      ( self :     Optional[int]				)    ->						Union[str, Any]:
													"""simple docstring"""


													__lowercase      =    self.get_auto_remove_tmp_dir()
													__lowercase      =    f'\n            {self.examples_dir}/pytorch/translation/run_translation_no_trainer.py\n            --model_name_or_path sshleifer/student_marian_en_ro_6_1\n            --source_lang en\n            --target_lang ro\n            --train_file tests/fixtures/tests_samples/wmt16/sample.json\n            --validation_file tests/fixtures/tests_samples/wmt16/sample.json\n            --output_dir {tmp_dir}\n            --max_train_steps=50\n            --num_warmup_steps=8\n            --num_beams=6\n            --learning_rate=3e-3\n            --per_device_train_batch_size=2\n            --per_device_eval_batch_size=1\n            --source_lang en_XX\n            --target_lang ro_RO\n            --checkpointing_steps epoch\n            --with_tracking\n        '.split()

													run_command(self._launch_args + testargs				)
													__lowercase      =    get_results(lowerCamelCase__				)
													self.assertGreaterEqual(result['''eval_bleu''']				,							30				)
													self.assertTrue(os.path.exists(os.path.join(lowerCamelCase__				,							'''epoch_0'''				)				)				)
													self.assertTrue(os.path.exists(os.path.join(lowerCamelCase__				,							'''translation_no_trainer'''				)				)				)






						@slow
						def     UpperCAmelCase_      ( self :     str				)    ->						List[str]:
													"""simple docstring"""


													__lowercase      =    logging.StreamHandler(sys.stdout				)
													logger.addHandler(lowerCamelCase__				)

													__lowercase      =    self.get_auto_remove_tmp_dir()
													__lowercase      =    f'\n            {self.examples_dir}/pytorch/semantic-segmentation/run_semantic_segmentation_no_trainer.py\n            --dataset_name huggingface/semantic-segmentation-test-sample\n            --output_dir {tmp_dir}\n            --max_train_steps=10\n            --num_warmup_steps=2\n            --learning_rate=2e-4\n            --per_device_train_batch_size=2\n            --per_device_eval_batch_size=1\n            --checkpointing_steps epoch\n        '.split()

													run_command(self._launch_args + testargs				)
													__lowercase      =    get_results(lowerCamelCase__				)
													self.assertGreaterEqual(result['''eval_overall_accuracy''']				,							0.1_0				)






						@mock.patch.dict(os.environ				,							{'''WANDB_MODE''': '''offline'''}				)
						def     UpperCAmelCase_      ( self :     Any				)    ->						Optional[Any]:
													"""simple docstring"""


													__lowercase      =    self.get_auto_remove_tmp_dir()
													__lowercase      =    f'\n            {self.examples_dir}/pytorch/image-classification/run_image_classification_no_trainer.py\n            --model_name_or_path google/vit-base-patch16-224-in21k\n            --dataset_name hf-internal-testing/cats_vs_dogs_sample\n            --learning_rate 1e-4\n            --per_device_train_batch_size 2\n            --per_device_eval_batch_size 1\n            --max_train_steps 2\n            --train_val_split 0.1\n            --seed 42\n            --output_dir {tmp_dir}\n            --with_tracking\n            --checkpointing_steps 1\n        '.split()

													if is_cuda_and_apex_available():
																				testargs.append('''--fp16'''				)

													run_command(self._launch_args + testargs				)
													__lowercase      =    get_results(lowerCamelCase__				)
													# The base model scores a 25%
													self.assertGreaterEqual(result['''eval_accuracy''']				,							0.6				)
													self.assertTrue(os.path.exists(os.path.join(lowerCamelCase__				,							'''step_1'''				)				)				)
													self.assertTrue(os.path.exists(os.path.join(lowerCamelCase__				,							'''image_classification_no_trainer'''				)				)				)












# This code is adapted from OpenAI's release
# https://github.com/openai/human-eval/blob/master/human_eval/execution.py

import contextlib
import faulthandler
import io
import multiprocessing
import os
import platform
import signal
import tempfile







def 		_A(       UpperCamelCase__							:			Any				,       UpperCamelCase__							:			int				,       UpperCamelCase__							:			List[Any]				,       UpperCamelCase__							:			int				)    -> List[Any]:
							'''simple docstring'''


							__lowercase      =    multiprocessing.Manager()
							__lowercase      =    manager.list()

							__lowercase      =    multiprocessing.Process(target=UpperCamelCase__				,       args=(check_program, result, timeout)				)
							p.start()
							p.join(timeout=timeout + 1				)
							if p.is_alive():
														p.kill()

							if not result:
														result.append('''timed out'''				)

							return {
							    "task_id": task_id,
							    "passed": result[0] == "passed",
							    "result": result[0],
							    "completion_id": completion_id,
							}







def 		_A(       UpperCamelCase__							:			int				,       UpperCamelCase__							:			List[Any]				,       UpperCamelCase__							:			Optional[int]				)    -> List[Any]:
							'''simple docstring'''


							with create_tempdir():
														# These system calls are needed when cleaning up tempdir.
														import os
														import shutil

														__lowercase      =    shutil.rmtree
														__lowercase      =    os.rmdir
														__lowercase      =    os.chdir

														# Disable functionalities that can make destructive changes to the test.
														reliability_guard()

														# Run program.
														try:
																					__lowercase      =    {}
																					with swallow_io():
																												with time_limit(UpperCamelCase__				):
																																			exec(UpperCamelCase__				,       UpperCamelCase__				)
																					result.append('''passed'''				)
														except TimeoutException:
																					result.append('''timed out'''				)
														except BaseException as e:
																					result.append(F'failed: {e}'				)

														# Needed for cleaning up.
														__lowercase      =    rmtree
														__lowercase      =    rmdir
														__lowercase      =    chdir







@contextlib.contextmanager
def 		_A(       UpperCamelCase__							:			Union[str, Any]				)    -> Optional[int]:
							'''simple docstring'''


							def signal_handler(UpperCamelCase__							:			List[str]				,       UpperCamelCase__							:			Optional[Any]				):
														raise TimeoutException('''Timed out!'''				)

							signal.setitimer(signal.ITIMER_REAL				,       UpperCamelCase__				)
							signal.signal(signal.SIGALRM				,       UpperCamelCase__				)
							try:
														yield
							finally:
														signal.setitimer(signal.ITIMER_REAL				,       0				)







@contextlib.contextmanager
def 		_A(       )    -> Any:
							'''simple docstring'''


							__lowercase      =    WriteOnlyStringIO()
							with contextlib.redirect_stdout(UpperCamelCase__				):
														with contextlib.redirect_stderr(UpperCamelCase__				):
																					with redirect_stdin(UpperCamelCase__				):
																												yield







@contextlib.contextmanager
def 		_A(       )    -> Tuple:
							'''simple docstring'''


							with tempfile.TemporaryDirectory() as dirname:
														with chdir(UpperCamelCase__				):
																					yield dirname







class a (     __SCREAMING_SNAKE_CASE  ):
						"""simple docstring"""

						pass







class a (     io.StringIO  ):
						"""simple docstring"""

						def     UpperCAmelCase_      ( self :     List[str]				,							*lowerCamelCase__ :     str				,							**lowerCamelCase__ :     str				)    ->						Tuple:
													"""simple docstring"""


													raise OSError






						def     UpperCAmelCase_      ( self :     List[str]				,							*lowerCamelCase__ :     List[str]				,							**lowerCamelCase__ :     Dict				)    ->						List[Any]:
													"""simple docstring"""


													raise OSError






						def     UpperCAmelCase_      ( self :     List[str]				,							*lowerCamelCase__ :     Dict				,							**lowerCamelCase__ :     Dict				)    ->						Optional[int]:
													"""simple docstring"""


													raise OSError






						def     UpperCAmelCase_      ( self :     Tuple				,							*lowerCamelCase__ :     Optional[Any]				,							**lowerCamelCase__ :     Tuple				)    ->						Optional[Any]:
													"""simple docstring"""


													return False







class a (     contextlib._RedirectStream  ):  # type: ignore
						"""simple docstring"""

						UpperCamelCase_	:       str       	=		'stdin'







@contextlib.contextmanager
def 		_A(       UpperCamelCase__							:			int				)    -> Any:
							'''simple docstring'''


							if root == ".":
														yield
														return
							__lowercase      =    os.getcwd()
							os.chdir(UpperCamelCase__				)
							try:
														yield
							except BaseException as exc:
														raise exc
							finally:
														os.chdir(UpperCamelCase__				)







def 		_A(       UpperCamelCase__							:			Optional[Any]=None				)    -> Any:
							'''simple docstring'''


							if maximum_memory_bytes is not None:
														import resource

														resource.setrlimit(resource.RLIMIT_AS				,       (maximum_memory_bytes, maximum_memory_bytes)				)
														resource.setrlimit(resource.RLIMIT_DATA				,       (maximum_memory_bytes, maximum_memory_bytes)				)
														if not platform.uname().system == "Darwin":
																					resource.setrlimit(resource.RLIMIT_STACK				,       (maximum_memory_bytes, maximum_memory_bytes)				)

							faulthandler.disable()

							import builtins

							__lowercase      =    None
							__lowercase      =    None

							import os

							__lowercase      =    '''1'''

							__lowercase      =    None
							__lowercase      =    None
							__lowercase      =    None
							__lowercase      =    None
							__lowercase      =    None
							__lowercase      =    None
							__lowercase      =    None
							__lowercase      =    None
							__lowercase      =    None
							__lowercase      =    None
							__lowercase      =    None
							__lowercase      =    None
							__lowercase      =    None
							__lowercase      =    None
							__lowercase      =    None
							__lowercase      =    None
							__lowercase      =    None
							__lowercase      =    None
							__lowercase      =    None
							__lowercase      =    None
							__lowercase      =    None
							__lowercase      =    None
							__lowercase      =    None
							__lowercase      =    None
							__lowercase      =    None
							__lowercase      =    None
							__lowercase      =    None

							import shutil

							__lowercase      =    None
							__lowercase      =    None
							__lowercase      =    None

							import subprocess

							__lowercase      =    None  # type: ignore

							__lowercase      =    None

							import sys

							__lowercase      =    None
							__lowercase      =    None
							__lowercase      =    None
							__lowercase      =    None
							__lowercase      =    None






