





import argparse
import json
import logging
import os
import shutil
import sys
import tempfile
import unittest
from unittest import mock

import torch
from accelerate.utils import write_basic_config

from transformers.testing_utils import TestCasePlus, get_gpu_count, run_command, slow, torch_device
from transformers.utils import is_apex_available


logging.basicConfig(level=logging.DEBUG)

__lowerCAmelCase :str     =    logging.getLogger()







def    A				(							):
					_snake_case       :					Any				       =   argparse.ArgumentParser()
					parser.add_argument("-f"				)
					_snake_case       :					Optional[Any]				       =   parser.parse_args()
					return args.f







def    A				(							UpperCAmelCase				):
					_snake_case       :					int				       =   {}
					_snake_case       :					str				       =   os.path.join(UpperCAmelCase  ,  "all_results.json"				)
					if os.path.exists(UpperCAmelCase				):
										with open(UpperCAmelCase  ,  "r"				) as f:
															_snake_case       :					List[str]				       =   json.load(UpperCAmelCase				)
					else:
										raise ValueError(F"""can't find {path}"""				)
					return results







def    A				(							):
					_snake_case       :					Tuple				       =   torch.cuda.is_available() and torch_device == "cuda"
					return is_using_cuda and is_apex_available()


__lowerCAmelCase :Dict     =    logging.StreamHandler(sys.stdout)
logger.addHandler(stream_handler)






class        _a(    __A    ):







		@classmethod
		def        lowercase       (    cls				)      ->      Tuple:


							'''simple docstring'''

							_snake_case       :					Tuple				       =   tempfile.mkdtemp()
							_snake_case       :					Union[str, Any]				       =   os.path.join(cls.tmpdir ,						"default_config.yml"				)
							write_basic_config(save_location=cls.configPath				)
							_snake_case       :					List[str]				       =   ["accelerate", "launch", "--config_file", cls.configPath]







		@classmethod
		def        lowercase       (    cls				)      ->      Union[str, Any]:


							'''simple docstring'''

							shutil.rmtree(cls.tmpdir				)







		@mock.patch.dict(os.environ ,						{"WANDB_MODE": "offline"}				)
		def        lowercase       (    self				)      ->      Tuple:


							'''simple docstring'''

							_snake_case       :					Any				       =   self.get_auto_remove_tmp_dir()
							_snake_case       :					int				       =   f"""
            {self.examples_dir}/pytorch/text-classification/run_glue_no_trainer.py
            --model_name_or_path distilbert-base-uncased
            --output_dir {tmp_dir}
            --train_file ./tests/fixtures/tests_samples/MRPC/train.csv
            --validation_file ./tests/fixtures/tests_samples/MRPC/dev.csv
            --per_device_train_batch_size=2
            --per_device_eval_batch_size=1
            --learning_rate=1e-4
            --seed=42
            --checkpointing_steps epoch
            --with_tracking
        """.split()

							if is_cuda_and_apex_available():
												testargs.append("--fp16"				)

							run_command(self._launch_args + testargs				)
							_snake_case       :					Optional[int]				       =   get_results(__snake_case				)
							self.assertGreaterEqual(result["eval_accuracy"] ,						0.75				)
							self.assertTrue(os.path.exists(os.path.join(__snake_case ,						"epoch_0"				)				)				)
							self.assertTrue(os.path.exists(os.path.join(__snake_case ,						"glue_no_trainer"				)				)				)







		@mock.patch.dict(os.environ ,						{"WANDB_MODE": "offline"}				)
		def        lowercase       (    self				)      ->      Dict:


							'''simple docstring'''

							_snake_case       :					str				       =   self.get_auto_remove_tmp_dir()
							_snake_case       :					Union[str, Any]				       =   f"""
            {self.examples_dir}/pytorch/language-modeling/run_clm_no_trainer.py
            --model_name_or_path distilgpt2
            --train_file ./tests/fixtures/sample_text.txt
            --validation_file ./tests/fixtures/sample_text.txt
            --block_size 128
            --per_device_train_batch_size 5
            --per_device_eval_batch_size 5
            --num_train_epochs 2
            --output_dir {tmp_dir}
            --checkpointing_steps epoch
            --with_tracking
        """.split()

							if torch.cuda.device_count() > 1:
												# Skipping because there are not enough batches to train the model + would need a drop_last to work.
												return

							run_command(self._launch_args + testargs				)
							_snake_case       :					List[Any]				       =   get_results(__snake_case				)
							self.assertLess(result["perplexity"] ,						1_0_0				)
							self.assertTrue(os.path.exists(os.path.join(__snake_case ,						"epoch_0"				)				)				)
							self.assertTrue(os.path.exists(os.path.join(__snake_case ,						"clm_no_trainer"				)				)				)







		@mock.patch.dict(os.environ ,						{"WANDB_MODE": "offline"}				)
		def        lowercase       (    self				)      ->      Tuple:


							'''simple docstring'''

							_snake_case       :					Optional[int]				       =   self.get_auto_remove_tmp_dir()
							_snake_case       :					Tuple				       =   f"""
            {self.examples_dir}/pytorch/language-modeling/run_mlm_no_trainer.py
            --model_name_or_path distilroberta-base
            --train_file ./tests/fixtures/sample_text.txt
            --validation_file ./tests/fixtures/sample_text.txt
            --output_dir {tmp_dir}
            --num_train_epochs=1
            --checkpointing_steps epoch
            --with_tracking
        """.split()

							run_command(self._launch_args + testargs				)
							_snake_case       :					List[str]				       =   get_results(__snake_case				)
							self.assertLess(result["perplexity"] ,						4_2				)
							self.assertTrue(os.path.exists(os.path.join(__snake_case ,						"epoch_0"				)				)				)
							self.assertTrue(os.path.exists(os.path.join(__snake_case ,						"mlm_no_trainer"				)				)				)







		@mock.patch.dict(os.environ ,						{"WANDB_MODE": "offline"}				)
		def        lowercase       (    self				)      ->      Any:


							'''simple docstring'''

							_snake_case       :					List[str]				       =   7 if get_gpu_count() > 1 else 2

							_snake_case       :					Optional[Any]				       =   self.get_auto_remove_tmp_dir()
							_snake_case       :					Dict				       =   f"""
            {self.examples_dir}/pytorch/token-classification/run_ner_no_trainer.py
            --model_name_or_path bert-base-uncased
            --train_file tests/fixtures/tests_samples/conll/sample.json
            --validation_file tests/fixtures/tests_samples/conll/sample.json
            --output_dir {tmp_dir}
            --learning_rate=2e-4
            --per_device_train_batch_size=2
            --per_device_eval_batch_size=2
            --num_train_epochs={epochs}
            --seed 7
            --checkpointing_steps epoch
            --with_tracking
        """.split()

							run_command(self._launch_args + testargs				)
							_snake_case       :					Any				       =   get_results(__snake_case				)
							self.assertGreaterEqual(result["eval_accuracy"] ,						0.75				)
							self.assertLess(result["train_loss"] ,						0.5				)
							self.assertTrue(os.path.exists(os.path.join(__snake_case ,						"epoch_0"				)				)				)
							self.assertTrue(os.path.exists(os.path.join(__snake_case ,						"ner_no_trainer"				)				)				)







		@unittest.skip(reason="Fix me @muellerzr"				)
		@mock.patch.dict(os.environ ,						{"WANDB_MODE": "offline"}				)
		def        lowercase       (    self				)      ->      Dict:


							'''simple docstring'''

							_snake_case       :					Union[str, Any]				       =   self.get_auto_remove_tmp_dir()
							_snake_case       :					Optional[Any]				       =   f"""
            {self.examples_dir}/pytorch/question-answering/run_qa_no_trainer.py
            --model_name_or_path bert-base-uncased
            --version_2_with_negative
            --train_file tests/fixtures/tests_samples/SQUAD/sample.json
            --validation_file tests/fixtures/tests_samples/SQUAD/sample.json
            --output_dir {tmp_dir}
            --seed=42
            --max_train_steps=10
            --num_warmup_steps=2
            --learning_rate=2e-4
            --per_device_train_batch_size=2
            --per_device_eval_batch_size=1
            --checkpointing_steps epoch
            --with_tracking
        """.split()

							run_command(self._launch_args + testargs				)
							_snake_case       :					List[Any]				       =   get_results(__snake_case				)
							# Because we use --version_2_with_negative the testing script uses SQuAD v2 metrics.
							self.assertGreaterEqual(result["eval_f1"] ,						2_8				)
							self.assertGreaterEqual(result["eval_exact"] ,						2_8				)
							self.assertTrue(os.path.exists(os.path.join(__snake_case ,						"epoch_0"				)				)				)
							self.assertTrue(os.path.exists(os.path.join(__snake_case ,						"qa_no_trainer"				)				)				)







		@mock.patch.dict(os.environ ,						{"WANDB_MODE": "offline"}				)
		def        lowercase       (    self				)      ->      Union[str, Any]:


							'''simple docstring'''

							_snake_case       :					Optional[Any]				       =   self.get_auto_remove_tmp_dir()
							_snake_case       :					Tuple				       =   f"""
            {self.examples_dir}/pytorch/multiple-choice/run_swag_no_trainer.py
            --model_name_or_path bert-base-uncased
            --train_file tests/fixtures/tests_samples/swag/sample.json
            --validation_file tests/fixtures/tests_samples/swag/sample.json
            --output_dir {tmp_dir}
            --max_train_steps=20
            --num_warmup_steps=2
            --learning_rate=2e-4
            --per_device_train_batch_size=2
            --per_device_eval_batch_size=1
            --with_tracking
        """.split()

							run_command(self._launch_args + testargs				)
							_snake_case       :					Dict				       =   get_results(__snake_case				)
							self.assertGreaterEqual(result["eval_accuracy"] ,						0.8				)
							self.assertTrue(os.path.exists(os.path.join(__snake_case ,						"swag_no_trainer"				)				)				)







		@slow
		@mock.patch.dict(os.environ ,						{"WANDB_MODE": "offline"}				)
		def        lowercase       (    self				)      ->      Optional[Any]:


							'''simple docstring'''

							_snake_case       :					Optional[int]				       =   self.get_auto_remove_tmp_dir()
							_snake_case       :					Dict				       =   f"""
            {self.examples_dir}/pytorch/summarization/run_summarization_no_trainer.py
            --model_name_or_path t5-small
            --train_file tests/fixtures/tests_samples/xsum/sample.json
            --validation_file tests/fixtures/tests_samples/xsum/sample.json
            --output_dir {tmp_dir}
            --max_train_steps=50
            --num_warmup_steps=8
            --learning_rate=2e-4
            --per_device_train_batch_size=2
            --per_device_eval_batch_size=1
            --checkpointing_steps epoch
            --with_tracking
        """.split()

							run_command(self._launch_args + testargs				)
							_snake_case       :					Optional[int]				       =   get_results(__snake_case				)
							self.assertGreaterEqual(result["eval_rouge1"] ,						1_0				)
							self.assertGreaterEqual(result["eval_rouge2"] ,						2				)
							self.assertGreaterEqual(result["eval_rougeL"] ,						7				)
							self.assertGreaterEqual(result["eval_rougeLsum"] ,						7				)
							self.assertTrue(os.path.exists(os.path.join(__snake_case ,						"epoch_0"				)				)				)
							self.assertTrue(os.path.exists(os.path.join(__snake_case ,						"summarization_no_trainer"				)				)				)







		@slow
		@mock.patch.dict(os.environ ,						{"WANDB_MODE": "offline"}				)
		def        lowercase       (    self				)      ->      Dict:


							'''simple docstring'''

							_snake_case       :					Dict				       =   self.get_auto_remove_tmp_dir()
							_snake_case       :					List[Any]				       =   f"""
            {self.examples_dir}/pytorch/translation/run_translation_no_trainer.py
            --model_name_or_path sshleifer/student_marian_en_ro_6_1
            --source_lang en
            --target_lang ro
            --train_file tests/fixtures/tests_samples/wmt16/sample.json
            --validation_file tests/fixtures/tests_samples/wmt16/sample.json
            --output_dir {tmp_dir}
            --max_train_steps=50
            --num_warmup_steps=8
            --num_beams=6
            --learning_rate=3e-3
            --per_device_train_batch_size=2
            --per_device_eval_batch_size=1
            --source_lang en_XX
            --target_lang ro_RO
            --checkpointing_steps epoch
            --with_tracking
        """.split()

							run_command(self._launch_args + testargs				)
							_snake_case       :					Optional[int]				       =   get_results(__snake_case				)
							self.assertGreaterEqual(result["eval_bleu"] ,						3_0				)
							self.assertTrue(os.path.exists(os.path.join(__snake_case ,						"epoch_0"				)				)				)
							self.assertTrue(os.path.exists(os.path.join(__snake_case ,						"translation_no_trainer"				)				)				)







		@slow
		def        lowercase       (    self				)      ->      Dict:


							'''simple docstring'''

							_snake_case       :					List[Any]				       =   logging.StreamHandler(sys.stdout				)
							logger.addHandler(__snake_case				)

							_snake_case       :					Optional[Any]				       =   self.get_auto_remove_tmp_dir()
							_snake_case       :					int				       =   f"""
            {self.examples_dir}/pytorch/semantic-segmentation/run_semantic_segmentation_no_trainer.py
            --dataset_name huggingface/semantic-segmentation-test-sample
            --output_dir {tmp_dir}
            --max_train_steps=10
            --num_warmup_steps=2
            --learning_rate=2e-4
            --per_device_train_batch_size=2
            --per_device_eval_batch_size=1
            --checkpointing_steps epoch
        """.split()

							run_command(self._launch_args + testargs				)
							_snake_case       :					Tuple				       =   get_results(__snake_case				)
							self.assertGreaterEqual(result["eval_overall_accuracy"] ,						0.10				)







		@mock.patch.dict(os.environ ,						{"WANDB_MODE": "offline"}				)
		def        lowercase       (    self				)      ->      Any:


							'''simple docstring'''

							_snake_case       :					int				       =   self.get_auto_remove_tmp_dir()
							_snake_case       :					Tuple				       =   f"""
            {self.examples_dir}/pytorch/image-classification/run_image_classification_no_trainer.py
            --model_name_or_path google/vit-base-patch16-224-in21k
            --dataset_name hf-internal-testing/cats_vs_dogs_sample
            --learning_rate 1e-4
            --per_device_train_batch_size 2
            --per_device_eval_batch_size 1
            --max_train_steps 2
            --train_val_split 0.1
            --seed 42
            --output_dir {tmp_dir}
            --with_tracking
            --checkpointing_steps 1
        """.split()

							if is_cuda_and_apex_available():
												testargs.append("--fp16"				)

							run_command(self._launch_args + testargs				)
							_snake_case       :					List[Any]				       =   get_results(__snake_case				)
							# The base model scores a 25%
							self.assertGreaterEqual(result["eval_accuracy"] ,						0.6				)
							self.assertTrue(os.path.exists(os.path.join(__snake_case ,						"step_1"				)				)				)
							self.assertTrue(os.path.exists(os.path.join(__snake_case ,						"image_classification_no_trainer"				)				)				)






import datasets

from .evaluate import evaluate


__lowerCAmelCase :Any     =    '\\n@article{hendrycks2021cuad,\n      title={CUAD: An Expert-Annotated NLP Dataset for Legal Contract Review},\n      author={Dan Hendrycks and Collin Burns and Anya Chen and Spencer Ball},\n      journal={arXiv preprint arXiv:2103.06268},\n      year={2021}\n}\n'

__lowerCAmelCase :Union[str, Any]     =    '\nThis metric wrap the official scoring script for version 1 of the Contract\nUnderstanding Atticus Dataset (CUAD).\nContract Understanding Atticus Dataset (CUAD) v1 is a corpus of more than 13,000 labels in 510\ncommercial legal contracts that have been manually labeled to identify 41 categories of important\nclauses that lawyers look for when reviewing contracts in connection with corporate transactions.\n'

__lowerCAmelCase :Optional[Any]     =    '\nComputes CUAD scores (EM, F1, AUPR, Precision@80%Recall, and Precision@90%Recall).\nArgs:\n    predictions: List of question-answers dictionaries with the following key-values:\n        - \'id\': id of the question-answer pair as given in the references (see below)\n        - \'prediction_text\': list of possible texts for the answer, as a list of strings\n        depending on a threshold on the confidence probability of each prediction.\n    references: List of question-answers dictionaries with the following key-values:\n        - \'id\': id of the question-answer pair (see above),\n        - \'answers\': a Dict in the CUAD dataset format\n            {\n                \'text\': list of possible texts for the answer, as a list of strings\n                \'answer_start\': list of start positions for the answer, as a list of ints\n            }\n            Note that answer_start values are not taken into account to compute the metric.\nReturns:\n    \'exact_match\': Exact match (the normalized answer exactly match the gold answer)\n    \'f1\': The F-score of predicted tokens versus the gold answer\n    \'aupr\': Area Under the Precision-Recall curve\n    \'prec_at_80_recall\': Precision at 80% recall\n    \'prec_at_90_recall\': Precision at 90% recall\nExamples:\n    >>> predictions = [{\'prediction_text\': [\'The seller:\', \'The buyer/End-User: Shenzhen LOHAS Supply Chain Management Co., Ltd.\'], \'id\': \'LohaCompanyltd_20191209_F-1_EX-10.16_11917878_EX-10.16_Supply Agreement__Parties\'}]\n    >>> references = [{\'answers\': {\'answer_start\': [143, 49], \'text\': [\'The seller:\', \'The buyer/End-User: Shenzhen LOHAS Supply Chain Management Co., Ltd.\']}, \'id\': \'LohaCompanyltd_20191209_F-1_EX-10.16_11917878_EX-10.16_Supply Agreement__Parties\'}]\n    >>> cuad_metric = datasets.load_metric("cuad")\n    >>> results = cuad_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {\'exact_match\': 100.0, \'f1\': 100.0, \'aupr\': 0.0, \'prec_at_80_recall\': 1.0, \'prec_at_90_recall\': 1.0}\n'






@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION	,   _KWARGS_DESCRIPTION    )
class        _a(    datasets.Metric    ):







		def        lowercase       (    self				)      ->      Optional[int]:


							'''simple docstring'''

							return datasets.MetricInfo(
							    description=_DESCRIPTION ,						citation=_CITATION ,						inputs_description=_KWARGS_DESCRIPTION ,						features=datasets.Features(
							        {
							            "predictions": {
							                "id": datasets.Value("string"				),
							                "prediction_text": datasets.features.Sequence(datasets.Value("string"				)				),
							            },
							            "references": {
							                "id": datasets.Value("string"				),
							                "answers": datasets.features.Sequence(
							                    {
							                        "text": datasets.Value("string"				),
							                        "answer_start": datasets.Value("int32"				),
							                    }				),
							            },
							        }				) ,						codebase_urls=["https://www.atticusprojectai.org/cuad"] ,						reference_urls=["https://www.atticusprojectai.org/cuad"] ,						)







		def        lowercase       (    self ,						__snake_case ,						__snake_case				)      ->      Union[str, Any]:


							'''simple docstring'''

							_snake_case       :					str				       =   {prediction["id"]: prediction["prediction_text"] for prediction in predictions}
							_snake_case       :					Union[str, Any]				       =   [
							    {
							        "paragraphs": [
							            {
							                "qas": [
							                    {
							                        "answers": [{"text": answer_text} for answer_text in ref["answers"]["text"]],
							                        "id": ref["id"],
							                    }
							                    for ref in references
							                ]
							            }
							        ]
							    }
							]
							_snake_case       :					Union[str, Any]				       =   evaluate(dataset=__snake_case ,						predictions=__snake_case				)
							return score