
"""simple docstring"""




import shutil
import tempfile
import unittest
from unittest.mock import patch

from transformers import (
    DefaultFlowCallback,
    IntervalStrategy,
    PrinterCallback,
    ProgressCallback,
    Trainer,
    TrainerCallback,
    TrainingArguments,
    is_torch_available,
)
from transformers.testing_utils import require_torch


if is_torch_available():
   from transformers.trainer import DEFAULT_CALLBACKS

   from .test_trainer import RegressionDataset, RegressionModelConfig, RegressionPreTrainedModel

class 						a__		( a_		):
   def __init__(     self							):
          lowercase       :							List[str]									=				[]
   def      __magic_name__     (     self      ,       _a      ,       _a      ,       _a      ,       **_a							):
          self.events.append("on_init_end"							)
   def      __magic_name__     (     self      ,       _a      ,       _a      ,       _a      ,       **_a							):
          self.events.append("on_train_begin"							)
   def      __magic_name__     (     self      ,       _a      ,       _a      ,       _a      ,       **_a							):
          self.events.append("on_train_end"							)
   def      __magic_name__     (     self      ,       _a      ,       _a      ,       _a      ,       **_a							):
          self.events.append("on_epoch_begin"							)
   def      __magic_name__     (     self      ,       _a      ,       _a      ,       _a      ,       **_a							):
          self.events.append("on_epoch_end"							)
   def      __magic_name__     (     self      ,       _a      ,       _a      ,       _a      ,       **_a							):
          self.events.append("on_step_begin"							)
   def      __magic_name__     (     self      ,       _a      ,       _a      ,       _a      ,       **_a							):
          self.events.append("on_step_end"							)
   def      __magic_name__     (     self      ,       _a      ,       _a      ,       _a      ,       **_a							):
          self.events.append("on_evaluate"							)
   def      __magic_name__     (     self      ,       _a      ,       _a      ,       _a      ,       **_a							):
          self.events.append("on_predict"							)
   def      __magic_name__     (     self      ,       _a      ,       _a      ,       _a      ,       **_a							):
          self.events.append("on_save"							)
   def      __magic_name__     (     self      ,       _a      ,       _a      ,       _a      ,       **_a							):
          self.events.append("on_log"							)

   def      __magic_name__     (     self      ,       _a      ,       _a      ,       _a      ,       **_a							):
          self.events.append("on_prediction_step"							)


@require_torch
class 						a__		( unittest.TestCase		):
   def      __magic_name__     (     self							):
          lowercase       :							List[str]									=				tempfile.mkdtemp()
   def      __magic_name__     (     self							):
          shutil.rmtree(self.output_dir							)
   def      __magic_name__     (     self      ,       _a=0      ,       _a=0      ,       _a=64      ,       _a=64      ,       _a=None      ,       _a=False      ,       **_a							):
          # disable_tqdm in TrainingArguments has a flaky default since it depends on the level of logging. We make sure
          # its set to False since the tests later on depend on its value.
          lowercase       :							List[str]									=				RegressionDataset(length=__a							)
          lowercase       :							List[str]									=				RegressionDataset(length=__a							)
          lowercase       :							int									=				RegressionModelConfig(a=__a      ,       b=__a							)
          lowercase       :							Union[str, Any]									=				RegressionPreTrainedModel(__a							)

          lowercase       :							str									=				TrainingArguments(self.output_dir      ,       disable_tqdm=__a      ,       report_to=[]      ,       **__a							)
          return Trainer(
              __a      ,       __a      ,       train_dataset=__a      ,       eval_dataset=__a      ,       callbacks=__a      ,       )
   def      __magic_name__     (     self      ,       _a      ,       _a							):
          self.assertEqual(len(__a							)      ,       len(__a							)							)

          # Order doesn't matter
          lowercase       :							Dict									=				sorted(__a      ,       key=lambda _a							: cb.__name__ if isinstance(__a      ,       __a							) else cb.__class__.__name__							)
          lowercase       :							Any									=				sorted(__a      ,       key=lambda _a							: cb.__name__ if isinstance(__a      ,       __a							) else cb.__class__.__name__							)

          for cba, cba in zip(__a      ,       __a							):
                 if isinstance(__a      ,       __a							) and isinstance(__a      ,       __a							):
                        self.assertEqual(__a      ,       __a							)
                 elif isinstance(__a      ,       __a							) and not isinstance(__a      ,       __a							):
                        self.assertEqual(__a      ,       cba.__class__							)
                 elif not isinstance(__a      ,       __a							) and isinstance(__a      ,       __a							):
                        self.assertEqual(cba.__class__      ,       __a							)
                 else:
                        self.assertEqual(__a      ,       __a							)
   def      __magic_name__     (     self      ,       _a							):
          lowercase       :							List[Any]									=				["on_init_end", "on_train_begin"]
          lowercase       :							str									=				0
          lowercase       :							Optional[int]									=				len(trainer.get_eval_dataloader()							)
          lowercase       :							Tuple									=				["on_prediction_step"] * len(trainer.get_eval_dataloader()							) + ["on_log", "on_evaluate"]
          for _ in range(trainer.state.num_train_epochs							):
                 expected_events.append("on_epoch_begin"							)
                 for _ in range(__a							):
                        step += 1
                        expected_events += ["on_step_begin", "on_step_end"]
                        if step % trainer.args.logging_steps == 0:
                               expected_events.append("on_log"							)
                        if trainer.args.evaluation_strategy == IntervalStrategy.STEPS and step % trainer.args.eval_steps == 0:
                               expected_events += evaluation_events.copy()
                        if step % trainer.args.save_steps == 0:
                               expected_events.append("on_save"							)
                 expected_events.append("on_epoch_end"							)
                 if trainer.args.evaluation_strategy == IntervalStrategy.EPOCH:
                        expected_events += evaluation_events.copy()
          expected_events += ["on_log", "on_train_end"]
          return expected_events
   def      __magic_name__     (     self							):
          lowercase       :							List[Any]									=				self.get_trainer()
          lowercase       :							str									=				DEFAULT_CALLBACKS.copy() + [ProgressCallback]
          self.check_callbacks_equality(trainer.callback_handler.callbacks      ,       __a							)

          # Callbacks passed at init are added to the default callbacks
          lowercase       :							List[Any]									=				self.get_trainer(callbacks=[MyTestTrainerCallback]							)
          expected_callbacks.append(__a							)
          self.check_callbacks_equality(trainer.callback_handler.callbacks      ,       __a							)

          # TrainingArguments.disable_tqdm controls if use ProgressCallback or PrinterCallback
          lowercase       :							Dict									=				self.get_trainer(disable_tqdm=__a							)
          lowercase       :							Optional[Any]									=				DEFAULT_CALLBACKS.copy() + [PrinterCallback]
          self.check_callbacks_equality(trainer.callback_handler.callbacks      ,       __a							)
   def      __magic_name__     (     self							):
          lowercase       :							Any									=				DEFAULT_CALLBACKS.copy() + [ProgressCallback]
          lowercase       :							List[Any]									=				self.get_trainer()

          # We can add, pop, or remove by class name
          trainer.remove_callback(__a							)
          expected_callbacks.remove(__a							)
          self.check_callbacks_equality(trainer.callback_handler.callbacks      ,       __a							)

          lowercase       :							Any									=				self.get_trainer()
          lowercase       :							Tuple									=				trainer.pop_callback(__a							)
          self.assertEqual(cb.__class__      ,       __a							)
          self.check_callbacks_equality(trainer.callback_handler.callbacks      ,       __a							)

          trainer.add_callback(__a							)
          expected_callbacks.insert(0      ,       __a							)
          self.check_callbacks_equality(trainer.callback_handler.callbacks      ,       __a							)

          # We can also add, pop, or remove by instance
          lowercase       :							Dict									=				self.get_trainer()
          lowercase       :							List[Any]									=				trainer.callback_handler.callbacks[0]
          trainer.remove_callback(__a							)
          expected_callbacks.remove(__a							)
          self.check_callbacks_equality(trainer.callback_handler.callbacks      ,       __a							)

          lowercase       :							Dict									=				self.get_trainer()
          lowercase       :							Tuple									=				trainer.callback_handler.callbacks[0]
          lowercase       :							List[str]									=				trainer.pop_callback(__a							)
          self.assertEqual(__a      ,       __a							)
          self.check_callbacks_equality(trainer.callback_handler.callbacks      ,       __a							)

          trainer.add_callback(__a							)
          expected_callbacks.insert(0      ,       __a							)
          self.check_callbacks_equality(trainer.callback_handler.callbacks      ,       __a							)

   def      __magic_name__     (     self							):
          import warnings

          # XXX: for now ignore scatter_gather warnings in this test since it's not relevant to what's being tested
          warnings.simplefilter(action="ignore"      ,       category=__a							)

          lowercase       :							int									=				self.get_trainer(callbacks=[MyTestTrainerCallback]							)
          trainer.train()
          lowercase       :							Optional[int]									=				trainer.callback_handler.callbacks[-2].events
          self.assertEqual(__a      ,       self.get_expected_events(__a							)							)

          # Independent log/save/eval
          lowercase       :							Any									=				self.get_trainer(callbacks=[MyTestTrainerCallback]      ,       logging_steps=5							)
          trainer.train()
          lowercase       :							Dict									=				trainer.callback_handler.callbacks[-2].events
          self.assertEqual(__a      ,       self.get_expected_events(__a							)							)

          lowercase       :							Optional[int]									=				self.get_trainer(callbacks=[MyTestTrainerCallback]      ,       save_steps=5							)
          trainer.train()
          lowercase       :							List[str]									=				trainer.callback_handler.callbacks[-2].events
          self.assertEqual(__a      ,       self.get_expected_events(__a							)							)

          lowercase       :							Dict									=				self.get_trainer(callbacks=[MyTestTrainerCallback]      ,       eval_steps=5      ,       evaluation_strategy="steps"							)
          trainer.train()
          lowercase       :							Union[str, Any]									=				trainer.callback_handler.callbacks[-2].events
          self.assertEqual(__a      ,       self.get_expected_events(__a							)							)

          lowercase       :							int									=				self.get_trainer(callbacks=[MyTestTrainerCallback]      ,       evaluation_strategy="epoch"							)
          trainer.train()
          lowercase       :							List[str]									=				trainer.callback_handler.callbacks[-2].events
          self.assertEqual(__a      ,       self.get_expected_events(__a							)							)

          # A bit of everything
          lowercase       :							str									=				self.get_trainer(
              callbacks=[MyTestTrainerCallback]      ,       logging_steps=3      ,       save_steps=10      ,       eval_steps=5      ,       evaluation_strategy="steps"      ,       )
          trainer.train()
          lowercase       :							List[str]									=				trainer.callback_handler.callbacks[-2].events
          self.assertEqual(__a      ,       self.get_expected_events(__a							)							)

          # warning should be emitted for duplicated callbacks
          with patch("transformers.trainer_callback.logger.warning"							) as warn_mock:
                 lowercase       :							Optional[Any]									=				self.get_trainer(
                     callbacks=[MyTestTrainerCallback, MyTestTrainerCallback]      ,       )
                 assert str(__a							) in warn_mock.call_args[0][0]



'''simple docstring'''


import argparse
import json
import logging
import os
import sys
from unittest.mock import patch

from transformers.testing_utils import TestCasePlus, get_gpu_count, slow


__lowerCAmelCase						:							int    							=						[
    os.path.join(os.path.dirname(__file__), dirname)
    for dirname in [
        "text-classification",
        "language-modeling",
        "summarization",
        "token-classification",
        "question-answering",
    ]
]
sys.path.extend(SRC_DIRS)


if SRC_DIRS is not None:
					import run_clm_flax
					import run_flax_glue
					import run_flax_ner
					import run_mlm_flax
					import run_qa
					import run_summarization_flax
					import run_ta_mlm_flax


logging.basicConfig(level=logging.DEBUG)

__lowerCAmelCase						:							List[Any]    							=						logging.getLogger()





def    lowerCAmelCase       (		):


	"""simple docstring"""
	__UpperCAmelCase      					=      argparse.ArgumentParser()
	parser.add_argument('''-f'''							)
	__UpperCAmelCase      					=      parser.parse_args()
	return args.f





def    lowerCAmelCase       (		UpperCamelCase__					: Union[str, Any]     ,       UpperCamelCase__					: Any="eval"							):


	"""simple docstring"""
	__UpperCAmelCase      					=      os.path.join(UpperCamelCase__     ,       f"""{split}_results.json"""							)
	if os.path.exists(UpperCamelCase__							):
		with open(UpperCamelCase__     ,       '''r'''							) as f:
			return json.load(UpperCamelCase__							)
	raise ValueError(f"""can't find {path}"""							)


__lowerCAmelCase						:							str    							=						logging.StreamHandler(sys.stdout)
logger.addHandler(stream_handler)






class     A ( UpperCAmelCase ):





							def    snake_case__    (   self					:				List[Any]						)				->						Union[str, Any]:
								__UpperCAmelCase      					=      self.get_auto_remove_tmp_dir()
								__UpperCAmelCase      					=      f"""
            run_glue.py
            --model_name_or_path distilbert-base-uncased
            --output_dir {tmp_dir}
            --train_file ./tests/fixtures/tests_samples/MRPC/train.csv
            --validation_file ./tests/fixtures/tests_samples/MRPC/dev.csv
            --per_device_train_batch_size=2
            --per_device_eval_batch_size=1
            --learning_rate=1e-4
            --eval_steps=2
            --warmup_steps=2
            --seed=42
            --max_seq_length=128
            """.split()

								with patch.object(__a  ,		'''argv'''  ,		__a						):
									run_flax_glue.main()
									__UpperCAmelCase      					=      get_results(__a						)
									self.assertGreaterEqual(result['''eval_accuracy''']  ,		0.7_5						)





							@slow
							def    snake_case__    (   self					:				Optional[Any]						)				->						Optional[int]:
								__UpperCAmelCase      					=      self.get_auto_remove_tmp_dir()
								__UpperCAmelCase      					=      f"""
            run_clm_flax.py
            --model_name_or_path distilgpt2
            --train_file ./tests/fixtures/sample_text.txt
            --validation_file ./tests/fixtures/sample_text.txt
            --do_train
            --do_eval
            --block_size 128
            --per_device_train_batch_size 4
            --per_device_eval_batch_size 4
            --num_train_epochs 2
            --logging_steps 2 --eval_steps 2
            --output_dir {tmp_dir}
            --overwrite_output_dir
            """.split()

								with patch.object(__a  ,		'''argv'''  ,		__a						):
									run_clm_flax.main()
									__UpperCAmelCase      					=      get_results(__a						)
									self.assertLess(result['''eval_perplexity''']  ,		1_0_0						)





							@slow
							def    snake_case__    (   self					:				Dict						)				->						List[str]:
								__UpperCAmelCase      					=      self.get_auto_remove_tmp_dir()
								__UpperCAmelCase      					=      f"""
            run_summarization.py
            --model_name_or_path t5-small
            --train_file tests/fixtures/tests_samples/xsum/sample.json
            --validation_file tests/fixtures/tests_samples/xsum/sample.json
            --test_file tests/fixtures/tests_samples/xsum/sample.json
            --output_dir {tmp_dir}
            --overwrite_output_dir
            --num_train_epochs=3
            --warmup_steps=8
            --do_train
            --do_eval
            --do_predict
            --learning_rate=2e-4
            --per_device_train_batch_size=2
            --per_device_eval_batch_size=1
            --predict_with_generate
        """.split()

								with patch.object(__a  ,		'''argv'''  ,		__a						):
									run_summarization_flax.main()
									__UpperCAmelCase      					=      get_results(__a  ,		split='''test'''						)
									self.assertGreaterEqual(result['''test_rouge1''']  ,		1_0						)
									self.assertGreaterEqual(result['''test_rouge2''']  ,		2						)
									self.assertGreaterEqual(result['''test_rougeL''']  ,		7						)
									self.assertGreaterEqual(result['''test_rougeLsum''']  ,		7						)





							@slow
							def    snake_case__    (   self					:				Any						)				->						List[Any]:
								__UpperCAmelCase      					=      self.get_auto_remove_tmp_dir()
								__UpperCAmelCase      					=      f"""
            run_mlm.py
            --model_name_or_path distilroberta-base
            --train_file ./tests/fixtures/sample_text.txt
            --validation_file ./tests/fixtures/sample_text.txt
            --output_dir {tmp_dir}
            --overwrite_output_dir
            --max_seq_length 128
            --per_device_train_batch_size 4
            --per_device_eval_batch_size 4
            --logging_steps 2 --eval_steps 2
            --do_train
            --do_eval
            --num_train_epochs=1
        """.split()

								with patch.object(__a  ,		'''argv'''  ,		__a						):
									run_mlm_flax.main()
									__UpperCAmelCase      					=      get_results(__a						)
									self.assertLess(result['''eval_perplexity''']  ,		4_2						)





							@slow
							def    snake_case__    (   self					:				Dict						)				->						str:
								__UpperCAmelCase      					=      self.get_auto_remove_tmp_dir()
								__UpperCAmelCase      					=      f"""
            run_t5_mlm_flax.py
            --model_name_or_path t5-small
            --train_file ./tests/fixtures/sample_text.txt
            --validation_file ./tests/fixtures/sample_text.txt
            --do_train
            --do_eval
            --max_seq_length 128
            --per_device_train_batch_size 4
            --per_device_eval_batch_size 4
            --num_train_epochs 2
            --logging_steps 2 --eval_steps 2
            --output_dir {tmp_dir}
            --overwrite_output_dir
            """.split()

								with patch.object(__a  ,		'''argv'''  ,		__a						):
									run_ta_mlm_flax.main()
									__UpperCAmelCase      					=      get_results(__a						)
									self.assertGreaterEqual(result['''eval_accuracy''']  ,		0.4_2						)





							@slow
							def    snake_case__    (   self					:				Dict						)				->						Tuple:
								# with so little data distributed training needs more epochs to get the score on par with 0/1 gpu
								__UpperCAmelCase      					=      7 if get_gpu_count() > 1 else 2

								__UpperCAmelCase      					=      self.get_auto_remove_tmp_dir()
								__UpperCAmelCase      					=      f"""
            run_flax_ner.py
            --model_name_or_path bert-base-uncased
            --train_file tests/fixtures/tests_samples/conll/sample.json
            --validation_file tests/fixtures/tests_samples/conll/sample.json
            --output_dir {tmp_dir}
            --overwrite_output_dir
            --do_train
            --do_eval
            --warmup_steps=2
            --learning_rate=2e-4
            --logging_steps 2 --eval_steps 2
            --per_device_train_batch_size=2
            --per_device_eval_batch_size=2
            --num_train_epochs={epochs}
            --seed 7
        """.split()

								with patch.object(__a  ,		'''argv'''  ,		__a						):
									run_flax_ner.main()
									__UpperCAmelCase      					=      get_results(__a						)
									self.assertGreaterEqual(result['''eval_accuracy''']  ,		0.7_5						)
									self.assertGreaterEqual(result['''eval_f1''']  ,		0.3						)






							@slow
							def    snake_case__    (   self					:				Optional[Any]						)				->						List[Any]:
								__UpperCAmelCase      					=      self.get_auto_remove_tmp_dir()
								__UpperCAmelCase      					=      f"""
            run_qa.py
            --model_name_or_path bert-base-uncased
            --version_2_with_negative
            --train_file tests/fixtures/tests_samples/SQUAD/sample.json
            --validation_file tests/fixtures/tests_samples/SQUAD/sample.json
            --output_dir {tmp_dir}
            --overwrite_output_dir
            --num_train_epochs=3
            --warmup_steps=2
            --do_train
            --do_eval
            --logging_steps 2 --eval_steps 2
            --learning_rate=2e-4
            --per_device_train_batch_size=2
            --per_device_eval_batch_size=1
        """.split()

								with patch.object(__a  ,		'''argv'''  ,		__a						):
									run_qa.main()
									__UpperCAmelCase      					=      get_results(__a						)
									self.assertGreaterEqual(result['''eval_f1''']  ,		3_0						)
									self.assertGreaterEqual(result['''eval_exact''']  ,		3_0						)
