



"""simple docstring"""
import ast
import os
import re
import shutil
import tempfile
import unittest
from unittest import mock

import torch

from accelerate.test_utils.examples import compare_against_test
from accelerate.test_utils.testing import TempDirTestCase, require_trackers, run_command, slow
from accelerate.utils import write_basic_config


# DataLoaders built from `test_samples/MRPC` for quick testing
# Should mock `{script_name}.get_dataloaders` via:
# @mock.patch("{script_name}.get_dataloaders", mocked_dataloaders)

SCREAMING_SNAKE_CASE_				= [
    '''cross_validation.py''',
    '''gradient_accumulation.py''',
    '''local_sgd.py''',
    '''multi_process_metrics.py''',
    '''memory.py''',
    '''automatic_gradient_accumulation.py''',
    '''fsdp_with_peak_mem_tracking.py''',
    '''deepspeed_with_config_support.py''',
    '''megatron_lm_gpt_pretraining.py''',
]




class 			snake_case_						(						unittest.TestCase				):






							def snake_case_  (   self			,     a_			,     a_			,     a_ = None			,     a_ = None      ):
									a_     :      int 						=     None
									a_     :      int 						=     os.path.abspath(os.path.join("examples"			,     "by_feature"      )      )
									a_     :      Optional[Any] 						=     os.path.abspath("examples"      )
									for item in os.listdir(UpperCamelCase_      ):
											if item not in EXCLUDE_EXAMPLES:
													a_     :      List[str] 						=     os.path.join(UpperCamelCase_			,     UpperCamelCase_      )
													if os.path.isfile(UpperCamelCase_      ) and ".py" in item_path:
															with self.subTest(
															    tested_script=UpperCamelCase_			,     feature_script=UpperCamelCase_			,     tested_section="main()" if parser_only else "training_function()"			,     ):
																	a_     :      Tuple 						=     compare_against_test(
																	    os.path.join(UpperCamelCase_			,     UpperCamelCase_      )			,     UpperCamelCase_			,     UpperCamelCase_			,     UpperCamelCase_      )
																	a_     :      Optional[int] 						=     '\n'.join(UpperCamelCase_      )
																	if special_strings is not None:
																			for string in special_strings:
																					a_     :      Union[str, Any] 						=     diff.replace(UpperCamelCase_			,     ""      )
																	self.assertEqual(UpperCamelCase_			,     ""      )






							def snake_case_  (   self      ):
									self.one_complete_example("complete_nlp_example.py"			,     UpperCamelCase_      )
									self.one_complete_example("complete_nlp_example.py"			,     UpperCamelCase_      )






							def snake_case_  (   self      ):
									a_     :      Optional[Any] 						=     os.path.abspath(os.path.join("examples"			,     "cv_example.py"      )      )
									a_     :      str 						=     [
									    ' ' * 1_6 + '{\n\n',
									    ' ' * 2_0 + '"accuracy": eval_metric["accuracy"],\n\n',
									    ' ' * 2_0 + '"f1": eval_metric["f1"],\n\n',
									    ' ' * 2_0 + '"train_loss": total_loss.item() / len(train_dataloader),\n\n',
									    ' ' * 2_0 + '"epoch": epoch,\n\n',
									    ' ' * 1_6 + '},\n\n',
									    ' ' * 1_6 + 'step=epoch,\n',
									    ' ' * 1_2,
									    ' ' * 8 + 'for step, batch in enumerate(active_dataloader):\n',
									]
									self.one_complete_example("complete_cv_example.py"			,     UpperCamelCase_			,     UpperCamelCase_			,     UpperCamelCase_      )
									self.one_complete_example("complete_cv_example.py"			,     UpperCamelCase_			,     UpperCamelCase_			,     UpperCamelCase_      )




@mock.patch.dict(os.environ     ,{"TESTING_MOCKED_DATALOADERS": "1"}				)
class 			snake_case_						(						a_				):
							__lowerCAmelCase    					=					False






							@classmethod
							def snake_case_  (   cls      ):
									super().setUpClass()
									a_     :      Dict 						=     tempfile.mkdtemp()
									a_     :      str 						=     os.path.join(cls._tmpdir			,     "default_config.yml"      )

									write_basic_config(save_location=cls.configPath      )
									a_     :      Union[str, Any] 						=     ['accelerate', 'launch', '--config_file', cls.configPath]






							@classmethod
							def snake_case_  (   cls      ):
									super().tearDownClass()
									shutil.rmtree(cls._tmpdir      )






							def snake_case_  (   self      ):
									a_     :      Optional[int] 						=     F"""
        examples/by_feature/checkpointing.py
        --checkpointing_steps epoch
        --output_dir {self.tmpdir}
        """.split()
									run_command(self._launch_args + testargs      )
									self.assertTrue(os.path.exists(os.path.join(self.tmpdir			,     "epoch_0"      )      )      )






							def snake_case_  (   self      ):
									a_     :      int 						=     F"""
        examples/by_feature/checkpointing.py
        --checkpointing_steps 1
        --output_dir {self.tmpdir}
        """.split()
									a_     :      str 						=     run_command(self._launch_args + testargs      )
									self.assertTrue(os.path.exists(os.path.join(self.tmpdir			,     "step_2"      )      )      )






							def snake_case_  (   self      ):
									a_     :      Optional[int] 						=     F"""
        examples/by_feature/checkpointing.py
        --resume_from_checkpoint {os.path.join(self.tmpdir			,     'epoch_0'      )}
        """.split()
									a_     :      Any 						=     run_command(self._launch_args + testargs			,     return_stdout=UpperCamelCase_      )
									self.assertNotIn("epoch 0:"			,     UpperCamelCase_      )
									self.assertIn("epoch 1:"			,     UpperCamelCase_      )






							def snake_case_  (   self      ):
									a_     :      List[str] 						=     F"""
        examples/by_feature/checkpointing.py
        --resume_from_checkpoint {os.path.join(self.tmpdir			,     'step_2'      )}
        """.split()
									a_     :      int 						=     run_command(self._launch_args + testargs			,     return_stdout=UpperCamelCase_      )
									if torch.cuda.is_available():
											a_     :      Tuple 						=     torch.cuda.device_count()
									else:
											a_     :      Any 						=     1
									if num_processes > 1:
											self.assertNotIn("epoch 0:"			,     UpperCamelCase_      )
											self.assertIn("epoch 1:"			,     UpperCamelCase_      )
									else:
											self.assertIn("epoch 0:"			,     UpperCamelCase_      )
											self.assertIn("epoch 1:"			,     UpperCamelCase_      )






							@slow
							def snake_case_  (   self      ):
									a_     :      int 						=     '\n        examples/by_feature/cross_validation.py\n        --num_folds 2\n        '.split()
									with mock.patch.dict(os.environ			,     {"TESTING_MOCKED_DATALOADERS": "0"}      ):
											a_     :      Any 						=     run_command(self._launch_args + testargs			,     return_stdout=UpperCamelCase_      )
											a_     :      Optional[Any] 						=     re.findall("({.+})"			,     UpperCamelCase_      )
											a_     :      Optional[int] 						=     [r for r in results if 'accuracy' in r][-1]
											a_     :      Dict 						=     ast.literal_eval(UpperCamelCase_      )
											self.assertGreaterEqual(results["accuracy"]			,     0.75      )






							def snake_case_  (   self      ):
									a_     :      List[Any] 						=     ['examples/by_feature/multi_process_metrics.py']
									run_command(self._launch_args + testargs      )






							@require_trackers
							@mock.patch.dict(os.environ			,     {"WANDB_MODE": "offline"}      )
							def snake_case_  (   self      ):





									with tempfile.TemporaryDirectory() as tmpdir:
											a_     :      Optional[int] 						=     F"""
            examples/by_feature/tracking.py
            --with_tracking
            --project_dir {tmpdir}
            """.split()
											run_command(self._launch_args + testargs      )
											self.assertTrue(os.path.exists(os.path.join(UpperCamelCase_			,     "tracking"      )      )      )






							def snake_case_  (   self      ):
									a_     :      Tuple 						=     ['examples/by_feature/gradient_accumulation.py']
									run_command(self._launch_args + testargs      )






							def snake_case_  (   self      ):
									a_     :      Optional[Any] 						=     ['examples/by_feature/local_sgd.py']
									run_command(self._launch_args + testargs      )




"""simple docstring"""
from ...configuration_utils import PretrainedConfig
from ...utils import logging


SCREAMING_SNAKE_CASE_				= logging.get_logger(__name__)

SCREAMING_SNAKE_CASE_				= {
    """uw-madison/mra-base-512-4""": """https://huggingface.co/uw-madison/mra-base-512-4/resolve/main/config.json""",
}




class 			snake_case_						(						a_				):
							__lowerCAmelCase    					=					"mra"






							def __init__(   self			,     a_=5_0_2_6_5			,     a_=7_6_8			,     a_=1_2			,     a_=1_2			,     a_=3_0_7_2			,     a_="gelu"			,     a_=0.1			,     a_=0.1			,     a_=5_1_2			,     a_=1			,     a_=0.02			,     a_=1e-5			,     a_="absolute"			,     a_=4			,     a_="full"			,     a_=0			,     a_=0			,     a_=1			,     a_=0			,     a_=2			,     **a_			,     ):
									super().__init__(pad_token_id=a_			,     bos_token_id=a_			,     eos_token_id=a_			,     **a_      )

									a_     :      Optional[int] 						=     vocab_size
									a_     :      Dict 						=     max_position_embeddings
									a_     :      str 						=     hidden_size
									a_     :      Optional[int] 						=     num_hidden_layers
									a_     :      List[Any] 						=     num_attention_heads
									a_     :      List[str] 						=     intermediate_size
									a_     :      Tuple 						=     hidden_act
									a_     :      List[Any] 						=     hidden_dropout_prob
									a_     :      List[str] 						=     attention_probs_dropout_prob
									a_     :      List[Any] 						=     initializer_range
									a_     :      Dict 						=     type_vocab_size
									a_     :      Union[str, Any] 						=     layer_norm_eps
									a_     :      List[str] 						=     position_embedding_type
									a_     :      Union[str, Any] 						=     block_per_row
									a_     :      Tuple 						=     approx_mode
									a_     :      Optional[Any] 						=     initial_prior_first_n_blocks
									a_     :      List[str] 						=     initial_prior_diagonal_n_blocks