






import inspect
import os

import torch
from transformers import AutoModel
from transformers.testing_utils import mockenv_context
from transformers.trainer_utils import set_seed

import accelerate
from accelerate.accelerator import Accelerator
from accelerate.state import AcceleratorState
from accelerate.test_utils.testing import (
    AccelerateTestCase,
    TempDirTestCase,
    execute_subprocess_async,
    require_cuda,
    require_fsdp,
    require_multi_gpu,
    slow,
)
from accelerate.utils.constants import (
    FSDP_AUTO_WRAP_POLICY,
    FSDP_BACKWARD_PREFETCH,
    FSDP_SHARDING_STRATEGY,
    FSDP_STATE_DICT_TYPE,
)
from accelerate.utils.dataclasses import FullyShardedDataParallelPlugin
from accelerate.utils.other import patch_environment


set_seed(42)

UpperCamelCase        =				"bert-base-cased"
UpperCamelCase        =				"fp16"
UpperCamelCase        =				"bf16"
UpperCamelCase        =				[FPaa, BFaa]

@require_fsdp
@require_cuda
class        lowerCAmelCase_				(      lowercase	):







				"""simple docstring"""




				def 				__a   ( self     :List[str]     ):
					super().setUp()

					UpperCamelCase__ :str						=      dict(
					    ACCELERATE_USE_FSDP="""true"""  ,     MASTER_ADDR="""localhost"""  ,     MASTER_PORT="""10999"""  ,     RANK="""0"""  ,     LOCAL_RANK="""0"""  ,     WORLD_SIZE="""1"""  ,     )







				def 				__a   ( self     :Union[str, Any]     ):
					from torch.distributed.fsdp.fully_sharded_data_parallel import ShardingStrategy

					for i, strategy in enumerate(lowerCamelCase__     ):
						UpperCamelCase__ :Optional[int]						=      self.dist_env.copy()
						UpperCamelCase__ :List[Any]						=      f"""{i + 1}"""
						UpperCamelCase__ :List[Any]						=      strategy
						with mockenv_context(**lowerCamelCase__     ):
							UpperCamelCase__ :Tuple						=      FullyShardedDataParallelPlugin()
							self.assertEqual(fsdp_plugin.sharding_strategy  ,     ShardingStrategy(i + 1     )     )







				def 				__a   ( self     :Union[str, Any]     ):
					from torch.distributed.fsdp.fully_sharded_data_parallel import BackwardPrefetch

					for i, prefetch_policy in enumerate(lowerCamelCase__     ):
						UpperCamelCase__ :Optional[int]						=      self.dist_env.copy()
						UpperCamelCase__ :Optional[int]						=      prefetch_policy
						with mockenv_context(**lowerCamelCase__     ):
							UpperCamelCase__ :Dict						=      FullyShardedDataParallelPlugin()
							if prefetch_policy == "NO_PREFETCH":
								self.assertIsNone(fsdp_plugin.backward_prefetch     )
							else:
								self.assertEqual(fsdp_plugin.backward_prefetch  ,     BackwardPrefetch(i + 1     )     )







				def 				__a   ( self     :Optional[Any]     ):
					from torch.distributed.fsdp.fully_sharded_data_parallel import StateDictType

					for i, state_dict_type in enumerate(lowerCamelCase__     ):
						UpperCamelCase__ :Optional[int]						=      self.dist_env.copy()
						UpperCamelCase__ :Tuple						=      state_dict_type
						with mockenv_context(**lowerCamelCase__     ):
							UpperCamelCase__ :List[str]						=      FullyShardedDataParallelPlugin()
							self.assertEqual(fsdp_plugin.state_dict_type  ,     StateDictType(i + 1     )     )
							if state_dict_type == "FULL_STATE_DICT":
								self.assertTrue(fsdp_plugin.state_dict_config.offload_to_cpu     )
								self.assertTrue(fsdp_plugin.state_dict_config.ranka_only     )







				def 				__a   ( self     :List[str]     ):
					UpperCamelCase__ :List[Any]						=      AutoModel.from_pretrained(lowerCamelCase__     )
					for policy in FSDP_AUTO_WRAP_POLICY:
						UpperCamelCase__ :Optional[int]						=      self.dist_env.copy()
						UpperCamelCase__ :int						=      policy
						if policy == "TRANSFORMER_BASED_WRAP":
							UpperCamelCase__ :Optional[Any]						=      """BertLayer"""
						elif policy == "SIZE_BASED_WRAP":
							UpperCamelCase__ :Union[str, Any]						=      """2000"""
						with mockenv_context(**lowerCamelCase__     ):
							UpperCamelCase__ :int						=      FullyShardedDataParallelPlugin()
							fsdp_plugin.set_auto_wrap_policy(lowerCamelCase__     )
							if policy == "NO_WRAP":
								self.assertIsNone(fsdp_plugin.auto_wrap_policy     )
							else:
								self.assertIsNotNone(fsdp_plugin.auto_wrap_policy     )

					UpperCamelCase__ :Optional[int]						=      self.dist_env.copy()
					UpperCamelCase__ :str						=      """TRANSFORMER_BASED_WRAP"""
					UpperCamelCase__ :Union[str, Any]						=      """T5Layer"""
					with mockenv_context(**lowerCamelCase__     ):
						UpperCamelCase__ :Any						=      FullyShardedDataParallelPlugin()
						with self.assertRaises(lowerCamelCase__     ) as cm:
							fsdp_plugin.set_auto_wrap_policy(lowerCamelCase__     )
						self.assertTrue("""Could not find the transformer layer class to wrap in the model.""" in str(cm.exception     )     )

					UpperCamelCase__ :Dict						=      self.dist_env.copy()
					UpperCamelCase__ :int						=      """SIZE_BASED_WRAP"""
					UpperCamelCase__ :Union[str, Any]						=      """0"""
					with mockenv_context(**lowerCamelCase__     ):
						UpperCamelCase__ :Optional[Any]						=      FullyShardedDataParallelPlugin()
						fsdp_plugin.set_auto_wrap_policy(lowerCamelCase__     )
						self.assertIsNone(fsdp_plugin.auto_wrap_policy     )







				def 				__a   ( self     :Optional[Any]     ):
					from torch.distributed.fsdp.fully_sharded_data_parallel import MixedPrecision
					from torch.distributed.fsdp.sharded_grad_scaler import ShardedGradScaler

					for mp_dtype in dtypes:
						UpperCamelCase__ :Dict						=      self.dist_env.copy()
						UpperCamelCase__ :Dict						=      mp_dtype
						with mockenv_context(**lowerCamelCase__     ):
							UpperCamelCase__ :Optional[Any]						=      Accelerator()
							if mp_dtype == "fp16":
								UpperCamelCase__ :Tuple						=      torch.floataa
							elif mp_dtype == "bf16":
								UpperCamelCase__ :Tuple						=      torch.bfloataa
							UpperCamelCase__ :int						=      MixedPrecision(param_dtype=lowerCamelCase__  ,     reduce_dtype=lowerCamelCase__  ,     buffer_dtype=lowerCamelCase__     )
							self.assertEqual(accelerator.state.fsdp_plugin.mixed_precision_policy  ,     lowerCamelCase__     )
							if mp_dtype == FPaa:
								self.assertTrue(isinstance(accelerator.scaler  ,     lowerCamelCase__     )     )
							elif mp_dtype == BFaa:
								self.assertIsNone(accelerator.scaler     )
							AcceleratorState._reset_state(lowerCamelCase__     )







				def 				__a   ( self     :Optional[Any]     ):
					from torch.distributed.fsdp.fully_sharded_data_parallel import CPUOffload

					for flag in [True, False]:
						UpperCamelCase__ :List[str]						=      self.dist_env.copy()
						UpperCamelCase__ :Dict						=      str(lowerCamelCase__     ).lower()
						with mockenv_context(**lowerCamelCase__     ):
							UpperCamelCase__ :List[str]						=      FullyShardedDataParallelPlugin()
							self.assertEqual(fsdp_plugin.cpu_offload  ,     CPUOffload(offload_params=lowerCamelCase__     )     )

@require_fsdp
@require_multi_gpu
@slow
class        lowerCAmelCase_				(      lowercase	):







				"""simple docstring"""




				def 				__a   ( self     :Dict     ):
					super().setUp()
					UpperCamelCase__ :str						=      0.82
					UpperCamelCase__ :int						=      [
					    """fsdp_shard_grad_op_transformer_based_wrap""",
					    """fsdp_full_shard_transformer_based_wrap""",
					]
					UpperCamelCase__ :int						=      {
					    """multi_gpu_fp16""": 32_00,
					    """fsdp_shard_grad_op_transformer_based_wrap_fp16""": 20_00,
					    """fsdp_full_shard_transformer_based_wrap_fp16""": 19_00,
					    # Disabling below test as it overwhelms the RAM memory usage
					    # on CI self-hosted runner leading to tests getting killed.
					    # "fsdp_full_shard_cpu_offload_transformer_based_wrap_fp32": 1500,  # fp16 was leading to indefinite hang
					}
					UpperCamelCase__ :Optional[Any]						=      1_60
					UpperCamelCase__ :List[str]						=      1_60

					UpperCamelCase__ :Union[str, Any]						=      inspect.getfile(accelerate.test_utils     )
					UpperCamelCase__ :Dict						=      os.path.sep.join(mod_file.split(os.path.sep     )[:-1] + ["""scripts""", """external_deps"""]     )







				def 				__a   ( self     :str     ):
					UpperCamelCase__ :int						=      os.path.join(self.test_scripts_folder  ,     """test_performance.py"""     )
					UpperCamelCase__ :List[str]						=      ["""accelerate""", """launch""", """--num_processes=2""", """--num_machines=1""", """--machine_rank=0""", """--use_fsdp"""]
					for config in self.performance_configs:
						UpperCamelCase__ :Optional[Any]						=      cmd.copy()
						for i, strategy in enumerate(lowerCamelCase__     ):
							if strategy.lower() in config:
								cmd_config.append(f"""--fsdp_sharding_strategy={i+1}"""     )
								break

						if "fp32" in config:
							cmd_config.append("""--mixed_precision=no"""     )
						else:
							cmd_config.append("""--mixed_precision=fp16"""     )

						if "cpu_offload" in config:
							cmd_config.append("""--fsdp_offload_params=True"""     )

						for policy in FSDP_AUTO_WRAP_POLICY:
							if policy.lower() in config:
								cmd_config.append(f"""--fsdp_auto_wrap_policy={policy}"""     )
								break

						if policy == "TRANSFORMER_BASED_WRAP":
							cmd_config.append("""--fsdp_transformer_layer_cls_to_wrap=BertLayer"""     )
						elif policy == "SIZE_BASED_WRAP":
							cmd_config.append("""--fsdp_min_num_params=2000"""     )

						cmd_config.extend(
						    [
						        self.test_file_path,
						        f"""--output_dir={self.tmpdir}""",
						        f"""--performance_lower_bound={self.performance_lower_bound}""",
						    ]     )
						with patch_environment(omp_num_threads=1     ):
							execute_subprocess_async(lowerCamelCase__  ,     env=os.environ.copy()     )







				def 				__a   ( self     :str     ):
					UpperCamelCase__ :List[Any]						=      os.path.join(self.test_scripts_folder  ,     """test_checkpointing.py"""     )
					UpperCamelCase__ :Any						=      [
					    """accelerate""",
					    """launch""",
					    """--num_processes=2""",
					    """--num_machines=1""",
					    """--machine_rank=0""",
					    """--use_fsdp""",
					    """--mixed_precision=fp16""",
					    """--fsdp_transformer_layer_cls_to_wrap=BertLayer""",
					]

					for i, strategy in enumerate(lowerCamelCase__     ):
						UpperCamelCase__ :Optional[Any]						=      cmd.copy()
						cmd_config.append(f"""--fsdp_sharding_strategy={i+1}"""     )
						if strategy != "FULL_SHARD":
							continue
						UpperCamelCase__ :Optional[int]						=      len(lowerCamelCase__     )
						for state_dict_type in FSDP_STATE_DICT_TYPE:
							UpperCamelCase__ :Tuple						=      cmd_config[:state_dict_config_index]
							cmd_config.append(f"""--fsdp_state_dict_type={state_dict_type}"""     )
							cmd_config.extend(
							    [
							        self.test_file_path,
							        f"""--output_dir={self.tmpdir}""",
							        """--partial_train_epoch=1""",
							    ]     )
							with patch_environment(omp_num_threads=1     ):
								execute_subprocess_async(lowerCamelCase__  ,     env=os.environ.copy()     )

							UpperCamelCase__ :List[Any]						=      cmd_config[:-1]
							UpperCamelCase__ :Tuple						=      os.path.join(self.tmpdir  ,     """epoch_0"""     )
							cmd_config.extend(
							    [
							        f"""--resume_from_checkpoint={resume_from_checkpoint}""",
							    ]     )
							with patch_environment(omp_num_threads=1     ):
								execute_subprocess_async(lowerCamelCase__  ,     env=os.environ.copy()     )







				def 				__a   ( self     :List[str]     ):
					UpperCamelCase__ :List[str]						=      os.path.join(self.test_scripts_folder  ,     """test_peak_memory_usage.py"""     )
					UpperCamelCase__ :Optional[int]						=      [
					    """accelerate""",
					    """launch""",
					    """--num_processes=2""",
					    """--num_machines=1""",
					    """--machine_rank=0""",
					]
					for spec, peak_mem_upper_bound in self.peak_memory_usage_upper_bound.items():
						UpperCamelCase__ :Optional[int]						=      cmd.copy()
						if "fp16" in spec:
							cmd_config.extend(["""--mixed_precision=fp16"""]     )
						else:
							cmd_config.extend(["""--mixed_precision=no"""]     )

						if "multi_gpu" in spec:
							continue
						else:
							cmd_config.extend(["""--use_fsdp"""]     )
							for i, strategy in enumerate(lowerCamelCase__     ):
								if strategy.lower() in spec:
									cmd_config.append(f"""--fsdp_sharding_strategy={i+1}"""     )
									break

							if "cpu_offload" in spec:
								cmd_config.append("""--fsdp_offload_params=True"""     )

							for policy in FSDP_AUTO_WRAP_POLICY:
								if policy.lower() in spec:
									cmd_config.append(f"""--fsdp_auto_wrap_policy={policy}"""     )
									break

							if policy == "TRANSFORMER_BASED_WRAP":
								cmd_config.append("""--fsdp_transformer_layer_cls_to_wrap=BertLayer"""     )
							elif policy == "SIZE_BASED_WRAP":
								cmd_config.append("""--fsdp_min_num_params=2000"""     )

						cmd_config.extend(
						    [
						        self.test_file_path,
						        f"""--output_dir={self.tmpdir}""",
						        f"""--peak_memory_upper_bound={peak_mem_upper_bound}""",
						        f"""--n_train={self.n_train}""",
						        f"""--n_val={self.n_val}""",
						    ]     )
						with patch_environment(omp_num_threads=1     ):
							execute_subprocess_async(lowerCamelCase__  ,     env=os.environ.copy()     )

'''simple docstring'''
import copy
import os

import cva
import numpy as np
from matplotlib import pyplot as plt


class 			SCREAMING_SNAKE_CASE__      :







				def __init__(       self     )->		Dict:







											'''simple docstring'''






											UpperCamelCase   			=     ''
											UpperCamelCase   			=     ''
											UpperCamelCase   			=     []
											UpperCamelCase   			=     0
											UpperCamelCase   			=     256
											UpperCamelCase   			=     0
											UpperCamelCase   			=     0
											UpperCamelCase   			=     0
											UpperCamelCase   			=     0







				def 						UpperCAmelCase_    (       self						,				A_     )->		str:







											'''simple docstring'''






											UpperCamelCase   			=     cva.imread(A_						,				0     )
											UpperCamelCase   			=     copy.deepcopy(self.img     )
											UpperCamelCase			,	UpperCamelCase			,	UpperCamelCase   			=     plt.hist(self.img.ravel()						,				256						,				[0, 256]						,				label='x'     )
											UpperCamelCase   			=     np.sum(A_     )
											for i in range(len(A_     )     ):
																		UpperCamelCase   			=     x[i] / self.k
																		self.sk += prk
																		UpperCamelCase   			=     (self.L - 1) * self.sk
																		if self.rem != 0:
																									UpperCamelCase   			=     int(last % last     )
																		UpperCamelCase   			=     int(last + 1 if self.rem >= 0.5 else last     )
																		self.last_list.append(A_     )
																		UpperCamelCase   			=     int(np.ma.count(self.img     ) / self.img[1].size     )
																		UpperCamelCase   			=     self.img[1].size
											for i in range(self.number_of_cols     ):
																		for j in range(self.number_of_rows     ):
																									UpperCamelCase   			=     self.img[j][i]
																									if num != self.last_list[num]:
																																UpperCamelCase   			=     self.last_list[num]
											cva.imwrite('output_data/output.jpg'						,				self.img     )







				def 						UpperCAmelCase_    (       self     )->		Any:







											'''simple docstring'''






											plt.hist(self.img.ravel()						,				256						,				[0, 256]     )







				def 						UpperCAmelCase_    (       self     )->		Optional[Any]:







											'''simple docstring'''






											cva.imshow('Output-Image'						,				self.img     )
											cva.imshow('Input-Image'						,				self.original_image     )
											cva.waitKey(5000     )
											cva.destroyAllWindows()


if __name__ == "__main__":
						lowerCAmelCase					:    Union[str, Any]           = os.path.join(os.path.basename(__file__), 'image_data/input.jpg')
						lowerCAmelCase					:    str           = ConstantStretch()
						stretcher.stretch(file_path)
						stretcher.plot_histogram()
						stretcher.show_image()




