
'''simple docstring'''
import inspect
import os

import torch
from transformers import AutoModel
from transformers.testing_utils import mockenv_context
from transformers.trainer_utils import set_seed

import accelerate
from accelerate.accelerator import Accelerator
from accelerate.state import AcceleratorState
from accelerate.test_utils.testing import (
    AccelerateTestCase,
    TempDirTestCase,
    execute_subprocess_async,
    require_cuda,
    require_fsdp,
    require_multi_gpu,
    slow,
)
from accelerate.utils.constants import (
    FSDP_AUTO_WRAP_POLICY,
    FSDP_BACKWARD_PREFETCH,
    FSDP_SHARDING_STRATEGY,
    FSDP_STATE_DICT_TYPE,
)
from accelerate.utils.dataclasses import FullyShardedDataParallelPlugin
from accelerate.utils.other import patch_environment


set_seed(42)

lowercase__											='bert-base-cased'
lowercase__											='fp16'
lowercase__											='bf16'
lowercase__											=[FPaa, BFaa]







@require_fsdp
@require_cuda
class   a_ (      UpperCamelCase__					):




			def 					lowerCAmelCase__					(       self	):
				super().setUp()

				a_      				=						dict(
				    ACCELERATE_USE_FSDP="""true"""					,   MASTER_ADDR="""localhost"""					,   MASTER_PORT="""10999"""					,   RANK="""0"""					,   LOCAL_RANK="""0"""					,   WORLD_SIZE="""1"""					,   )




			def 					lowerCAmelCase__					(       self	):
				from torch.distributed.fsdp.fully_sharded_data_parallel import ShardingStrategy

				for i, strategy in enumerate(UpperCAmelCase	):
					a_      				=						self.dist_env.copy()
					a_      				=						f'''{i + 1}'''
					a_      				=						strategy
					with mockenv_context(**UpperCAmelCase	):
						a_      				=						FullyShardedDataParallelPlugin()
						self.assertEqual(fsdp_plugin.sharding_strategy					,   ShardingStrategy(i + 1	)	)




			def 					lowerCAmelCase__					(       self	):
				from torch.distributed.fsdp.fully_sharded_data_parallel import BackwardPrefetch

				for i, prefetch_policy in enumerate(UpperCAmelCase	):
					a_      				=						self.dist_env.copy()
					a_      				=						prefetch_policy
					with mockenv_context(**UpperCAmelCase	):
						a_      				=						FullyShardedDataParallelPlugin()
						if prefetch_policy == "NO_PREFETCH":
							self.assertIsNone(fsdp_plugin.backward_prefetch	)
						else:
							self.assertEqual(fsdp_plugin.backward_prefetch					,   BackwardPrefetch(i + 1	)	)




			def 					lowerCAmelCase__					(       self	):
				from torch.distributed.fsdp.fully_sharded_data_parallel import StateDictType

				for i, state_dict_type in enumerate(UpperCAmelCase	):
					a_      				=						self.dist_env.copy()
					a_      				=						state_dict_type
					with mockenv_context(**UpperCAmelCase	):
						a_      				=						FullyShardedDataParallelPlugin()
						self.assertEqual(fsdp_plugin.state_dict_type					,   StateDictType(i + 1	)	)
						if state_dict_type == "FULL_STATE_DICT":
							self.assertTrue(fsdp_plugin.state_dict_config.offload_to_cpu	)
							self.assertTrue(fsdp_plugin.state_dict_config.ranka_only	)




			def 					lowerCAmelCase__					(       self	):
				a_      				=						AutoModel.from_pretrained(UpperCAmelCase	)
				for policy in FSDP_AUTO_WRAP_POLICY:
					a_      				=						self.dist_env.copy()
					a_      				=						policy
					if policy == "TRANSFORMER_BASED_WRAP":
						a_      				=						"""BertLayer"""
					elif policy == "SIZE_BASED_WRAP":
						a_      				=						"""2000"""
					with mockenv_context(**UpperCAmelCase	):
						a_      				=						FullyShardedDataParallelPlugin()
						fsdp_plugin.set_auto_wrap_policy(UpperCAmelCase	)
						if policy == "NO_WRAP":
							self.assertIsNone(fsdp_plugin.auto_wrap_policy	)
						else:
							self.assertIsNotNone(fsdp_plugin.auto_wrap_policy	)

				a_      				=						self.dist_env.copy()
				a_      				=						"""TRANSFORMER_BASED_WRAP"""
				a_      				=						"""T5Layer"""
				with mockenv_context(**UpperCAmelCase	):
					a_      				=						FullyShardedDataParallelPlugin()
					with self.assertRaises(UpperCAmelCase	) as cm:
						fsdp_plugin.set_auto_wrap_policy(UpperCAmelCase	)
					self.assertTrue("""Could not find the transformer layer class to wrap in the model.""" in str(cm.exception	)	)

				a_      				=						self.dist_env.copy()
				a_      				=						"""SIZE_BASED_WRAP"""
				a_      				=						"""0"""
				with mockenv_context(**UpperCAmelCase	):
					a_      				=						FullyShardedDataParallelPlugin()
					fsdp_plugin.set_auto_wrap_policy(UpperCAmelCase	)
					self.assertIsNone(fsdp_plugin.auto_wrap_policy	)




			def 					lowerCAmelCase__					(       self	):
				from torch.distributed.fsdp.fully_sharded_data_parallel import MixedPrecision
				from torch.distributed.fsdp.sharded_grad_scaler import ShardedGradScaler

				for mp_dtype in dtypes:
					a_      				=						self.dist_env.copy()
					a_      				=						mp_dtype
					with mockenv_context(**UpperCAmelCase	):
						a_      				=						Accelerator()
						if mp_dtype == "fp16":
							a_      				=						torch.floataa
						elif mp_dtype == "bf16":
							a_      				=						torch.bfloataa
						a_      				=						MixedPrecision(param_dtype=UpperCAmelCase					,   reduce_dtype=UpperCAmelCase					,   buffer_dtype=UpperCAmelCase	)
						self.assertEqual(accelerator.state.fsdp_plugin.mixed_precision_policy					,   UpperCAmelCase	)
						if mp_dtype == FPaa:
							self.assertTrue(isinstance(accelerator.scaler					,   UpperCAmelCase	)	)
						elif mp_dtype == BFaa:
							self.assertIsNone(accelerator.scaler	)
						AcceleratorState._reset_state(UpperCAmelCase	)






			def 					lowerCAmelCase__					(       self	):
				from torch.distributed.fsdp.fully_sharded_data_parallel import CPUOffload

				for flag in [True, False]:
					a_      				=						self.dist_env.copy()
					a_      				=						str(UpperCAmelCase	).lower()
					with mockenv_context(**UpperCAmelCase	):
						a_      				=						FullyShardedDataParallelPlugin()
						self.assertEqual(fsdp_plugin.cpu_offload					,   CPUOffload(offload_params=UpperCAmelCase	)	)







@require_fsdp
@require_multi_gpu
@slow
class   a_ (      UpperCamelCase__					):




			def 					lowerCAmelCase__					(       self	):
				super().setUp()
				a_      				=						0.82
				a_      				=						[
				    """fsdp_shard_grad_op_transformer_based_wrap""",
				    """fsdp_full_shard_transformer_based_wrap""",
				]
				a_      				=						{
				    """multi_gpu_fp16""": 32_00,
				    """fsdp_shard_grad_op_transformer_based_wrap_fp16""": 20_00,
				    """fsdp_full_shard_transformer_based_wrap_fp16""": 19_00,
				    # Disabling below test as it overwhelms the RAM memory usage
				    # on CI self-hosted runner leading to tests getting killed.
				    # "fsdp_full_shard_cpu_offload_transformer_based_wrap_fp32": 1500,  # fp16 was leading to indefinite hang
				}
				a_      				=						1_60
				a_      				=						1_60

				a_      				=						inspect.getfile(accelerate.test_utils	)
				a_      				=						os.path.sep.join(mod_file.split(os.path.sep	)[:-1] + ["""scripts""", """external_deps"""]	)




			def 					lowerCAmelCase__					(       self	):
				a_      				=						os.path.join(self.test_scripts_folder					,   """test_performance.py"""	)
				a_      				=						["""accelerate""", """launch""", """--num_processes=2""", """--num_machines=1""", """--machine_rank=0""", """--use_fsdp"""]
				for config in self.performance_configs:
					a_      				=						cmd.copy()
					for i, strategy in enumerate(UpperCAmelCase	):
						if strategy.lower() in config:
							cmd_config.append(f'''--fsdp_sharding_strategy={i+1}'''	)
							break

					if "fp32" in config:
						cmd_config.append("""--mixed_precision=no"""	)
					else:
						cmd_config.append("""--mixed_precision=fp16"""	)

					if "cpu_offload" in config:
						cmd_config.append("""--fsdp_offload_params=True"""	)

					for policy in FSDP_AUTO_WRAP_POLICY:
						if policy.lower() in config:
							cmd_config.append(f'''--fsdp_auto_wrap_policy={policy}'''	)
							break

					if policy == "TRANSFORMER_BASED_WRAP":
						cmd_config.append("""--fsdp_transformer_layer_cls_to_wrap=BertLayer"""	)
					elif policy == "SIZE_BASED_WRAP":
						cmd_config.append("""--fsdp_min_num_params=2000"""	)

					cmd_config.extend(
					    [
					        self.test_file_path,
					        f'''--output_dir={self.tmpdir}''',
					        f'''--performance_lower_bound={self.performance_lower_bound}''',
					    ]	)
					with patch_environment(omp_num_threads=1	):
						execute_subprocess_async(UpperCAmelCase					,   env=os.environ.copy()	)




			def 					lowerCAmelCase__					(       self	):
				a_      				=						os.path.join(self.test_scripts_folder					,   """test_checkpointing.py"""	)
				a_      				=						[
				    """accelerate""",
				    """launch""",
				    """--num_processes=2""",
				    """--num_machines=1""",
				    """--machine_rank=0""",
				    """--use_fsdp""",
				    """--mixed_precision=fp16""",
				    """--fsdp_transformer_layer_cls_to_wrap=BertLayer""",
				]

				for i, strategy in enumerate(UpperCAmelCase	):
					a_      				=						cmd.copy()
					cmd_config.append(f'''--fsdp_sharding_strategy={i+1}'''	)
					if strategy != "FULL_SHARD":
						continue
					a_      				=						len(UpperCAmelCase	)
					for state_dict_type in FSDP_STATE_DICT_TYPE:
						a_      				=						cmd_config[:state_dict_config_index]
						cmd_config.append(f'''--fsdp_state_dict_type={state_dict_type}'''	)
						cmd_config.extend(
						    [
						        self.test_file_path,
						        f'''--output_dir={self.tmpdir}''',
						        """--partial_train_epoch=1""",
						    ]	)
						with patch_environment(omp_num_threads=1	):
							execute_subprocess_async(UpperCAmelCase					,   env=os.environ.copy()	)

						a_      				=						cmd_config[:-1]
						a_      				=						os.path.join(self.tmpdir					,   """epoch_0"""	)
						cmd_config.extend(
						    [
						        f'''--resume_from_checkpoint={resume_from_checkpoint}''',
						    ]	)
						with patch_environment(omp_num_threads=1	):
							execute_subprocess_async(UpperCAmelCase					,   env=os.environ.copy()	)






			def 					lowerCAmelCase__					(       self	):
				a_      				=						os.path.join(self.test_scripts_folder					,   """test_peak_memory_usage.py"""	)
				a_      				=						[
				    """accelerate""",
				    """launch""",
				    """--num_processes=2""",
				    """--num_machines=1""",
				    """--machine_rank=0""",
				]
				for spec, peak_mem_upper_bound in self.peak_memory_usage_upper_bound.items():
					a_      				=						cmd.copy()
					if "fp16" in spec:
						cmd_config.extend(["""--mixed_precision=fp16"""]	)
					else:
						cmd_config.extend(["""--mixed_precision=no"""]	)

					if "multi_gpu" in spec:
						continue
					else:
						cmd_config.extend(["""--use_fsdp"""]	)
						for i, strategy in enumerate(UpperCAmelCase	):
							if strategy.lower() in spec:
								cmd_config.append(f'''--fsdp_sharding_strategy={i+1}'''	)
								break

						if "cpu_offload" in spec:
							cmd_config.append("""--fsdp_offload_params=True"""	)

						for policy in FSDP_AUTO_WRAP_POLICY:
							if policy.lower() in spec:
								cmd_config.append(f'''--fsdp_auto_wrap_policy={policy}'''	)
								break

						if policy == "TRANSFORMER_BASED_WRAP":
							cmd_config.append("""--fsdp_transformer_layer_cls_to_wrap=BertLayer"""	)
						elif policy == "SIZE_BASED_WRAP":
							cmd_config.append("""--fsdp_min_num_params=2000"""	)

					cmd_config.extend(
					    [
					        self.test_file_path,
					        f'''--output_dir={self.tmpdir}''',
					        f'''--peak_memory_upper_bound={peak_mem_upper_bound}''',
					        f'''--n_train={self.n_train}''',
					        f'''--n_val={self.n_val}''',
					    ]	)
					with patch_environment(omp_num_threads=1	):
						execute_subprocess_async(UpperCAmelCase					,   env=os.environ.copy()	)


'''simple docstring'''
from __future__ import annotations






def      UpperCamelCase_				(      A__						):
	a_      				=						[True] * limit
	a_      				=						False
	a_      				=						False
	a_      				=						True

	for i in range(3  ,      int(limit**0.5 + 1						)  ,      2						):
		a_      				=						i * 2
		while index < limit:
			a_      				=						False
			a_      				=						index + i

	a_      				=						[2]

	for i in range(3  ,      A__  ,      2						):
		if is_prime[i]:
			primes.append(A__						)

	return primes






def      UpperCamelCase_				(      A__ = 1_00_00_00						):
	a_      				=						prime_sieve(A__						)
	a_      				=						0
	a_      				=						0

	for i in range(len(A__						)						):
		for j in range(i + length  ,      len(A__						)						):
			a_      				=						sum(primes[i:j]						)
			if sol >= ceiling:
				break

			if sol in primes:
				a_      				=						j - i
				a_      				=						sol

	return largest


if __name__ == "__main__":
		print(F"""{solution() = }""")
