





'''simple docstring'''






import importlib
import math
import os
from dataclasses import dataclass
from enum import Enum
from typing import Any, Dict, Optional, Tuple, Union

import flax
import jax.numpy as jnp

from ..utils import BaseOutput


__lowerCamelCase						: Tuple		=   'scheduler_config.json'







class 		UpperCAmelCase   (				lowerCAmelCase__):


  """simple docstring"""







  lowerCAmelCase_					=				1
  lowerCAmelCase_					=				2
  lowerCAmelCase_					=				3
  lowerCAmelCase_					=				4
  lowerCAmelCase_					=				5







@dataclass
class 		UpperCAmelCase   (				lowerCAmelCase__):


  """simple docstring"""







  lowerCAmelCase_					=				42







class 		UpperCAmelCase   :


  """simple docstring"""







  lowerCAmelCase_					=				SCHEDULER_CONFIG_NAME
  lowerCAmelCase_					=				["dtype"]
  lowerCAmelCase_					=				[]
  lowerCAmelCase_					=				True


  @classmethod
  def   UpperCamelCase__      (				cls    :					List[str]							,	UpperCamelCase__    :					Optional[int] = None							,	UpperCamelCase__    :					int = None							,	UpperCamelCase__    :					Tuple=False							,	**UpperCamelCase__    :					Any							,	)     ->						Tuple:
    _UpperCamelCase			,     _UpperCamelCase       			=cls.load_config(
        pretrained_model_name_or_path=_lowerCamelCase							,	subfolder=_lowerCamelCase							,	return_unused_kwargs=_lowerCamelCase							,	**_lowerCamelCase							,	)
    _UpperCamelCase			,     _UpperCamelCase       			=cls.from_config(_lowerCamelCase							,	return_unused_kwargs=_lowerCamelCase							,	**_lowerCamelCase  )

    if hasattr(_lowerCamelCase							,	'''create_state'''  ) and getattr(_lowerCamelCase							,	'''has_state'''							,	_lowerCamelCase  ):
      _UpperCamelCase       			=scheduler.create_state()

    if return_unused_kwargs:
      return scheduler, state, unused_kwargs

    return scheduler, state


  def   UpperCamelCase__      (				self    :					int							,	UpperCamelCase__    :					Optional[int]							,	UpperCamelCase__    :					str = False							,	**UpperCamelCase__    :					Any  )     ->						str:
    self.save_config(save_directory=_lowerCamelCase							,	push_to_hub=_lowerCamelCase							,	**_lowerCamelCase  )


  @property
  def   UpperCamelCase__      (				self    :					Tuple  )     ->						List[Any]:
    return self._get_compatibles()




  @classmethod
  def   UpperCamelCase__      (				cls    :					Optional[int]  )     ->						int:
    _UpperCamelCase       			=list(set([cls.__name__] + cls._compatibles  )  )
    _UpperCamelCase       			=importlib.import_module(__name__.split('''.'''  )[0]  )
    _UpperCamelCase       			=[
        getattr(_lowerCamelCase							,	_lowerCamelCase  ) for c in compatible_classes_str if hasattr(_lowerCamelCase							,	_lowerCamelCase  )
    ]
    return compatible_classes




def 					_a   (__SCREAMING_SNAKE_CASE  ,      __SCREAMING_SNAKE_CASE   ):


  """simple docstring"""

  assert len(lowerCamelCase_   ) >= x.ndim
  return jnp.broadcast_to(x.reshape(x.shape + (1,) * (len(lowerCamelCase_   ) - x.ndim)   )  ,      lowerCamelCase_   )


def 					_a   (__SCREAMING_SNAKE_CASE  ,      __SCREAMING_SNAKE_CASE=0.9_9_9  ,      __SCREAMING_SNAKE_CASE=jnp.floataa   ):


  """simple docstring"""

  def alpha_bar(__SCREAMING_SNAKE_CASE   ):
    return math.cos((time_step + 0.0_0_8) / 1.0_0_8 * math.pi / 2   ) ** 2

  _UpperCamelCase       			=[]
  for i in range(lowerCamelCase_   ):
    _UpperCamelCase       			=i / num_diffusion_timesteps
    _UpperCamelCase       			=(i + 1) / num_diffusion_timesteps
    betas.append(min(1 - alpha_bar(lowerCamelCase_   ) / alpha_bar(lowerCamelCase_   )  ,      lowerCamelCase_   )   )
  return jnp.array(lowerCamelCase_  ,      dtype=lowerCamelCase_   )







@flax.struct.dataclass
class 		UpperCAmelCase   :


  """simple docstring"""







  lowerCAmelCase_					=				42
  lowerCAmelCase_					=				42
  lowerCAmelCase_					=				42


  @classmethod
  def   UpperCamelCase__      (				cls    :					List[str]							,	UpperCamelCase__    :					int  )     ->						Optional[Any]:
    _UpperCamelCase       			=scheduler.config

    if config.trained_betas is not None:
      _UpperCamelCase       			=jnp.asarray(config.trained_betas							,	dtype=scheduler.dtype  )
    elif config.beta_schedule == "linear":
      _UpperCamelCase       			=jnp.linspace(config.beta_start							,	config.beta_end							,	config.num_train_timesteps							,	dtype=scheduler.dtype  )
    elif config.beta_schedule == "scaled_linear":
      # this schedule is very specific to the latent diffusion model.
      _UpperCamelCase       			=(
          jnp.linspace(
              config.beta_start**0.5							,	config.beta_end**0.5							,	config.num_train_timesteps							,	dtype=scheduler.dtype  )
          ** 2
      )
    elif config.beta_schedule == "squaredcos_cap_v2":
      # Glide cosine schedule
      _UpperCamelCase       			=betas_for_alpha_bar(config.num_train_timesteps							,	dtype=scheduler.dtype  )
    else:
      raise NotImplementedError(
          F'''beta_schedule {config.beta_schedule} is not implemented for scheduler {scheduler.__class__.__name__}'''  )

    _UpperCamelCase       			=1.0 - betas

    _UpperCamelCase       			=jnp.cumprod(_lowerCamelCase							,	axis=0  )

    return cls(
        alphas=_lowerCamelCase							,	betas=_lowerCamelCase							,	alphas_cumprod=_lowerCamelCase							,	)




def 					_a   (__SCREAMING_SNAKE_CASE  ,      __SCREAMING_SNAKE_CASE  ,      __SCREAMING_SNAKE_CASE  ,      __SCREAMING_SNAKE_CASE   ):


  """simple docstring"""

  _UpperCamelCase       			=state.alphas_cumprod

  _UpperCamelCase       			=alphas_cumprod[timesteps] ** 0.5
  _UpperCamelCase       			=sqrt_alpha_prod.flatten()
  _UpperCamelCase       			=broadcast_to_shape_from_left(lowerCamelCase_  ,      original_samples.shape   )

  _UpperCamelCase       			=(1 - alphas_cumprod[timesteps]) ** 0.5
  _UpperCamelCase       			=sqrt_one_minus_alpha_prod.flatten()
  _UpperCamelCase       			=broadcast_to_shape_from_left(lowerCamelCase_  ,      original_samples.shape   )

  return sqrt_alpha_prod, sqrt_one_minus_alpha_prod


def 					_a   (__SCREAMING_SNAKE_CASE  ,      __SCREAMING_SNAKE_CASE  ,      __SCREAMING_SNAKE_CASE  ,      __SCREAMING_SNAKE_CASE   ):


  """simple docstring"""

  _UpperCamelCase			,     _UpperCamelCase       			=get_sqrt_alpha_prod(lowerCamelCase_  ,      lowerCamelCase_  ,      lowerCamelCase_  ,      lowerCamelCase_   )
  _UpperCamelCase       			=sqrt_alpha_prod * original_samples + sqrt_one_minus_alpha_prod * noise
  return noisy_samples


def 					_a   (__SCREAMING_SNAKE_CASE  ,      __SCREAMING_SNAKE_CASE  ,      __SCREAMING_SNAKE_CASE  ,      __SCREAMING_SNAKE_CASE   ):


  """simple docstring"""

  _UpperCamelCase			,     _UpperCamelCase       			=get_sqrt_alpha_prod(lowerCamelCase_  ,      lowerCamelCase_  ,      lowerCamelCase_  ,      lowerCamelCase_   )
  _UpperCamelCase       			=sqrt_alpha_prod * noise - sqrt_one_minus_alpha_prod * sample
  return velocity













'''simple docstring'''






import inspect
import os

import torch
from transformers import AutoModel
from transformers.testing_utils import mockenv_context
from transformers.trainer_utils import set_seed

import accelerate
from accelerate.accelerator import Accelerator
from accelerate.state import AcceleratorState
from accelerate.test_utils.testing import (
    AccelerateTestCase,
    TempDirTestCase,
    execute_subprocess_async,
    require_cuda,
    require_fsdp,
    require_multi_gpu,
    slow,
)
from accelerate.utils.constants import (
    FSDP_AUTO_WRAP_POLICY,
    FSDP_BACKWARD_PREFETCH,
    FSDP_SHARDING_STRATEGY,
    FSDP_STATE_DICT_TYPE,
)
from accelerate.utils.dataclasses import FullyShardedDataParallelPlugin
from accelerate.utils.other import patch_environment


set_seed(42)

__lowerCamelCase						: List[Any]		=   'bert-base-cased'
__lowerCamelCase						: str		=   'fp16'
__lowerCamelCase						: Optional[int]		=   'bf16'
__lowerCamelCase						: List[Any]		=   [FPaa, BFaa]







@require_fsdp
@require_cuda
class 		UpperCAmelCase   (				lowercase_):


  """simple docstring"""







  def   UpperCamelCase__      (				self    :					Tuple  )     ->						Union[str, Any]:
    super().setUp()

    _UpperCamelCase       			=dict(
        ACCELERATE_USE_FSDP='''true'''							,	MASTER_ADDR='''localhost'''							,	MASTER_PORT='''10999'''							,	RANK='''0'''							,	LOCAL_RANK='''0'''							,	WORLD_SIZE='''1'''							,	)


  def   UpperCamelCase__      (				self    :					int  )     ->						List[Any]:
    from torch.distributed.fsdp.fully_sharded_data_parallel import ShardingStrategy

    for i, strategy in enumerate(UpperCamelCase__  ):
      _UpperCamelCase       			=self.dist_env.copy()
      _UpperCamelCase       			=F'''{i + 1}'''
      _UpperCamelCase       			=strategy
      with mockenv_context(**UpperCamelCase__  ):
        _UpperCamelCase       			=FullyShardedDataParallelPlugin()
        self.assertEqual(fsdp_plugin.sharding_strategy							,	ShardingStrategy(i + 1  )  )


  def   UpperCamelCase__      (				self    :					Union[str, Any]  )     ->						Any:
    from torch.distributed.fsdp.fully_sharded_data_parallel import BackwardPrefetch

    for i, prefetch_policy in enumerate(UpperCamelCase__  ):
      _UpperCamelCase       			=self.dist_env.copy()
      _UpperCamelCase       			=prefetch_policy
      with mockenv_context(**UpperCamelCase__  ):
        _UpperCamelCase       			=FullyShardedDataParallelPlugin()
        if prefetch_policy == "NO_PREFETCH":
          self.assertIsNone(fsdp_plugin.backward_prefetch  )
        else:
          self.assertEqual(fsdp_plugin.backward_prefetch							,	BackwardPrefetch(i + 1  )  )


  def   UpperCamelCase__      (				self    :					Optional[Any]  )     ->						Tuple:
    from torch.distributed.fsdp.fully_sharded_data_parallel import StateDictType

    for i, state_dict_type in enumerate(UpperCamelCase__  ):
      _UpperCamelCase       			=self.dist_env.copy()
      _UpperCamelCase       			=state_dict_type
      with mockenv_context(**UpperCamelCase__  ):
        _UpperCamelCase       			=FullyShardedDataParallelPlugin()
        self.assertEqual(fsdp_plugin.state_dict_type							,	StateDictType(i + 1  )  )
        if state_dict_type == "FULL_STATE_DICT":
          self.assertTrue(fsdp_plugin.state_dict_config.offload_to_cpu  )
          self.assertTrue(fsdp_plugin.state_dict_config.ranka_only  )


  def   UpperCamelCase__      (				self    :					Any  )     ->						List[Any]:
    _UpperCamelCase       			=AutoModel.from_pretrained(UpperCamelCase__  )
    for policy in FSDP_AUTO_WRAP_POLICY:
      _UpperCamelCase       			=self.dist_env.copy()
      _UpperCamelCase       			=policy
      if policy == "TRANSFORMER_BASED_WRAP":
        _UpperCamelCase       			='''BertLayer'''
      elif policy == "SIZE_BASED_WRAP":
        _UpperCamelCase       			='''2000'''
      with mockenv_context(**UpperCamelCase__  ):
        _UpperCamelCase       			=FullyShardedDataParallelPlugin()
        fsdp_plugin.set_auto_wrap_policy(UpperCamelCase__  )
        if policy == "NO_WRAP":
          self.assertIsNone(fsdp_plugin.auto_wrap_policy  )
        else:
          self.assertIsNotNone(fsdp_plugin.auto_wrap_policy  )

    _UpperCamelCase       			=self.dist_env.copy()
    _UpperCamelCase       			='''TRANSFORMER_BASED_WRAP'''
    _UpperCamelCase       			='''T5Layer'''
    with mockenv_context(**UpperCamelCase__  ):
      _UpperCamelCase       			=FullyShardedDataParallelPlugin()
      with self.assertRaises(UpperCamelCase__  ) as cm:
        fsdp_plugin.set_auto_wrap_policy(UpperCamelCase__  )
      self.assertTrue('''Could not find the transformer layer class to wrap in the model.''' in str(cm.exception  )  )

    _UpperCamelCase       			=self.dist_env.copy()
    _UpperCamelCase       			='''SIZE_BASED_WRAP'''
    _UpperCamelCase       			='''0'''
    with mockenv_context(**UpperCamelCase__  ):
      _UpperCamelCase       			=FullyShardedDataParallelPlugin()
      fsdp_plugin.set_auto_wrap_policy(UpperCamelCase__  )
      self.assertIsNone(fsdp_plugin.auto_wrap_policy  )


  def   UpperCamelCase__      (				self    :					Any  )     ->						List[Any]:
    from torch.distributed.fsdp.fully_sharded_data_parallel import MixedPrecision
    from torch.distributed.fsdp.sharded_grad_scaler import ShardedGradScaler

    for mp_dtype in dtypes:
      _UpperCamelCase       			=self.dist_env.copy()
      _UpperCamelCase       			=mp_dtype
      with mockenv_context(**UpperCamelCase__  ):
        _UpperCamelCase       			=Accelerator()
        if mp_dtype == "fp16":
          _UpperCamelCase       			=torch.floataa
        elif mp_dtype == "bf16":
          _UpperCamelCase       			=torch.bfloataa
        _UpperCamelCase       			=MixedPrecision(param_dtype=UpperCamelCase__							,	reduce_dtype=UpperCamelCase__							,	buffer_dtype=UpperCamelCase__  )
        self.assertEqual(accelerator.state.fsdp_plugin.mixed_precision_policy							,	UpperCamelCase__  )
        if mp_dtype == FPaa:
          self.assertTrue(isinstance(accelerator.scaler							,	UpperCamelCase__  )  )
        elif mp_dtype == BFaa:
          self.assertIsNone(accelerator.scaler  )
        AcceleratorState._reset_state(UpperCamelCase__  )




  def   UpperCamelCase__      (				self    :					Dict  )     ->						Tuple:
    from torch.distributed.fsdp.fully_sharded_data_parallel import CPUOffload

    for flag in [True, False]:
      _UpperCamelCase       			=self.dist_env.copy()
      _UpperCamelCase       			=str(UpperCamelCase__  ).lower()
      with mockenv_context(**UpperCamelCase__  ):
        _UpperCamelCase       			=FullyShardedDataParallelPlugin()
        self.assertEqual(fsdp_plugin.cpu_offload							,	CPUOffload(offload_params=UpperCamelCase__  )  )







@require_fsdp
@require_multi_gpu
@slow
class 		UpperCAmelCase   (				lowercase_):


  """simple docstring"""







  def   UpperCamelCase__      (				self    :					Union[str, Any]  )     ->						List[str]:
    super().setUp()
    _UpperCamelCase       			=0.82
    _UpperCamelCase       			=[
        '''fsdp_shard_grad_op_transformer_based_wrap''',
        '''fsdp_full_shard_transformer_based_wrap''',
    ]
    _UpperCamelCase       			={
        '''multi_gpu_fp16''': 3200,
        '''fsdp_shard_grad_op_transformer_based_wrap_fp16''': 2000,
        '''fsdp_full_shard_transformer_based_wrap_fp16''': 1900,
        # Disabling below test as it overwhelms the RAM memory usage
        # on CI self-hosted runner leading to tests getting killed.
        # "fsdp_full_shard_cpu_offload_transformer_based_wrap_fp32": 1500,  # fp16 was leading to indefinite hang
    }
    _UpperCamelCase       			=160
    _UpperCamelCase       			=160

    _UpperCamelCase       			=inspect.getfile(accelerate.test_utils  )
    _UpperCamelCase       			=os.path.sep.join(mod_file.split(os.path.sep  )[:-1] + ['''scripts''', '''external_deps''']  )


  def   UpperCamelCase__      (				self    :					int  )     ->						str:
    _UpperCamelCase       			=os.path.join(self.test_scripts_folder							,	'''test_performance.py'''  )
    _UpperCamelCase       			=['''accelerate''', '''launch''', '''--num_processes=2''', '''--num_machines=1''', '''--machine_rank=0''', '''--use_fsdp''']
    for config in self.performance_configs:
      _UpperCamelCase       			=cmd.copy()
      for i, strategy in enumerate(UpperCamelCase__  ):
        if strategy.lower() in config:
          cmd_config.append(F'''--fsdp_sharding_strategy={i+1}'''  )
          break

      if "fp32" in config:
        cmd_config.append('''--mixed_precision=no'''  )
      else:
        cmd_config.append('''--mixed_precision=fp16'''  )

      if "cpu_offload" in config:
        cmd_config.append('''--fsdp_offload_params=True'''  )

      for policy in FSDP_AUTO_WRAP_POLICY:
        if policy.lower() in config:
          cmd_config.append(F'''--fsdp_auto_wrap_policy={policy}'''  )
          break

      if policy == "TRANSFORMER_BASED_WRAP":
        cmd_config.append('''--fsdp_transformer_layer_cls_to_wrap=BertLayer'''  )
      elif policy == "SIZE_BASED_WRAP":
        cmd_config.append('''--fsdp_min_num_params=2000'''  )

      cmd_config.extend(
          [
              self.test_file_path,
              F'''--output_dir={self.tmpdir}''',
              F'''--performance_lower_bound={self.performance_lower_bound}''',
          ]  )
      with patch_environment(omp_num_threads=1  ):
        execute_subprocess_async(UpperCamelCase__							,	env=os.environ.copy()  )


  def   UpperCamelCase__      (				self    :					Optional[Any]  )     ->						Any:
    _UpperCamelCase       			=os.path.join(self.test_scripts_folder							,	'''test_checkpointing.py'''  )
    _UpperCamelCase       			=[
        '''accelerate''',
        '''launch''',
        '''--num_processes=2''',
        '''--num_machines=1''',
        '''--machine_rank=0''',
        '''--use_fsdp''',
        '''--mixed_precision=fp16''',
        '''--fsdp_transformer_layer_cls_to_wrap=BertLayer''',
    ]

    for i, strategy in enumerate(UpperCamelCase__  ):
      _UpperCamelCase       			=cmd.copy()
      cmd_config.append(F'''--fsdp_sharding_strategy={i+1}'''  )
      if strategy != "FULL_SHARD":
        continue
      _UpperCamelCase       			=len(UpperCamelCase__  )
      for state_dict_type in FSDP_STATE_DICT_TYPE:
        _UpperCamelCase       			=cmd_config[:state_dict_config_index]
        cmd_config.append(F'''--fsdp_state_dict_type={state_dict_type}'''  )
        cmd_config.extend(
            [
                self.test_file_path,
                F'''--output_dir={self.tmpdir}''',
                '''--partial_train_epoch=1''',
            ]  )
        with patch_environment(omp_num_threads=1  ):
          execute_subprocess_async(UpperCamelCase__							,	env=os.environ.copy()  )

        _UpperCamelCase       			=cmd_config[:-1]
        _UpperCamelCase       			=os.path.join(self.tmpdir							,	'''epoch_0'''  )
        cmd_config.extend(
            [
                F'''--resume_from_checkpoint={resume_from_checkpoint}''',
            ]  )
        with patch_environment(omp_num_threads=1  ):
          execute_subprocess_async(UpperCamelCase__							,	env=os.environ.copy()  )




  def   UpperCamelCase__      (				self    :					Dict  )     ->						int:
    _UpperCamelCase       			=os.path.join(self.test_scripts_folder							,	'''test_peak_memory_usage.py'''  )
    _UpperCamelCase       			=[
        '''accelerate''',
        '''launch''',
        '''--num_processes=2''',
        '''--num_machines=1''',
        '''--machine_rank=0''',
    ]
    for spec, peak_mem_upper_bound in self.peak_memory_usage_upper_bound.items():
      _UpperCamelCase       			=cmd.copy()
      if "fp16" in spec:
        cmd_config.extend(['''--mixed_precision=fp16''']  )
      else:
        cmd_config.extend(['''--mixed_precision=no''']  )

      if "multi_gpu" in spec:
        continue
      else:
        cmd_config.extend(['''--use_fsdp''']  )
        for i, strategy in enumerate(UpperCamelCase__  ):
          if strategy.lower() in spec:
            cmd_config.append(F'''--fsdp_sharding_strategy={i+1}'''  )
            break

        if "cpu_offload" in spec:
          cmd_config.append('''--fsdp_offload_params=True'''  )

        for policy in FSDP_AUTO_WRAP_POLICY:
          if policy.lower() in spec:
            cmd_config.append(F'''--fsdp_auto_wrap_policy={policy}'''  )
            break

        if policy == "TRANSFORMER_BASED_WRAP":
          cmd_config.append('''--fsdp_transformer_layer_cls_to_wrap=BertLayer'''  )
        elif policy == "SIZE_BASED_WRAP":
          cmd_config.append('''--fsdp_min_num_params=2000'''  )

      cmd_config.extend(
          [
              self.test_file_path,
              F'''--output_dir={self.tmpdir}''',
              F'''--peak_memory_upper_bound={peak_mem_upper_bound}''',
              F'''--n_train={self.n_train}''',
              F'''--n_val={self.n_val}''',
          ]  )
      with patch_environment(omp_num_threads=1  ):
        execute_subprocess_async(UpperCamelCase__							,	env=os.environ.copy()  )






