

import flax.linen as nn
import jax
import jax.numpy as jnp





class 					__lowerCAmelCase	(   nn.Module				):
  _a		     =    42
  _a		     =    jnp.floataa




  def     A__	(							self   )    ->					int:
        '''simple docstring'''






        _lowercase      			=nn.Conv(
            self.out_channels				,  kernel_size=(3, 3)				,  strides=(1, 1)				,  padding=((1, 1), (1, 1))				,  dtype=self.dtype				,  )




  def __call__(							self				,  lowerCAmelCase   )    ->					Any:
        '''simple docstring'''






        _lowercase			,			_lowercase			,			_lowercase			,			_lowercase      			=hidden_states.shape
        _lowercase      			=jax.image.resize(
            lowerCAmelCase				,  shape=(batch, height * 2, width * 2, channels)				,  method='nearest'				,  )
        _lowercase      			=self.conv(lowerCAmelCase   )
        return hidden_states





class 					__lowerCAmelCase	(   nn.Module				):
  _a		     =    42
  _a		     =    jnp.floataa




  def     A__	(							self   )    ->					Any:
        '''simple docstring'''






        _lowercase      			=nn.Conv(
            self.out_channels				,  kernel_size=(3, 3)				,  strides=(2, 2)				,  padding=((1, 1), (1, 1))				,  dtype=self.dtype				,  )




  def __call__(							self				,  lowerCAmelCase   )    ->					Optional[int]:
        '''simple docstring'''






        _lowercase      			=self.conv(lowerCAmelCase   )
        return hidden_states





class 					__lowerCAmelCase	(   nn.Module				):
  _a		     =    42
  _a		     =    None
  _a		     =    0.0
  _a		     =    None
  _a		     =    jnp.floataa




  def     A__	(							self   )    ->					Optional[int]:
        '''simple docstring'''






        _lowercase      			=self.in_channels if self.out_channels is None else self.out_channels

        _lowercase      			=nn.GroupNorm(num_groups=32				,  epsilon=1e-5   )
        _lowercase      			=nn.Conv(
            lowerCAmelCase				,  kernel_size=(3, 3)				,  strides=(1, 1)				,  padding=((1, 1), (1, 1))				,  dtype=self.dtype				,  )

        _lowercase      			=nn.Dense(lowerCAmelCase				,  dtype=self.dtype   )

        _lowercase      			=nn.GroupNorm(num_groups=32				,  epsilon=1e-5   )
        _lowercase      			=nn.Dropout(self.dropout_prob   )
        _lowercase      			=nn.Conv(
            lowerCAmelCase				,  kernel_size=(3, 3)				,  strides=(1, 1)				,  padding=((1, 1), (1, 1))				,  dtype=self.dtype				,  )

        _lowercase      			=self.in_channels != out_channels if self.use_nin_shortcut is None else self.use_nin_shortcut

        _lowercase      			=None
        if use_nin_shortcut:
              _lowercase      			=nn.Conv(
                  lowerCAmelCase				,  kernel_size=(1, 1)				,  strides=(1, 1)				,  padding='VALID'				,  dtype=self.dtype				,  )




  def __call__(							self				,  lowerCAmelCase				,  lowerCAmelCase				,  lowerCAmelCase=True   )    ->					Tuple:
        '''simple docstring'''






        _lowercase      			=hidden_states
        _lowercase      			=self.norma(lowerCAmelCase   )
        _lowercase      			=nn.swish(lowerCAmelCase   )
        _lowercase      			=self.conva(lowerCAmelCase   )

        _lowercase      			=self.time_emb_proj(nn.swish(lowerCAmelCase   )   )
        _lowercase      			=jnp.expand_dims(jnp.expand_dims(lowerCAmelCase				,  1   )				,  1   )
        _lowercase      			=hidden_states + temb

        _lowercase      			=self.norma(lowerCAmelCase   )
        _lowercase      			=nn.swish(lowerCAmelCase   )
        _lowercase      			=self.dropout(lowerCAmelCase				,  lowerCAmelCase   )
        _lowercase      			=self.conva(lowerCAmelCase   )

        if self.conv_shortcut is not None:
              _lowercase      			=self.conv_shortcut(lowerCAmelCase   )

        return hidden_states + residual





# We ignore warnings about stepping the scheduler since we step it ourselves during gradient accumulation

import warnings

from .state import AcceleratorState, GradientState


warnings.filterwarnings('ignore', category=UserWarning, module='torch.optim.lr_scheduler')





class 					__lowerCAmelCase	:




  def __init__(							self				,  lowerCAmelCase				,  lowerCAmelCase				,  lowerCAmelCase = True				,  lowerCAmelCase = False   )    ->					str:
        '''simple docstring'''






        _lowercase      			=scheduler
        _lowercase      			=optimizers if isinstance(lowerCAmelCase				,  (list, tuple)   ) else [optimizers]
        _lowercase      			=split_batches
        _lowercase      			=step_with_optimizer
        _lowercase      			=GradientState()




  def     A__	(							self				,  *lowerCAmelCase				,  **lowerCAmelCase   )    ->					List[Any]:
        '''simple docstring'''






        if not self.step_with_optimizer:
              # No link between scheduler and optimizer -> just step
              self.scheduler.step(*lowerCAmelCase				,  **lowerCAmelCase   )
              return

        # Otherwise, first make sure the optimizer was stepped.
        if not self.gradient_state.sync_gradients:
              if self.gradient_state.adjust_scheduler:
                    self.scheduler._step_count += 1
              return

        for opt in self.optimizers:
              if opt.step_was_skipped:
                    return
        if self.split_batches:
              # Split batches -> the training dataloader batch size is not changed so one step per training step
              self.scheduler.step(*lowerCAmelCase				,  **lowerCAmelCase   )
        else:
              # Otherwise the training dataloader batch size was multiplied by `num_processes`, so we need to do
              # num_processes steps per training step
              _lowercase      			=AcceleratorState().num_processes
              for _ in range(lowerCAmelCase   ):
                    # Special case when using OneCycle and `drop_last` was not used
                    if hasattr(self.scheduler				,  'total_steps'   ):
                          if self.scheduler._step_count <= self.scheduler.total_steps:
                                self.scheduler.step(*lowerCAmelCase				,  **lowerCAmelCase   )
                    else:
                          self.scheduler.step(*lowerCAmelCase				,  **lowerCAmelCase   )




  def     A__	(							self   )    ->					str:
        '''simple docstring'''






        return self.scheduler.get_last_lr()




  def     A__	(							self   )    ->					Tuple:
        '''simple docstring'''






        return self.scheduler.state_dict()




  def     A__	(							self				,  lowerCAmelCase   )    ->					str:
        '''simple docstring'''






        self.scheduler.load_state_dict(lowerCAmelCase   )




  def     A__	(							self   )    ->					int:
        '''simple docstring'''






        return self.scheduler.get_lr()




  def     A__	(							self				,  *lowerCAmelCase				,  **lowerCAmelCase   )    ->					Optional[int]:
        '''simple docstring'''






        return self.scheduler.print_lr(*lowerCAmelCase				,  **lowerCAmelCase   )


