


'''simple docstring'''




from typing import Optional, Tuple, Union

import flax
import flax.linen as nn
import jax
import jax.numpy as jnp
from flax.core.frozen_dict import FrozenDict

from ..configuration_utils import ConfigMixin, flax_register_to_config
from ..utils import BaseOutput
from .embeddings_flax import FlaxTimestepEmbedding, FlaxTimesteps
from .modeling_flax_utils import FlaxModelMixin
from .unet_ad_blocks_flax import (
    FlaxCrossAttnDownBlockaD,
    FlaxDownBlockaD,
    FlaxUNetMidBlockaDCrossAttn,
)







@flax.struct.dataclass
class 					a			(				snake_case__  ):
 '''simple docstring'''




 __lowerCAmelCase						: jnp.ndarray
 __lowerCAmelCase						: jnp.ndarray







class 					a			(				nn.Module  ):
 '''simple docstring'''




 __lowerCAmelCase						: int
 __lowerCAmelCase						: Tuple[int]							  =		(16, 32, 96, 256)
 __lowerCAmelCase						: jnp.dtype							  =		jnp.floataa




 def   __UpperCamelCase   (    self      )      ->    Tuple:
    _a					:  str						=  nn.Conv(
        self.block_out_channels[0]		,      kernel_size=(3, 3)		,      padding=((1, 1), (1, 1))		,      dtype=self.dtype		,      )

    _a					:  Any						=  []
    for i in range(len(self.block_out_channels      ) - 1      ):
       _a					:  Dict						=  self.block_out_channels[i]
       _a					:  Any						=  self.block_out_channels[i + 1]
       _a					:  str						=  nn.Conv(
           lowerCamelCase_		,      kernel_size=(3, 3)		,      padding=((1, 1), (1, 1))		,      dtype=self.dtype		,      )
       blocks.append(lowerCamelCase_      )
       _a					:  Optional[Any]						=  nn.Conv(
           lowerCamelCase_		,      kernel_size=(3, 3)		,      strides=(2, 2)		,      padding=((1, 1), (1, 1))		,      dtype=self.dtype		,      )
       blocks.append(lowerCamelCase_      )
    _a					:  Tuple						=  blocks

    _a					:  Optional[Any]						=  nn.Conv(
        self.conditioning_embedding_channels		,      kernel_size=(3, 3)		,      padding=((1, 1), (1, 1))		,      kernel_init=nn.initializers.zeros_init()		,      bias_init=nn.initializers.zeros_init()		,      dtype=self.dtype		,      )




 def __call__(    self		,      lowerCamelCase_      )      ->    int:
    _a					:  List[Any]						=  self.conv_in(lowerCamelCase_      )
    _a					:  Union[str, Any]						=  nn.silu(lowerCamelCase_      )

    for block in self.blocks:
       _a					:  int						=  block(lowerCamelCase_      )
       _a					:  List[Any]						=  nn.silu(lowerCamelCase_      )

    _a					:  List[Any]						=  self.conv_out(lowerCamelCase_      )

    return embedding







@flax_register_to_config
class 					a			(				nn.Module			,							snake_case__			,							snake_case__  ):
 '''simple docstring'''




 __lowerCAmelCase						: int							  =		32
 __lowerCAmelCase						: int							  =		4
 __lowerCAmelCase						: Tuple[str]							  =		(
     "CrossAttnDownBlock2D",
     "CrossAttnDownBlock2D",
     "CrossAttnDownBlock2D",
     "DownBlock2D",
 )
 __lowerCAmelCase						: Union[bool, Tuple[bool]]							  =		False
 __lowerCAmelCase						: Tuple[int]							  =		(320, 640, 1280, 1280)
 __lowerCAmelCase						: int							  =		2
 __lowerCAmelCase						: Union[int, Tuple[int]]							  =		8
 __lowerCAmelCase						: Optional[Union[int, Tuple[int]]]							  =		None
 __lowerCAmelCase						: int							  =		1280
 __lowerCAmelCase						: float							  =		0.0
 __lowerCAmelCase						: bool							  =		False
 __lowerCAmelCase						: jnp.dtype							  =		jnp.floataa
 __lowerCAmelCase						: bool							  =		True
 __lowerCAmelCase						: int							  =		0
 __lowerCAmelCase						: str							  =		"rgb"
 __lowerCAmelCase						: Tuple[int]							  =		(16, 32, 96, 256)




 def   __UpperCamelCase   (    self		,      lowerCamelCase_      )      ->    FrozenDict:
    # init input tensors
    _a					:  int						=  (1, self.in_channels, self.sample_size, self.sample_size)
    _a					:  Union[str, Any]						=  jnp.zeros(lowerCamelCase_		,      dtype=jnp.floataa      )
    _a					:  Optional[Any]						=  jnp.ones((1,)		,      dtype=jnp.intaa      )
    _a					:  Optional[Any]						=  jnp.zeros((1, 1, self.cross_attention_dim)		,      dtype=jnp.floataa      )
    _a					:  Dict						=  (1, 3, self.sample_size * 8, self.sample_size * 8)
    _a					:  int						=  jnp.zeros(lowerCamelCase_		,      dtype=jnp.floataa      )

    _a		,						_a					:  int						=  jax.random.split(lowerCamelCase_      )
    _a					:  Optional[int]						=  {'params': params_rng, 'dropout': dropout_rng}

    return self.init(lowerCamelCase_		,      lowerCamelCase_		,      lowerCamelCase_		,      lowerCamelCase_		,      lowerCamelCase_      )["params"]




 def   __UpperCamelCase   (    self      )      ->    List[str]:
    _a					:  Union[str, Any]						=  self.block_out_channels
    _a					:  Any						=  block_out_channels[0] * 4

    # If `num_attention_heads` is not defined (which is the case for most models)
    # it will default to `attention_head_dim`. This looks weird upon first reading it and it is.
    # The reason for this behavior is to correct for incorrectly named variables that were introduced
    # when this library was created. The incorrect naming was only discovered much later in https://github.com/huggingface/diffusers/issues/2011#issuecomment-1547958131
    # Changing `attention_head_dim` to `num_attention_heads` for 40,000+ configurations is too backwards breaking
    # which is why we correct for the naming here.
    _a					:  List[Any]						=  self.num_attention_heads or self.attention_head_dim

    # input
    _a					:  Union[str, Any]						=  nn.Conv(
        block_out_channels[0]		,      kernel_size=(3, 3)		,      strides=(1, 1)		,      padding=((1, 1), (1, 1))		,      dtype=self.dtype		,      )

    # time
    _a					:  int						=  FlaxTimesteps(
        block_out_channels[0]		,      flip_sin_to_cos=self.flip_sin_to_cos		,      freq_shift=self.config.freq_shift      )
    _a					:  Optional[Any]						=  FlaxTimestepEmbedding(lowerCamelCase_		,      dtype=self.dtype      )

    _a					:  Optional[Any]						=  FlaxControlNetConditioningEmbedding(
        conditioning_embedding_channels=block_out_channels[0]		,      block_out_channels=self.conditioning_embedding_out_channels		,      )

    _a					:  Union[str, Any]						=  self.only_cross_attention
    if isinstance(lowerCamelCase_		,      lowerCamelCase_      ):
       _a					:  int						=  (only_cross_attention,) * len(self.down_block_types      )

    if isinstance(lowerCamelCase_		,      lowerCamelCase_      ):
       _a					:  List[str]						=  (num_attention_heads,) * len(self.down_block_types      )

    # down
    _a					:  Optional[int]						=  []
    _a					:  Tuple						=  []

    _a					:  Optional[int]						=  block_out_channels[0]

    _a					:  str						=  nn.Conv(
        lowerCamelCase_		,      kernel_size=(1, 1)		,      padding='VALID'		,      kernel_init=nn.initializers.zeros_init()		,      bias_init=nn.initializers.zeros_init()		,      dtype=self.dtype		,      )
    controlnet_down_blocks.append(lowerCamelCase_      )

    for i, down_block_type in enumerate(self.down_block_types      ):
       _a					:  Optional[int]						=  output_channel
       _a					:  Union[str, Any]						=  block_out_channels[i]
       _a					:  str						=  i == len(lowerCamelCase_      ) - 1

       if down_block_type == "CrossAttnDownBlock2D":
          _a					:  Optional[int]						=  FlaxCrossAttnDownBlockaD(
              in_channels=lowerCamelCase_		,      out_channels=lowerCamelCase_		,      dropout=self.dropout		,      num_layers=self.layers_per_block		,      num_attention_heads=num_attention_heads[i]		,      add_downsample=not is_final_block		,      use_linear_projection=self.use_linear_projection		,      only_cross_attention=only_cross_attention[i]		,      dtype=self.dtype		,      )
       else:
          _a					:  List[str]						=  FlaxDownBlockaD(
              in_channels=lowerCamelCase_		,      out_channels=lowerCamelCase_		,      dropout=self.dropout		,      num_layers=self.layers_per_block		,      add_downsample=not is_final_block		,      dtype=self.dtype		,      )

       down_blocks.append(lowerCamelCase_      )

       for _ in range(self.layers_per_block      ):
          _a					:  Dict						=  nn.Conv(
              lowerCamelCase_		,      kernel_size=(1, 1)		,      padding='VALID'		,      kernel_init=nn.initializers.zeros_init()		,      bias_init=nn.initializers.zeros_init()		,      dtype=self.dtype		,      )
          controlnet_down_blocks.append(lowerCamelCase_      )

       if not is_final_block:
          _a					:  Union[str, Any]						=  nn.Conv(
              lowerCamelCase_		,      kernel_size=(1, 1)		,      padding='VALID'		,      kernel_init=nn.initializers.zeros_init()		,      bias_init=nn.initializers.zeros_init()		,      dtype=self.dtype		,      )
          controlnet_down_blocks.append(lowerCamelCase_      )

    _a					:  Tuple						=  down_blocks
    _a					:  Optional[Any]						=  controlnet_down_blocks

    # mid
    _a					:  List[Any]						=  block_out_channels[-1]
    _a					:  str						=  FlaxUNetMidBlockaDCrossAttn(
        in_channels=lowerCamelCase_		,      dropout=self.dropout		,      num_attention_heads=num_attention_heads[-1]		,      use_linear_projection=self.use_linear_projection		,      dtype=self.dtype		,      )

    _a					:  str						=  nn.Conv(
        lowerCamelCase_		,      kernel_size=(1, 1)		,      padding='VALID'		,      kernel_init=nn.initializers.zeros_init()		,      bias_init=nn.initializers.zeros_init()		,      dtype=self.dtype		,      )




 def __call__(    self		,      lowerCamelCase_		,      lowerCamelCase_		,      lowerCamelCase_		,      lowerCamelCase_		,      lowerCamelCase_ = 1.0		,      lowerCamelCase_ = True		,      lowerCamelCase_ = False		,      )      ->    Union[FlaxControlNetOutput, Tuple]:
    _a					:  Any						=  self.controlnet_conditioning_channel_order
    if channel_order == "bgr":
       _a					:  List[Any]						=  jnp.flip(lowerCamelCase_		,      axis=1      )

    # 1. time
    if not isinstance(lowerCamelCase_		,      jnp.ndarray      ):
       _a					:  Any						=  jnp.array([timesteps]		,      dtype=jnp.intaa      )
    elif isinstance(lowerCamelCase_		,      jnp.ndarray      ) and len(timesteps.shape      ) == 0:
       _a					:  Tuple						=  timesteps.astype(dtype=jnp.floataa      )
       _a					:  Optional[Any]						=  jnp.expand_dims(lowerCamelCase_		,      0      )

    _a					:  int						=  self.time_proj(lowerCamelCase_      )
    _a					:  Optional[Any]						=  self.time_embedding(lowerCamelCase_      )

    # 2. pre-process
    _a					:  Dict						=  jnp.transpose(lowerCamelCase_		,      (0, 2, 3, 1)      )
    _a					:  str						=  self.conv_in(lowerCamelCase_      )

    _a					:  List[str]						=  jnp.transpose(lowerCamelCase_		,      (0, 2, 3, 1)      )
    _a					:  Union[str, Any]						=  self.controlnet_cond_embedding(lowerCamelCase_      )
    sample += controlnet_cond

    # 3. down
    _a					:  List[Any]						=  (sample,)
    for down_block in self.down_blocks:
       if isinstance(lowerCamelCase_		,      lowerCamelCase_      ):
          _a		,						_a					:  Union[str, Any]						=  down_block(lowerCamelCase_		,      lowerCamelCase_		,      lowerCamelCase_		,      deterministic=not train      )
       else:
          _a		,						_a					:  Union[str, Any]						=  down_block(lowerCamelCase_		,      lowerCamelCase_		,      deterministic=not train      )
       down_block_res_samples += res_samples

    # 4. mid
    _a					:  List[str]						=  self.mid_block(lowerCamelCase_		,      lowerCamelCase_		,      lowerCamelCase_		,      deterministic=not train      )

    # 5. contronet blocks
    _a					:  List[str]						=  ()
    for down_block_res_sample, controlnet_block in zip(lowerCamelCase_		,      self.controlnet_down_blocks      ):
       _a					:  List[str]						=  controlnet_block(lowerCamelCase_      )
       controlnet_down_block_res_samples += (down_block_res_sample,)

    _a					:  Optional[int]						=  controlnet_down_block_res_samples

    _a					:  Optional[int]						=  self.controlnet_mid_block(lowerCamelCase_      )

    # 6. scaling
    _a					:  Dict						=  [sample * conditioning_scale for sample in down_block_res_samples]
    mid_block_res_sample *= conditioning_scale

    if not return_dict:
       return (down_block_res_samples, mid_block_res_sample)

    return FlaxControlNetOutput(
        down_block_res_samples=lowerCamelCase_		,      mid_block_res_sample=lowerCamelCase_      )






'''simple docstring'''




import inspect
import os
import unittest
from dataclasses import dataclass

import torch

from accelerate import Accelerator, DistributedDataParallelKwargs, GradScalerKwargs
from accelerate.state import AcceleratorState
from accelerate.test_utils import execute_subprocess_async, require_cuda, require_multi_gpu
from accelerate.utils import KwargsHandler







@dataclass
class 					a			(				snake_case__  ):
 '''simple docstring'''




 __lowerCAmelCase						: int							  =		0
 __lowerCAmelCase						: bool							  =		False
 __lowerCAmelCase						: float							  =		3.0







class 					a			(				unittest.TestCase  ):
 '''simple docstring'''




 def   __UpperCamelCase   (    self      )      ->    Dict:
    # If no defaults are changed, `to_kwargs` returns an empty dict.
    self.assertDictEqual(MockClass().to_kwargs()		,      {}      )
    self.assertDictEqual(MockClass(a=2      ).to_kwargs()		,      {'a': 2}      )
    self.assertDictEqual(MockClass(a=2		,      b=lowerCamelCase_      ).to_kwargs()		,      {'a': 2, 'b': True}      )
    self.assertDictEqual(MockClass(a=2		,      c=2.25      ).to_kwargs()		,      {'a': 2, 'c': 2.25}      )




 @require_cuda
 def   __UpperCamelCase   (    self      )      ->    int:
    # If no defaults are changed, `to_kwargs` returns an empty dict.
    _a					:  List[str]						=  GradScalerKwargs(init_scale=1_0_2_4		,      growth_factor=2      )
    AcceleratorState._reset_state()
    _a					:  Optional[int]						=  Accelerator(mixed_precision='fp16'		,      kwargs_handlers=[scaler_handler]      )
    print(accelerator.use_fpaa      )
    _a					:  Tuple						=  accelerator.scaler

    # Check the kwargs have been applied
    self.assertEqual(scaler._init_scale		,      1024.0      )
    self.assertEqual(scaler._growth_factor		,      2.0      )

    # Check the other values are at the default
    self.assertEqual(scaler._backoff_factor		,      0.5      )
    self.assertEqual(scaler._growth_interval		,      2_0_0_0      )
    self.assertEqual(scaler._enabled		,      lowerCamelCase_      )




 @require_multi_gpu
 def   __UpperCamelCase   (    self      )      ->    Optional[Any]:
    _a					:  List[str]						=  ['torchrun', F'''--nproc_per_node={torch.cuda.device_count()}''', inspect.getfile(self.__class__      )]
    execute_subprocess_async(lowerCamelCase_		,      env=os.environ.copy()      )


if __name__ == "__main__":
 UpperCAmelCase_      :		Tuple          =		DistributedDataParallelKwargs(bucket_cap_mb=15, find_unused_parameters=True)
 UpperCAmelCase_      :		str          =		Accelerator(kwargs_handlers=[ddp_scaler])
 UpperCAmelCase_      :		List[Any]          =		torch.nn.Linear(100, 200)
 UpperCAmelCase_      :		Tuple          =		accelerator.prepare(model)

 # Check the values changed in kwargs
 UpperCAmelCase_      :		List[str]          =		""
 UpperCAmelCase_      :		Any          =		model.bucket_bytes_cap // (1024 * 1024)
 if observed_bucket_cap_map != 15:
  error_msg += f"Kwargs badly passed, should have `15` but found {observed_bucket_cap_map}.\n"
 if model.find_unused_parameters is not True:
  error_msg += f"Kwargs badly passed, should have `True` but found {model.find_unused_parameters}.\n"

 # Check the values of the defaults
 if model.dim != 0:
  error_msg += f"Default value not respected, should have `0` but found {model.dim}.\n"
 if model.broadcast_buffers is not True:
  error_msg += f"Default value not respected, should have `True` but found {model.broadcast_buffers}.\n"
 if model.gradient_as_bucket_view is not False:
  error_msg += f"Default value not respected, should have `False` but found {model.gradient_as_bucket_view}.\n"

 # Raise error at the end to make sure we don't stop at the first failure.
 if len(error_msg) > 0:
  raise ValueError(error_msg)


