


from typing import Optional, Tuple, Union

import flax
import flax.linen as nn
import jax
import jax.numpy as jnp
from flax.core.frozen_dict import FrozenDict

from ..configuration_utils import ConfigMixin, flax_register_to_config
from ..utils import BaseOutput
from .embeddings_flax import FlaxTimestepEmbedding, FlaxTimesteps
from .modeling_flax_utils import FlaxModelMixin
from .unet_ad_blocks_flax import (
    FlaxCrossAttnDownBlockaD,
    FlaxCrossAttnUpBlockaD,
    FlaxDownBlockaD,
    FlaxUNetMidBlockaDCrossAttn,
    FlaxUpBlockaD,
)






@flax.struct.dataclass
class 	__lowercase					(	__snake_case	):
      UpperCamelCase										=							42






@flax_register_to_config
class 	__lowercase					(	nn.Module			,       __snake_case			,       __snake_case	):
      UpperCamelCase										=							32
      UpperCamelCase										=							4
      UpperCamelCase										=							4
      UpperCamelCase										=							(
          "CrossAttnDownBlock2D",
          "CrossAttnDownBlock2D",
          "CrossAttnDownBlock2D",
          "DownBlock2D",
      )
      UpperCamelCase										=							("UpBlock2D", "CrossAttnUpBlock2D", "CrossAttnUpBlock2D", "CrossAttnUpBlock2D")
      UpperCamelCase										=							False
      UpperCamelCase										=							(3_20, 6_40, 12_80, 12_80)
      UpperCamelCase										=							2
      UpperCamelCase										=							8
      UpperCamelCase										=							None
      UpperCamelCase										=							12_80
      UpperCamelCase										=							0.0
      UpperCamelCase										=							False
      UpperCamelCase										=							jnp.floataa
      UpperCamelCase										=							True
      UpperCamelCase										=							0
      UpperCamelCase										=							False





      def 							_lowercase					(		self    :       Dict       ,							__lowerCamelCase    :       jax.random.KeyArray							)     -> FrozenDict:






         """simple docstring"""







         UpperCAmelCase      					=  (1, self.in_channels, self.sample_size, self.sample_size)
         UpperCAmelCase      					=  jnp.zeros(__lowerCamelCase       ,							dtype=jnp.floataa							)
         UpperCAmelCase      					=  jnp.ones((1,)       ,							dtype=jnp.intaa							)
         UpperCAmelCase      					=  jnp.zeros((1, 1, self.cross_attention_dim)       ,							dtype=jnp.floataa							)

         UpperCAmelCase					,	UpperCAmelCase      					=  jax.random.split(__lowerCamelCase							)
         UpperCAmelCase      					=  {"""params""": params_rng, """dropout""": dropout_rng}

         return self.init(__lowerCamelCase       ,							__lowerCamelCase       ,							__lowerCamelCase       ,							__lowerCamelCase							)["params"]





      def 							_lowercase					(		self    :       str							)     -> Tuple:






         """simple docstring"""







         UpperCAmelCase      					=  self.block_out_channels
         UpperCAmelCase      					=  block_out_channels[0] * 4

         if self.num_attention_heads is not None:
            raise ValueError(
                """At the moment it is not possible to define the number of attention heads via `num_attention_heads` because of a naming issue as described in https://github.com/huggingface/diffusers/issues/2011#issuecomment-1547958131. Passing `num_attention_heads` will only be supported in diffusers v0.19."""							)

         # If `num_attention_heads` is not defined (which is the case for most models)
         # it will default to `attention_head_dim`. This looks weird upon first reading it and it is.
         # The reason for this behavior is to correct for incorrectly named variables that were introduced
         # when this library was created. The incorrect naming was only discovered much later in https://github.com/huggingface/diffusers/issues/2011#issuecomment-1547958131
         # Changing `attention_head_dim` to `num_attention_heads` for 40,000+ configurations is too backwards breaking
         # which is why we correct for the naming here.
         UpperCAmelCase      					=  self.num_attention_heads or self.attention_head_dim

         # input
         UpperCAmelCase      					=  nn.Conv(
             block_out_channels[0]       ,							kernel_size=(3, 3)       ,							strides=(1, 1)       ,							padding=((1, 1), (1, 1))       ,							dtype=self.dtype       ,							)

         # time
         UpperCAmelCase      					=  FlaxTimesteps(
             block_out_channels[0]       ,							flip_sin_to_cos=self.flip_sin_to_cos       ,							freq_shift=self.config.freq_shift							)
         UpperCAmelCase      					=  FlaxTimestepEmbedding(__lowerCamelCase       ,							dtype=self.dtype							)

         UpperCAmelCase      					=  self.only_cross_attention
         if isinstance(__lowerCamelCase       ,							__lowerCamelCase							):
            UpperCAmelCase      					=  (only_cross_attention,) * len(self.down_block_types							)

         if isinstance(__lowerCamelCase       ,							__lowerCamelCase							):
            UpperCAmelCase      					=  (num_attention_heads,) * len(self.down_block_types							)

         # down
         UpperCAmelCase      					=  []
         UpperCAmelCase      					=  block_out_channels[0]
         for i, down_block_type in enumerate(self.down_block_types							):
            UpperCAmelCase      					=  output_channel
            UpperCAmelCase      					=  block_out_channels[i]
            UpperCAmelCase      					=  i == len(__lowerCamelCase							) - 1

            if down_block_type == "CrossAttnDownBlock2D":
               UpperCAmelCase      					=  FlaxCrossAttnDownBlockaD(
                   in_channels=__lowerCamelCase       ,							out_channels=__lowerCamelCase       ,							dropout=self.dropout       ,							num_layers=self.layers_per_block       ,							num_attention_heads=num_attention_heads[i]       ,							add_downsample=not is_final_block       ,							use_linear_projection=self.use_linear_projection       ,							only_cross_attention=only_cross_attention[i]       ,							use_memory_efficient_attention=self.use_memory_efficient_attention       ,							dtype=self.dtype       ,							)
            else:
               UpperCAmelCase      					=  FlaxDownBlockaD(
                   in_channels=__lowerCamelCase       ,							out_channels=__lowerCamelCase       ,							dropout=self.dropout       ,							num_layers=self.layers_per_block       ,							add_downsample=not is_final_block       ,							dtype=self.dtype       ,							)

            down_blocks.append(__lowerCamelCase							)
         UpperCAmelCase      					=  down_blocks

         # mid
         UpperCAmelCase      					=  FlaxUNetMidBlockaDCrossAttn(
             in_channels=block_out_channels[-1]       ,							dropout=self.dropout       ,							num_attention_heads=num_attention_heads[-1]       ,							use_linear_projection=self.use_linear_projection       ,							use_memory_efficient_attention=self.use_memory_efficient_attention       ,							dtype=self.dtype       ,							)

         # up
         UpperCAmelCase      					=  []
         UpperCAmelCase      					=  list(reversed(__lowerCamelCase							)							)
         UpperCAmelCase      					=  list(reversed(__lowerCamelCase							)							)
         UpperCAmelCase      					=  list(reversed(__lowerCamelCase							)							)
         UpperCAmelCase      					=  reversed_block_out_channels[0]
         for i, up_block_type in enumerate(self.up_block_types							):
            UpperCAmelCase      					=  output_channel
            UpperCAmelCase      					=  reversed_block_out_channels[i]
            UpperCAmelCase      					=  reversed_block_out_channels[min(i + 1       ,							len(__lowerCamelCase							) - 1							)]

            UpperCAmelCase      					=  i == len(__lowerCamelCase							) - 1

            if up_block_type == "CrossAttnUpBlock2D":
               UpperCAmelCase      					=  FlaxCrossAttnUpBlockaD(
                   in_channels=__lowerCamelCase       ,							out_channels=__lowerCamelCase       ,							prev_output_channel=__lowerCamelCase       ,							num_layers=self.layers_per_block + 1       ,							num_attention_heads=reversed_num_attention_heads[i]       ,							add_upsample=not is_final_block       ,							dropout=self.dropout       ,							use_linear_projection=self.use_linear_projection       ,							only_cross_attention=only_cross_attention[i]       ,							use_memory_efficient_attention=self.use_memory_efficient_attention       ,							dtype=self.dtype       ,							)
            else:
               UpperCAmelCase      					=  FlaxUpBlockaD(
                   in_channels=__lowerCamelCase       ,							out_channels=__lowerCamelCase       ,							prev_output_channel=__lowerCamelCase       ,							num_layers=self.layers_per_block + 1       ,							add_upsample=not is_final_block       ,							dropout=self.dropout       ,							dtype=self.dtype       ,							)

            up_blocks.append(__lowerCamelCase							)
            UpperCAmelCase      					=  output_channel
         UpperCAmelCase      					=  up_blocks

         # out
         UpperCAmelCase      					=  nn.GroupNorm(num_groups=3_2       ,							epsilon=1e-5							)
         UpperCAmelCase      					=  nn.Conv(
             self.out_channels       ,							kernel_size=(3, 3)       ,							strides=(1, 1)       ,							padding=((1, 1), (1, 1))       ,							dtype=self.dtype       ,							)





      def __call__(		self    :       Optional[Any]       ,							__lowerCamelCase    :       Optional[Any]       ,							__lowerCamelCase    :       Dict       ,							__lowerCamelCase    :       Optional[int]       ,							__lowerCamelCase    :       Optional[Any]=None       ,							__lowerCamelCase    :       Optional[int]=None       ,							__lowerCamelCase    :       bool = True       ,							__lowerCamelCase    :       bool = False       ,							)     -> Union[FlaxUNetaDConditionOutput, Tuple]:






         """simple docstring"""







         if not isinstance(__lowerCamelCase       ,							jnp.ndarray							):
            UpperCAmelCase      					=  jnp.array([timesteps]       ,							dtype=jnp.intaa							)
         elif isinstance(__lowerCamelCase       ,							jnp.ndarray							) and len(timesteps.shape							) == 0:
            UpperCAmelCase      					=  timesteps.astype(dtype=jnp.floataa							)
            UpperCAmelCase      					=  jnp.expand_dims(__lowerCamelCase       ,							0							)

         UpperCAmelCase      					=  self.time_proj(__lowerCamelCase							)
         UpperCAmelCase      					=  self.time_embedding(__lowerCamelCase							)

         # 2. pre-process
         UpperCAmelCase      					=  jnp.transpose(__lowerCamelCase       ,							(0, 2, 3, 1)							)
         UpperCAmelCase      					=  self.conv_in(__lowerCamelCase							)

         # 3. down
         UpperCAmelCase      					=  (sample,)
         for down_block in self.down_blocks:
            if isinstance(__lowerCamelCase       ,							__lowerCamelCase							):
               UpperCAmelCase					,	UpperCAmelCase      					=  down_block(__lowerCamelCase       ,							__lowerCamelCase       ,							__lowerCamelCase       ,							deterministic=not train							)
            else:
               UpperCAmelCase					,	UpperCAmelCase      					=  down_block(__lowerCamelCase       ,							__lowerCamelCase       ,							deterministic=not train							)
            down_block_res_samples += res_samples

         if down_block_additional_residuals is not None:
            UpperCAmelCase      					=  ()

            for down_block_res_sample, down_block_additional_residual in zip(
                __lowerCamelCase       ,							__lowerCamelCase							):
               down_block_res_sample += down_block_additional_residual
               new_down_block_res_samples += (down_block_res_sample,)

            UpperCAmelCase      					=  new_down_block_res_samples

         # 4. mid
         UpperCAmelCase      					=  self.mid_block(__lowerCamelCase       ,							__lowerCamelCase       ,							__lowerCamelCase       ,							deterministic=not train							)

         if mid_block_additional_residual is not None:
            sample += mid_block_additional_residual

         # 5. up
         for up_block in self.up_blocks:
            UpperCAmelCase      					=  down_block_res_samples[-(self.layers_per_block + 1) :]
            UpperCAmelCase      					=  down_block_res_samples[: -(self.layers_per_block + 1)]
            if isinstance(__lowerCamelCase       ,							__lowerCamelCase							):
               UpperCAmelCase      					=  up_block(
                   __lowerCamelCase       ,							temb=__lowerCamelCase       ,							encoder_hidden_states=__lowerCamelCase       ,							res_hidden_states_tuple=__lowerCamelCase       ,							deterministic=not train       ,							)
            else:
               UpperCAmelCase      					=  up_block(__lowerCamelCase       ,							temb=__lowerCamelCase       ,							res_hidden_states_tuple=__lowerCamelCase       ,							deterministic=not train							)

        # 6. post-process
         UpperCAmelCase      					=  self.conv_norm_out(__lowerCamelCase							)
         UpperCAmelCase      					=  nn.silu(__lowerCamelCase							)
         UpperCAmelCase      					=  self.conv_out(__lowerCamelCase							)
         UpperCAmelCase      					=  jnp.transpose(__lowerCamelCase       ,							(0, 3, 1, 2)							)

         if not return_dict:
            return (sample,)

         return FlaxUNetaDConditionOutput(sample=__lowerCamelCase							)







import math
import time

from transformers import Trainer, is_torch_tpu_available
from transformers.trainer_utils import PredictionOutput, speed_metrics


if is_torch_tpu_available(check_device=False):
 import torch_xla.core.xla_model as xm
 import torch_xla.debug.metrics as met






class 	__lowercase					(	__snake_case	):





      def __init__(		self    :       Dict       ,							*__lowerCamelCase    :       Dict       ,							__lowerCamelCase    :       Union[str, Any]=None       ,							__lowerCamelCase    :       str=None       ,							**__lowerCamelCase    :       Optional[int]							)     -> Optional[int]:






         """simple docstring"""







         super().__init__(*__lowerCamelCase       ,							**__lowerCamelCase							)
         UpperCAmelCase      					=  eval_examples
         UpperCAmelCase      					=  post_process_function





      def 							_lowercase					(		self    :       Any       ,							__lowerCamelCase    :       int=None       ,							__lowerCamelCase    :       int=None       ,							__lowerCamelCase    :       Tuple=None       ,							__lowerCamelCase    :       str = "eval"							)     -> List[str]:






         """simple docstring"""







         UpperCAmelCase      					=  self.eval_dataset if eval_dataset is None else eval_dataset
         UpperCAmelCase      					=  self.get_eval_dataloader(__lowerCamelCase							)
         UpperCAmelCase      					=  self.eval_examples if eval_examples is None else eval_examples

         # Temporarily disable metric computation, we will do it in the loop here.
         UpperCAmelCase      					=  self.compute_metrics
         UpperCAmelCase      					=  None
         UpperCAmelCase      					=  self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop
         UpperCAmelCase      					=  time.time()
         try:
            UpperCAmelCase      					=  eval_loop(
                __lowerCamelCase       ,							description="""Evaluation"""       ,							prediction_loss_only=True if compute_metrics is None else None       ,							ignore_keys=__lowerCamelCase       ,							metric_key_prefix=__lowerCamelCase       ,							)
         finally:
            UpperCAmelCase      					=  compute_metrics
         UpperCAmelCase      					=  self.args.eval_batch_size * self.args.world_size
         if F"""{metric_key_prefix}_jit_compilation_time""" in output.metrics:
            start_time += output.metrics[F"""{metric_key_prefix}_jit_compilation_time"""]
         output.metrics.update(
             speed_metrics(
                 __lowerCamelCase       ,							__lowerCamelCase       ,							num_samples=output.num_samples       ,							num_steps=math.ceil(output.num_samples / total_batch_size							)       ,							)							)
         if self.post_process_function is not None and self.compute_metrics is not None and self.args.should_save:
            # Only the main node write the results by default
            UpperCAmelCase      					=  self.post_process_function(__lowerCamelCase       ,							__lowerCamelCase       ,							output.predictions							)
            UpperCAmelCase      					=  self.compute_metrics(__lowerCamelCase							)

            # Prefix all keys with metric_key_prefix + '_'
            for key in list(metrics.keys()							):
               if not key.startswith(F"""{metric_key_prefix}_"""							):
                  UpperCAmelCase      					=  metrics.pop(__lowerCamelCase							)
            metrics.update(output.metrics							)
         else:
            UpperCAmelCase      					=  output.metrics

         if self.args.should_log:
            # Only the main node log the results by default
            self.log(__lowerCamelCase							)

         if self.args.tpu_metrics_debug or self.args.debug:
            # tpu-comment: Logging debug metrics for PyTorch/XLA (compile, execute times, ops, etc.)
            xm.master_print(met.metrics_report()							)

         UpperCAmelCase      					=  self.callback_handler.on_evaluate(self.args       ,							self.state       ,							self.control       ,							__lowerCamelCase							)
         return metrics





      def 							_lowercase					(		self    :       Optional[Any]       ,							__lowerCamelCase    :       Union[str, Any]       ,							__lowerCamelCase    :       Any       ,							__lowerCamelCase    :       Dict=None       ,							__lowerCamelCase    :       str = "test"							)     -> Dict:






         """simple docstring"""







         UpperCAmelCase      					=  self.get_test_dataloader(__lowerCamelCase							)

         # Temporarily disable metric computation, we will do it in the loop here.
         UpperCAmelCase      					=  self.compute_metrics
         UpperCAmelCase      					=  None
         UpperCAmelCase      					=  self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop
         UpperCAmelCase      					=  time.time()
         try:
            UpperCAmelCase      					=  eval_loop(
                __lowerCamelCase       ,							description="""Prediction"""       ,							prediction_loss_only=True if compute_metrics is None else None       ,							ignore_keys=__lowerCamelCase       ,							metric_key_prefix=__lowerCamelCase       ,							)
         finally:
            UpperCAmelCase      					=  compute_metrics
         UpperCAmelCase      					=  self.args.eval_batch_size * self.args.world_size
         if F"""{metric_key_prefix}_jit_compilation_time""" in output.metrics:
            start_time += output.metrics[F"""{metric_key_prefix}_jit_compilation_time"""]
         output.metrics.update(
             speed_metrics(
                 __lowerCamelCase       ,							__lowerCamelCase       ,							num_samples=output.num_samples       ,							num_steps=math.ceil(output.num_samples / total_batch_size							)       ,							)							)

         if self.post_process_function is None or self.compute_metrics is None:
            return output

         UpperCAmelCase      					=  self.post_process_function(__lowerCamelCase       ,							__lowerCamelCase       ,							output.predictions       ,							"""predict"""							)
         UpperCAmelCase      					=  self.compute_metrics(__lowerCamelCase							)

         # Prefix all keys with metric_key_prefix + '_'
         for key in list(metrics.keys()							):
            if not key.startswith(F"""{metric_key_prefix}_"""							):
               UpperCAmelCase      					=  metrics.pop(__lowerCamelCase							)
         metrics.update(output.metrics							)
         return PredictionOutput(predictions=predictions.predictions       ,							label_ids=predictions.label_ids       ,							metrics=__lowerCamelCase							)



