





import argparse
from pathlib import Path

import fairseq
import torch
from fairseq.models.xmod import XMODModel as FairseqXmodModel
from packaging import version

from transformers import XmodConfig, XmodForMaskedLM, XmodForSequenceClassification
from transformers.utils import logging


if version.parse(fairseq.__version__) < version.parse("""0.12.2"""):
							raise Exception("""requires fairseq >= 0.12.2""")
if version.parse(fairseq.__version__) > version.parse("""2"""):
							raise Exception("""requires fairseq < v2""")

logging.set_verbosity_info()
lowerCAmelCase							:       str					 =	logging.get_logger(__name__)

lowerCAmelCase							:       Dict					 =	"""Hello, World!"""
lowerCAmelCase							:       Tuple					 =	"""en_XX"""






def 		lowerCAmelCase  (	UpperCamelCase__   :   str       ,			UpperCamelCase__   :   str       ,			UpperCamelCase__   :   bool					)      ->    int:

						"""simple docstring"""





						__SCREAMING_SNAKE_CASE:				str           =   Path('''data_bin'''					)
						__SCREAMING_SNAKE_CASE:				Optional[int]           =   FairseqXmodModel.from_pretrained(
						    model_name_or_path=str(Path(UpperCamelCase__					).parent					)       ,			checkpoint_file=Path(UpperCamelCase__					).name       ,			_name='''xmod_base'''       ,			arch='''xmod_base'''       ,			task='''multilingual_masked_lm'''       ,			data_name_or_path=str(UpperCamelCase__					)       ,			bpe='''sentencepiece'''       ,			sentencepiece_model=str(Path(UpperCamelCase__					).parent / '''sentencepiece.bpe.model'''					)       ,			src_dict=str(data_dir / '''dict.txt'''					)       ,			)
						xmod.eval()  # disable dropout
						print(UpperCamelCase__					)

						__SCREAMING_SNAKE_CASE:				Dict           =   xmod.model.encoder.sentence_encoder
						__SCREAMING_SNAKE_CASE:				List[Any]           =   XmodConfig(
						    vocab_size=xmod_sent_encoder.embed_tokens.num_embeddings       ,			hidden_size=xmod.cfg.model.encoder_embed_dim       ,			num_hidden_layers=xmod.cfg.model.encoder_layers       ,			num_attention_heads=xmod.cfg.model.encoder_attention_heads       ,			intermediate_size=xmod.cfg.model.encoder_ffn_embed_dim       ,			max_position_embeddings=514       ,			type_vocab_size=1       ,			layer_norm_eps=1E-5       ,			pre_norm=xmod.cfg.model.encoder_normalize_before       ,			adapter_reduction_factor=getattr(xmod.cfg.model       ,			'''bottleneck'''       ,			2					)       ,			adapter_layer_norm=xmod.cfg.model.adapter_layer_norm       ,			adapter_reuse_layer_norm=xmod.cfg.model.adapter_reuse_layer_norm       ,			ln_before_adapter=xmod.cfg.model.ln_before_adapter       ,			languages=xmod.cfg.model.languages       ,			)
						if classification_head:
												__SCREAMING_SNAKE_CASE:				int           =   xmod.model.classification_heads['''mnli'''].out_proj.weight.shape[0]

						print('''Our X-MOD config:'''       ,			UpperCamelCase__					)

						__SCREAMING_SNAKE_CASE:				List[str]           =   XmodForSequenceClassification(UpperCamelCase__					) if classification_head else XmodForMaskedLM(UpperCamelCase__					)
						model.eval()

						# Now let's copy all the weights.
						# Embeddings
						__SCREAMING_SNAKE_CASE:				Optional[Any]           =   xmod_sent_encoder.embed_tokens.weight
						__SCREAMING_SNAKE_CASE:				Optional[int]           =   xmod_sent_encoder.embed_positions.weight
						__SCREAMING_SNAKE_CASE:				Optional[Any]           =   torch.zeros_like(
						    model.roberta.embeddings.token_type_embeddings.weight					)  # just zero them out b/c xmod doesn't use them.

						__SCREAMING_SNAKE_CASE:				Tuple           =   xmod_sent_encoder.layernorm_embedding.weight
						__SCREAMING_SNAKE_CASE:				int           =   xmod_sent_encoder.layernorm_embedding.bias

						for i in range(config.num_hidden_layers					):
												# Encoder: start of layer
												__SCREAMING_SNAKE_CASE:				Dict           =   model.roberta.encoder.layer[i]
												__SCREAMING_SNAKE_CASE:				Tuple           =   xmod_sent_encoder.layers[i]

												# self attention
												__SCREAMING_SNAKE_CASE:				int           =   layer.attention.self
												if not (
												    xmod_layer.self_attn.k_proj.weight.data.shape
												    == xmod_layer.self_attn.q_proj.weight.data.shape
												    == xmod_layer.self_attn.v_proj.weight.data.shape
												    == torch.Size((config.hidden_size, config.hidden_size)					)
												):
																		raise AssertionError('''Dimensions of self-attention weights do not match.'''					)

												__SCREAMING_SNAKE_CASE:				List[str]           =   xmod_layer.self_attn.q_proj.weight
												__SCREAMING_SNAKE_CASE:				Any           =   xmod_layer.self_attn.q_proj.bias
												__SCREAMING_SNAKE_CASE:				List[str]           =   xmod_layer.self_attn.k_proj.weight
												__SCREAMING_SNAKE_CASE:				int           =   xmod_layer.self_attn.k_proj.bias
												__SCREAMING_SNAKE_CASE:				Optional[Any]           =   xmod_layer.self_attn.v_proj.weight
												__SCREAMING_SNAKE_CASE:				Union[str, Any]           =   xmod_layer.self_attn.v_proj.bias

												# self-attention output
												__SCREAMING_SNAKE_CASE:				str           =   layer.attention.output
												if self_output.dense.weight.shape != xmod_layer.self_attn.out_proj.weight.shape:
																		raise AssertionError('''Dimensions of self-attention output weights do not match.'''					)
												__SCREAMING_SNAKE_CASE:				Dict           =   xmod_layer.self_attn.out_proj.weight
												__SCREAMING_SNAKE_CASE:				int           =   xmod_layer.self_attn.out_proj.bias
												__SCREAMING_SNAKE_CASE:				int           =   xmod_layer.self_attn_layer_norm.weight
												__SCREAMING_SNAKE_CASE:				Dict           =   xmod_layer.self_attn_layer_norm.bias

												# intermediate
												__SCREAMING_SNAKE_CASE:				Any           =   layer.intermediate
												if intermediate.dense.weight.shape != xmod_layer.fca.weight.shape:
																		raise AssertionError('''Dimensions of intermediate weights do not match.'''					)
												__SCREAMING_SNAKE_CASE:				str           =   xmod_layer.fca.weight
												__SCREAMING_SNAKE_CASE:				int           =   xmod_layer.fca.bias

												# output
												__SCREAMING_SNAKE_CASE:				List[str]           =   layer.output
												if bert_output.dense.weight.shape != xmod_layer.fca.weight.shape:
																		raise AssertionError('''Dimensions of feed-forward weights do not match.'''					)
												__SCREAMING_SNAKE_CASE:				List[Any]           =   xmod_layer.fca.weight
												__SCREAMING_SNAKE_CASE:				Union[str, Any]           =   xmod_layer.fca.bias
												__SCREAMING_SNAKE_CASE:				List[str]           =   xmod_layer.final_layer_norm.weight
												__SCREAMING_SNAKE_CASE:				Dict           =   xmod_layer.final_layer_norm.bias
												if bert_output.adapter_layer_norm is not None:
																		__SCREAMING_SNAKE_CASE:				List[str]           =   xmod_layer.adapter_layer_norm.weight
																		__SCREAMING_SNAKE_CASE:				List[Any]           =   xmod_layer.adapter_layer_norm.bias

												if sorted(bert_output.adapter_modules.keys()					) != sorted(xmod_layer.adapter_modules.keys()					):
																		raise AssertionError('''Lists of language adapters do not match.'''					)
												for lang_code, adapter in xmod_layer.adapter_modules.items():
																		__SCREAMING_SNAKE_CASE:				Tuple           =   bert_output.adapter_modules[lang_code]
																		__SCREAMING_SNAKE_CASE:				List[str]           =   xmod_layer.adapter_modules[lang_code]
																		__SCREAMING_SNAKE_CASE:				List[Any]           =   from_adapter.fca.weight
																		__SCREAMING_SNAKE_CASE:				str           =   from_adapter.fca.bias
																		__SCREAMING_SNAKE_CASE:				Optional[Any]           =   from_adapter.fca.weight
																		__SCREAMING_SNAKE_CASE:				int           =   from_adapter.fca.bias

												# end of layer

						if xmod_sent_encoder.layer_norm is not None:
												__SCREAMING_SNAKE_CASE:				List[str]           =   xmod_sent_encoder.layer_norm.weight
												__SCREAMING_SNAKE_CASE:				Dict           =   xmod_sent_encoder.layer_norm.bias

						if classification_head:
												__SCREAMING_SNAKE_CASE:				Optional[Any]           =   xmod.model.classification_heads['''mnli'''].dense.weight
												__SCREAMING_SNAKE_CASE:				Dict           =   xmod.model.classification_heads['''mnli'''].dense.bias
												__SCREAMING_SNAKE_CASE:				Any           =   xmod.model.classification_heads['''mnli'''].out_proj.weight
												__SCREAMING_SNAKE_CASE:				List[str]           =   xmod.model.classification_heads['''mnli'''].out_proj.bias
						else:
												# LM Head
												__SCREAMING_SNAKE_CASE:				str           =   xmod.model.encoder.lm_head.dense.weight
												__SCREAMING_SNAKE_CASE:				Optional[Any]           =   xmod.model.encoder.lm_head.dense.bias
												__SCREAMING_SNAKE_CASE:				Union[str, Any]           =   xmod.model.encoder.lm_head.layer_norm.weight
												__SCREAMING_SNAKE_CASE:				Dict           =   xmod.model.encoder.lm_head.layer_norm.bias
												__SCREAMING_SNAKE_CASE:				int           =   xmod.model.encoder.lm_head.weight
												__SCREAMING_SNAKE_CASE:				int           =   xmod.model.encoder.lm_head.bias

						# Let's check that we get the same results.
						__SCREAMING_SNAKE_CASE:				List[Any]           =   xmod.encode(UpperCamelCase__					).unsqueeze(0					)  # batch of size 1
						model.roberta.set_default_language(UpperCamelCase__					)

						__SCREAMING_SNAKE_CASE:				List[str]           =   model(UpperCamelCase__					)[0]
						if classification_head:
												__SCREAMING_SNAKE_CASE:				Optional[Any]           =   xmod.model.classification_heads['''mnli'''](xmod.extract_features(UpperCamelCase__					)					)
						else:
												__SCREAMING_SNAKE_CASE:				Optional[Any]           =   xmod.model(UpperCamelCase__       ,			lang_id=[SAMPLE_LANGUAGE]					)[0]
						print(our_output.shape       ,			their_output.shape					)
						__SCREAMING_SNAKE_CASE:				Union[str, Any]           =   torch.max(torch.abs(our_output - their_output					)					).item()
						print(F"""max_absolute_diff = {max_absolute_diff}"""					)  # ~ 1e-7
						__SCREAMING_SNAKE_CASE:				str           =   torch.allclose(UpperCamelCase__       ,			UpperCamelCase__       ,			atol=1E-3					)
						print('''Do both models output the same tensors?'''       ,			'''🔥''' if success else '''💩'''					)
						if not success:
												raise Exception('''Something went wRoNg'''					)

						Path(UpperCamelCase__					).mkdir(parents=UpperCamelCase__       ,			exist_ok=UpperCamelCase__					)
						print(F"""Saving model to {pytorch_dump_folder_path}"""					)
						model.save_pretrained(UpperCamelCase__					)


if __name__ == "__main__":
							lowerCAmelCase							:       Any					 =	argparse.ArgumentParser()
							# Required parameters
							parser.add_argument(
							    """--xmod_checkpoint_path""", default=None, type=str, required=True, help="""Path the official PyTorch dump."""
							)
							parser.add_argument(
							    """--pytorch_dump_folder_path""", default=None, type=str, required=True, help="""Path to the output PyTorch model."""
							)
							parser.add_argument(
							    """--classification_head""", action="""store_true""", help="""Whether to convert a final classification head."""
							)
							lowerCAmelCase							:       List[Any]					 =	parser.parse_args()
							convert_xmod_checkpoint_to_pytorch(
							    args.xmod_checkpoint_path, args.pytorch_dump_folder_path, args.classification_head
							)










from typing import Optional, Tuple, Union

import flax
import flax.linen as nn
import jax
import jax.numpy as jnp
from flax.core.frozen_dict import FrozenDict

from ..configuration_utils import ConfigMixin, flax_register_to_config
from ..utils import BaseOutput
from .embeddings_flax import FlaxTimestepEmbedding, FlaxTimesteps
from .modeling_flax_utils import FlaxModelMixin
from .unet_ad_blocks_flax import (
    FlaxCrossAttnDownBlockaD,
    FlaxCrossAttnUpBlockaD,
    FlaxDownBlockaD,
    FlaxUNetMidBlockaDCrossAttn,
    FlaxUpBlockaD,
)





@flax.struct.dataclass
class 				a			(      __lowercase			):
				SCREAMING_SNAKE_CASE__     :   jnp.ndarray





@flax_register_to_config
class 				a			(      nn.Module   ,__lowercase   ,__lowercase			):
				SCREAMING_SNAKE_CASE__     :   int		      =     32
				SCREAMING_SNAKE_CASE__     :   int		      =     4
				SCREAMING_SNAKE_CASE__     :   int		      =     4
				SCREAMING_SNAKE_CASE__     :   Tuple[str]		      =     (
				    "CrossAttnDownBlock2D",
				    "CrossAttnDownBlock2D",
				    "CrossAttnDownBlock2D",
				    "DownBlock2D",
				)
				SCREAMING_SNAKE_CASE__     :   Tuple[str]		      =     ("UpBlock2D", "CrossAttnUpBlock2D", "CrossAttnUpBlock2D", "CrossAttnUpBlock2D")
				SCREAMING_SNAKE_CASE__     :   Union[bool, Tuple[bool]]		      =     False
				SCREAMING_SNAKE_CASE__     :   Tuple[int]		      =     (320, 640, 1280, 1280)
				SCREAMING_SNAKE_CASE__     :   int		      =     2
				SCREAMING_SNAKE_CASE__     :   Union[int, Tuple[int]]		      =     8
				SCREAMING_SNAKE_CASE__     :   Optional[Union[int, Tuple[int]]]		      =     None
				SCREAMING_SNAKE_CASE__     :   int		      =     1280
				SCREAMING_SNAKE_CASE__     :   float		      =     0.0
				SCREAMING_SNAKE_CASE__     :   bool		      =     False
				SCREAMING_SNAKE_CASE__     :   jnp.dtype		      =     jnp.floataa
				SCREAMING_SNAKE_CASE__     :   bool		      =     True
				SCREAMING_SNAKE_CASE__     :   int		      =     0
				SCREAMING_SNAKE_CASE__     :   bool		      =     False






				def 			snake_case_    (  self			, _lowerCAmelCase					):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				int           =   (1, self.in_channels, self.sample_size, self.sample_size)
										__SCREAMING_SNAKE_CASE:				Tuple           =   jnp.zeros(_lowerCAmelCase			, dtype=jnp.floataa					)
										__SCREAMING_SNAKE_CASE:				Optional[Any]           =   jnp.ones((1,)			, dtype=jnp.intaa					)
										__SCREAMING_SNAKE_CASE:				Optional[int]           =   jnp.zeros((1, 1, self.cross_attention_dim)			, dtype=jnp.floataa					)

										__SCREAMING_SNAKE_CASE       ,__SCREAMING_SNAKE_CASE:				Optional[int]           =   jax.random.split(_lowerCAmelCase					)
										__SCREAMING_SNAKE_CASE:				Optional[Any]           =   {'''params''': params_rng, '''dropout''': dropout_rng}

										return self.init(_lowerCAmelCase			, _lowerCAmelCase			, _lowerCAmelCase			, _lowerCAmelCase					)["params"]






				def 			snake_case_    (  self					):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				int           =   self.block_out_channels
										__SCREAMING_SNAKE_CASE:				Union[str, Any]           =   block_out_channels[0] * 4

										if self.num_attention_heads is not None:
																raise ValueError(
																    '''At the moment it is not possible to define the number of attention heads via `num_attention_heads` because of a naming issue as described in https://github.com/huggingface/diffusers/issues/2011#issuecomment-1547958131. Passing `num_attention_heads` will only be supported in diffusers v0.19.'''					)

										# If `num_attention_heads` is not defined (which is the case for most models)
										# it will default to `attention_head_dim`. This looks weird upon first reading it and it is.
										# The reason for this behavior is to correct for incorrectly named variables that were introduced
										# when this library was created. The incorrect naming was only discovered much later in https://github.com/huggingface/diffusers/issues/2011#issuecomment-1547958131
										# Changing `attention_head_dim` to `num_attention_heads` for 40,000+ configurations is too backwards breaking
										# which is why we correct for the naming here.
										__SCREAMING_SNAKE_CASE:				Any           =   self.num_attention_heads or self.attention_head_dim

										# input
										__SCREAMING_SNAKE_CASE:				str           =   nn.Conv(
										    block_out_channels[0]			, kernel_size=(3, 3)			, strides=(1, 1)			, padding=((1, 1), (1, 1))			, dtype=self.dtype			, )

										# time
										__SCREAMING_SNAKE_CASE:				int           =   FlaxTimesteps(
										    block_out_channels[0]			, flip_sin_to_cos=self.flip_sin_to_cos			, freq_shift=self.config.freq_shift					)
										__SCREAMING_SNAKE_CASE:				Union[str, Any]           =   FlaxTimestepEmbedding(_lowerCAmelCase			, dtype=self.dtype					)

										__SCREAMING_SNAKE_CASE:				Optional[int]           =   self.only_cross_attention
										if isinstance(_lowerCAmelCase			, _lowerCAmelCase					):
																__SCREAMING_SNAKE_CASE:				Union[str, Any]           =   (only_cross_attention,) * len(self.down_block_types					)

										if isinstance(_lowerCAmelCase			, _lowerCAmelCase					):
																__SCREAMING_SNAKE_CASE:				Any           =   (num_attention_heads,) * len(self.down_block_types					)

										# down
										__SCREAMING_SNAKE_CASE:				Union[str, Any]           =   []
										__SCREAMING_SNAKE_CASE:				List[str]           =   block_out_channels[0]
										for i, down_block_type in enumerate(self.down_block_types					):
																__SCREAMING_SNAKE_CASE:				List[str]           =   output_channel
																__SCREAMING_SNAKE_CASE:				str           =   block_out_channels[i]
																__SCREAMING_SNAKE_CASE:				Any           =   i == len(_lowerCAmelCase					) - 1

																if down_block_type == "CrossAttnDownBlock2D":
																						__SCREAMING_SNAKE_CASE:				str           =   FlaxCrossAttnDownBlockaD(
																						    in_channels=_lowerCAmelCase			, out_channels=_lowerCAmelCase			, dropout=self.dropout			, num_layers=self.layers_per_block			, num_attention_heads=num_attention_heads[i]			, add_downsample=not is_final_block			, use_linear_projection=self.use_linear_projection			, only_cross_attention=only_cross_attention[i]			, use_memory_efficient_attention=self.use_memory_efficient_attention			, dtype=self.dtype			, )
																else:
																						__SCREAMING_SNAKE_CASE:				Tuple           =   FlaxDownBlockaD(
																						    in_channels=_lowerCAmelCase			, out_channels=_lowerCAmelCase			, dropout=self.dropout			, num_layers=self.layers_per_block			, add_downsample=not is_final_block			, dtype=self.dtype			, )

																down_blocks.append(_lowerCAmelCase					)
										__SCREAMING_SNAKE_CASE:				str           =   down_blocks

										# mid
										__SCREAMING_SNAKE_CASE:				Union[str, Any]           =   FlaxUNetMidBlockaDCrossAttn(
										    in_channels=block_out_channels[-1]			, dropout=self.dropout			, num_attention_heads=num_attention_heads[-1]			, use_linear_projection=self.use_linear_projection			, use_memory_efficient_attention=self.use_memory_efficient_attention			, dtype=self.dtype			, )

										# up
										__SCREAMING_SNAKE_CASE:				Optional[int]           =   []
										__SCREAMING_SNAKE_CASE:				Tuple           =   list(reversed(_lowerCAmelCase					)					)
										__SCREAMING_SNAKE_CASE:				Optional[int]           =   list(reversed(_lowerCAmelCase					)					)
										__SCREAMING_SNAKE_CASE:				str           =   list(reversed(_lowerCAmelCase					)					)
										__SCREAMING_SNAKE_CASE:				Optional[int]           =   reversed_block_out_channels[0]
										for i, up_block_type in enumerate(self.up_block_types					):
																__SCREAMING_SNAKE_CASE:				int           =   output_channel
																__SCREAMING_SNAKE_CASE:				List[str]           =   reversed_block_out_channels[i]
																__SCREAMING_SNAKE_CASE:				List[str]           =   reversed_block_out_channels[min(i + 1			, len(_lowerCAmelCase					) - 1					)]

																__SCREAMING_SNAKE_CASE:				Union[str, Any]           =   i == len(_lowerCAmelCase					) - 1

																if up_block_type == "CrossAttnUpBlock2D":
																						__SCREAMING_SNAKE_CASE:				Optional[int]           =   FlaxCrossAttnUpBlockaD(
																						    in_channels=_lowerCAmelCase			, out_channels=_lowerCAmelCase			, prev_output_channel=_lowerCAmelCase			, num_layers=self.layers_per_block + 1			, num_attention_heads=reversed_num_attention_heads[i]			, add_upsample=not is_final_block			, dropout=self.dropout			, use_linear_projection=self.use_linear_projection			, only_cross_attention=only_cross_attention[i]			, use_memory_efficient_attention=self.use_memory_efficient_attention			, dtype=self.dtype			, )
																else:
																						__SCREAMING_SNAKE_CASE:				int           =   FlaxUpBlockaD(
																						    in_channels=_lowerCAmelCase			, out_channels=_lowerCAmelCase			, prev_output_channel=_lowerCAmelCase			, num_layers=self.layers_per_block + 1			, add_upsample=not is_final_block			, dropout=self.dropout			, dtype=self.dtype			, )

																up_blocks.append(_lowerCAmelCase					)
																__SCREAMING_SNAKE_CASE:				Any           =   output_channel
										__SCREAMING_SNAKE_CASE:				Union[str, Any]           =   up_blocks

										# out
										__SCREAMING_SNAKE_CASE:				Optional[int]           =   nn.GroupNorm(num_groups=32			, epsilon=1e-5					)
										__SCREAMING_SNAKE_CASE:				Optional[Any]           =   nn.Conv(
										    self.out_channels			, kernel_size=(3, 3)			, strides=(1, 1)			, padding=((1, 1), (1, 1))			, dtype=self.dtype			, )






				def __call__(  self			, _lowerCAmelCase			, _lowerCAmelCase			, _lowerCAmelCase			, _lowerCAmelCase=None			, _lowerCAmelCase=None			, _lowerCAmelCase = True			, _lowerCAmelCase = False			, ):

										"""simple docstring"""




										if not isinstance(_lowerCAmelCase			, jnp.ndarray					):
																__SCREAMING_SNAKE_CASE:				Dict           =   jnp.array([timesteps]			, dtype=jnp.intaa					)
										elif isinstance(_lowerCAmelCase			, jnp.ndarray					) and len(timesteps.shape					) == 0:
																__SCREAMING_SNAKE_CASE:				Optional[Any]           =   timesteps.astype(dtype=jnp.floataa					)
																__SCREAMING_SNAKE_CASE:				Union[str, Any]           =   jnp.expand_dims(_lowerCAmelCase			, 0					)

										__SCREAMING_SNAKE_CASE:				Any           =   self.time_proj(_lowerCAmelCase					)
										__SCREAMING_SNAKE_CASE:				Union[str, Any]           =   self.time_embedding(_lowerCAmelCase					)

										# 2. pre-process
										__SCREAMING_SNAKE_CASE:				int           =   jnp.transpose(_lowerCAmelCase			, (0, 2, 3, 1)					)
										__SCREAMING_SNAKE_CASE:				List[str]           =   self.conv_in(_lowerCAmelCase					)

										# 3. down
										__SCREAMING_SNAKE_CASE:				Dict           =   (sample,)
										for down_block in self.down_blocks:
																if isinstance(_lowerCAmelCase			, _lowerCAmelCase					):
																						__SCREAMING_SNAKE_CASE       ,__SCREAMING_SNAKE_CASE:				Optional[Any]           =   down_block(_lowerCAmelCase			, _lowerCAmelCase			, _lowerCAmelCase			, deterministic=not train					)
																else:
																						__SCREAMING_SNAKE_CASE       ,__SCREAMING_SNAKE_CASE:				Tuple           =   down_block(_lowerCAmelCase			, _lowerCAmelCase			, deterministic=not train					)
																down_block_res_samples += res_samples

										if down_block_additional_residuals is not None:
																__SCREAMING_SNAKE_CASE:				Union[str, Any]           =   ()

																for down_block_res_sample, down_block_additional_residual in zip(
																    _lowerCAmelCase			, _lowerCAmelCase					):
																						down_block_res_sample += down_block_additional_residual
																						new_down_block_res_samples += (down_block_res_sample,)

																__SCREAMING_SNAKE_CASE:				Union[str, Any]           =   new_down_block_res_samples

										# 4. mid
										__SCREAMING_SNAKE_CASE:				Dict           =   self.mid_block(_lowerCAmelCase			, _lowerCAmelCase			, _lowerCAmelCase			, deterministic=not train					)

										if mid_block_additional_residual is not None:
																sample += mid_block_additional_residual

										# 5. up
										for up_block in self.up_blocks:
																__SCREAMING_SNAKE_CASE:				Union[str, Any]           =   down_block_res_samples[-(self.layers_per_block + 1) :]
																__SCREAMING_SNAKE_CASE:				Union[str, Any]           =   down_block_res_samples[: -(self.layers_per_block + 1)]
																if isinstance(_lowerCAmelCase			, _lowerCAmelCase					):
																						__SCREAMING_SNAKE_CASE:				Any           =   up_block(
																						    _lowerCAmelCase			, temb=_lowerCAmelCase			, encoder_hidden_states=_lowerCAmelCase			, res_hidden_states_tuple=_lowerCAmelCase			, deterministic=not train			, )
																else:
																						__SCREAMING_SNAKE_CASE:				List[str]           =   up_block(_lowerCAmelCase			, temb=_lowerCAmelCase			, res_hidden_states_tuple=_lowerCAmelCase			, deterministic=not train					)

        # 6. post-process
										__SCREAMING_SNAKE_CASE:				Optional[Any]           =   self.conv_norm_out(_lowerCAmelCase					)
										__SCREAMING_SNAKE_CASE:				Optional[Any]           =   nn.silu(_lowerCAmelCase					)
										__SCREAMING_SNAKE_CASE:				Tuple           =   self.conv_out(_lowerCAmelCase					)
										__SCREAMING_SNAKE_CASE:				int           =   jnp.transpose(_lowerCAmelCase			, (0, 3, 1, 2)					)

										if not return_dict:
																return (sample,)

										return FlaxUNetaDConditionOutput(sample=_lowerCAmelCase					)



