





import argparse
import json
from pathlib import Path

import requests
import torch
from huggingface_hub import hf_hub_download
from PIL import Image

from transformers import (
    BertTokenizer,
    ViltConfig,
    ViltForImageAndTextRetrieval,
    ViltForImagesAndTextClassification,
    ViltForMaskedLM,
    ViltForQuestionAnswering,
    ViltImageProcessor,
    ViltProcessor,
)
from transformers.utils import logging


logging.set_verbosity_info()
_UpperCAmelCase       : Tuple			     =  logging.get_logger(__name__)


def  SCREAMING_SNAKE_CASE   ( _UpperCAmelCase		,			_UpperCAmelCase=False		,			_UpperCAmelCase=False		,			_UpperCAmelCase=False				)    ->  Dict:
    lowerCamelCase__					:   int      				=	[]
    for i in range(config.num_hidden_layers				):
        # encoder layers: output projection, 2 feedforward neural networks and 2 layernorms
        rename_keys.append((F"""transformer.blocks.{i}.norm1.weight""", F"""vilt.encoder.layer.{i}.layernorm_before.weight""")				)
        rename_keys.append((F"""transformer.blocks.{i}.norm1.bias""", F"""vilt.encoder.layer.{i}.layernorm_before.bias""")				)
        rename_keys.append(
            (F"""transformer.blocks.{i}.attn.proj.weight""", F"""vilt.encoder.layer.{i}.attention.output.dense.weight""")				)
        rename_keys.append(
            (F"""transformer.blocks.{i}.attn.proj.bias""", F"""vilt.encoder.layer.{i}.attention.output.dense.bias""")				)
        rename_keys.append((F"""transformer.blocks.{i}.norm2.weight""", F"""vilt.encoder.layer.{i}.layernorm_after.weight""")				)
        rename_keys.append((F"""transformer.blocks.{i}.norm2.bias""", F"""vilt.encoder.layer.{i}.layernorm_after.bias""")				)
        rename_keys.append(
            (F"""transformer.blocks.{i}.mlp.fc1.weight""", F"""vilt.encoder.layer.{i}.intermediate.dense.weight""")				)
        rename_keys.append((F"""transformer.blocks.{i}.mlp.fc1.bias""", F"""vilt.encoder.layer.{i}.intermediate.dense.bias""")				)
        rename_keys.append((F"""transformer.blocks.{i}.mlp.fc2.weight""", F"""vilt.encoder.layer.{i}.output.dense.weight""")				)
        rename_keys.append((F"""transformer.blocks.{i}.mlp.fc2.bias""", F"""vilt.encoder.layer.{i}.output.dense.bias""")				)

    # embeddings
    rename_keys.extend(
        [
            # text embeddings
            ('text_embeddings.word_embeddings.weight', 'vilt.embeddings.text_embeddings.word_embeddings.weight'),
            (
                'text_embeddings.position_embeddings.weight',
                'vilt.embeddings.text_embeddings.position_embeddings.weight',
            ),
            ('text_embeddings.position_ids', 'vilt.embeddings.text_embeddings.position_ids'),
            (
                'text_embeddings.token_type_embeddings.weight',
                'vilt.embeddings.text_embeddings.token_type_embeddings.weight',
            ),
            ('text_embeddings.LayerNorm.weight', 'vilt.embeddings.text_embeddings.LayerNorm.weight'),
            ('text_embeddings.LayerNorm.bias', 'vilt.embeddings.text_embeddings.LayerNorm.bias'),
            # patch embeddings
            ('transformer.cls_token', 'vilt.embeddings.cls_token'),
            ('transformer.patch_embed.proj.weight', 'vilt.embeddings.patch_embeddings.projection.weight'),
            ('transformer.patch_embed.proj.bias', 'vilt.embeddings.patch_embeddings.projection.bias'),
            ('transformer.pos_embed', 'vilt.embeddings.position_embeddings'),
            # token type embeddings
            ('token_type_embeddings.weight', 'vilt.embeddings.token_type_embeddings.weight'),
        ]				)

    # final layernorm + pooler
    rename_keys.extend(
        [
            ('transformer.norm.weight', 'vilt.layernorm.weight'),
            ('transformer.norm.bias', 'vilt.layernorm.bias'),
            ('pooler.dense.weight', 'vilt.pooler.dense.weight'),
            ('pooler.dense.bias', 'vilt.pooler.dense.bias'),
        ]				)

    # classifier head(s)
    if vqa_model:
        # classification head
        rename_keys.extend(
            [
                ('vqa_classifier.0.weight', 'classifier.0.weight'),
                ('vqa_classifier.0.bias', 'classifier.0.bias'),
                ('vqa_classifier.1.weight', 'classifier.1.weight'),
                ('vqa_classifier.1.bias', 'classifier.1.bias'),
                ('vqa_classifier.3.weight', 'classifier.3.weight'),
                ('vqa_classifier.3.bias', 'classifier.3.bias'),
            ]				)
    elif nlvr_model:
        # classification head
        rename_keys.extend(
            [
                ('nlvr2_classifier.0.weight', 'classifier.0.weight'),
                ('nlvr2_classifier.0.bias', 'classifier.0.bias'),
                ('nlvr2_classifier.1.weight', 'classifier.1.weight'),
                ('nlvr2_classifier.1.bias', 'classifier.1.bias'),
                ('nlvr2_classifier.3.weight', 'classifier.3.weight'),
                ('nlvr2_classifier.3.bias', 'classifier.3.bias'),
            ]				)
    else:
        pass

    return rename_keys


def  SCREAMING_SNAKE_CASE   ( _UpperCAmelCase		,			_UpperCAmelCase				)    ->  Any:

    for i in range(config.num_hidden_layers				):
        lowerCamelCase__					:   str      				=	'''vilt.'''
        # read in weights + bias of input projection layer (in timm, this is a single matrix + bias)
        lowerCamelCase__					:   Union[str, Any]      				=	state_dict.pop(F"""transformer.blocks.{i}.attn.qkv.weight"""				)
        lowerCamelCase__					:   List[Any]      				=	state_dict.pop(F"""transformer.blocks.{i}.attn.qkv.bias"""				)
        # next, add query, keys and values (in that order) to the state dict
        lowerCamelCase__					:   str      				=	in_proj_weight[
            : config.hidden_size, :
        ]
        lowerCamelCase__					:   Optional[Any]      				=	in_proj_bias[: config.hidden_size]
        lowerCamelCase__					:   Dict      				=	in_proj_weight[
            config.hidden_size : config.hidden_size * 2, :
        ]
        lowerCamelCase__					:   int      				=	in_proj_bias[
            config.hidden_size : config.hidden_size * 2
        ]
        lowerCamelCase__					:   List[Any]      				=	in_proj_weight[
            -config.hidden_size :, :
        ]
        lowerCamelCase__					:   Union[str, Any]      				=	in_proj_bias[-config.hidden_size :]


def  SCREAMING_SNAKE_CASE   ( _UpperCAmelCase				)    ->  List[str]:
    lowerCamelCase__					:   List[str]      				=	['''head.weight''', '''head.bias''']
    for k in ignore_keys:
        state_dict.pop(lowerCamelCase_		,			lowerCamelCase_				)


def  SCREAMING_SNAKE_CASE   ( _UpperCAmelCase		,			_UpperCAmelCase		,			_UpperCAmelCase				)    ->  Union[str, Any]:
    lowerCamelCase__					:   Any      				=	dct.pop(lowerCamelCase_				)
    lowerCamelCase__					:   Optional[Any]      				=	val


@torch.no_grad()
def  SCREAMING_SNAKE_CASE   ( _UpperCAmelCase		,			_UpperCAmelCase				)    ->  int:
    lowerCamelCase__					:   List[str]      				=	ViltConfig(image_size=384		,			patch_size=32		,			tie_word_embeddings=lowerCamelCase_				)
    lowerCamelCase__					:   Dict      				=	False
    lowerCamelCase__					:   Tuple      				=	False
    lowerCamelCase__					:   Union[str, Any]      				=	False
    lowerCamelCase__					:   int      				=	False
    if "vqa" in checkpoint_url:
        lowerCamelCase__					:   Tuple      				=	True
        lowerCamelCase__					:   Any      				=	3129
        lowerCamelCase__					:   Optional[Any]      				=	'''huggingface/label-files'''
        lowerCamelCase__					:   List[str]      				=	'''vqa2-id2label.json'''
        lowerCamelCase__					:   Union[str, Any]      				=	json.load(open(hf_hub_download(lowerCamelCase_		,			lowerCamelCase_		,			repo_type='dataset'				)		,			'r'				)				)
        lowerCamelCase__					:   Any      				=	{int(lowerCamelCase_				): v for k, v in idalabel.items()}
        lowerCamelCase__					:   Dict      				=	idalabel
        lowerCamelCase__					:   int      				=	{v: k for k, v in idalabel.items()}
        lowerCamelCase__					:   str      				=	ViltForQuestionAnswering(lowerCamelCase_				)
    elif "nlvr" in checkpoint_url:
        lowerCamelCase__					:   int      				=	True
        lowerCamelCase__					:   Dict      				=	2
        lowerCamelCase__					:   List[str]      				=	{0: '''False''', 1: '''True'''}
        lowerCamelCase__					:   Optional[Any]      				=	{v: k for k, v in config.idalabel.items()}
        lowerCamelCase__					:   int      				=	3
        lowerCamelCase__					:   List[Any]      				=	ViltForImagesAndTextClassification(lowerCamelCase_				)
    elif "irtr" in checkpoint_url:
        lowerCamelCase__					:   Optional[Any]      				=	True
        lowerCamelCase__					:   int      				=	ViltForImageAndTextRetrieval(lowerCamelCase_				)
    elif "mlm_itm" in checkpoint_url:
        lowerCamelCase__					:   int      				=	True
        lowerCamelCase__					:   str      				=	ViltForMaskedLM(lowerCamelCase_				)
    else:
        raise ValueError('Unknown model type'				)

    # load state_dict of original model, remove and rename some keys
    lowerCamelCase__					:   Tuple      				=	torch.hub.load_state_dict_from_url(lowerCamelCase_		,			map_location='cpu'				)['''state_dict''']
    lowerCamelCase__					:   Any      				=	create_rename_keys(lowerCamelCase_		,			lowerCamelCase_		,			lowerCamelCase_		,			lowerCamelCase_				)
    for src, dest in rename_keys:
        rename_key(lowerCamelCase_		,			lowerCamelCase_		,			lowerCamelCase_				)
    read_in_q_k_v(lowerCamelCase_		,			lowerCamelCase_				)
    if mlm_model or irtr_model:
        lowerCamelCase__					:   Optional[int]      				=	['''itm_score.fc.weight''', '''itm_score.fc.bias''']
        for k in ignore_keys:
            state_dict.pop(lowerCamelCase_		,			lowerCamelCase_				)

    # load state dict into HuggingFace model
    model.eval()
    if mlm_model:
        lowerCamelCase__					:   Dict      				=	model.load_state_dict(lowerCamelCase_		,			strict=lowerCamelCase_				)
        assert missing_keys == ["mlm_score.decoder.bias"]
    else:
        model.load_state_dict(lowerCamelCase_				)

    # Define processor
    lowerCamelCase__					:   Optional[Any]      				=	ViltImageProcessor(size=384				)
    lowerCamelCase__					:   int      				=	BertTokenizer.from_pretrained('bert-base-uncased'				)
    lowerCamelCase__					:   List[Any]      				=	ViltProcessor(lowerCamelCase_		,			lowerCamelCase_				)

    # Forward pass on example inputs (image + text)
    if nlvr_model:
        lowerCamelCase__					:   List[Any]      				=	Image.open(requests.get('https://lil.nlp.cornell.edu/nlvr/exs/ex0_0.jpg'		,			stream=lowerCamelCase_				).raw				)
        lowerCamelCase__					:   Dict      				=	Image.open(requests.get('https://lil.nlp.cornell.edu/nlvr/exs/ex0_0.jpg'		,			stream=lowerCamelCase_				).raw				)
        lowerCamelCase__					:   Dict      				=	(
            '''The left image contains twice the number of dogs as the right image, and at least two dogs in total are'''
            ''' standing.'''
        )
        lowerCamelCase__					:   List[Any]      				=	processor(lowerCamelCase_		,			lowerCamelCase_		,			return_tensors='pt'				)
        lowerCamelCase__					:   str      				=	processor(lowerCamelCase_		,			lowerCamelCase_		,			return_tensors='pt'				)
        lowerCamelCase__					:   Optional[int]      				=	model(
            input_ids=encoding_a.input_ids		,			pixel_values=encoding_a.pixel_values		,			pixel_values_a=encoding_a.pixel_values		,			)
    else:
        lowerCamelCase__					:   List[str]      				=	Image.open(requests.get('http://images.cocodataset.org/val2017/000000039769.jpg'		,			stream=lowerCamelCase_				).raw				)
        if mlm_model:
            lowerCamelCase__					:   int      				=	'''a bunch of [MASK] laying on a [MASK].'''
        else:
            lowerCamelCase__					:   Dict      				=	'''How many cats are there?'''
        lowerCamelCase__					:   List[Any]      				=	processor(lowerCamelCase_		,			lowerCamelCase_		,			return_tensors='pt'				)
        lowerCamelCase__					:   Tuple      				=	model(**lowerCamelCase_				)

    # Verify outputs
    if mlm_model:
        lowerCamelCase__					:   Union[str, Any]      				=	torch.Size([1, 11, 3_0522]				)
        lowerCamelCase__					:   Dict      				=	torch.tensor([-12.5_061, -12.5_123, -12.5_174]				)
        assert outputs.logits.shape == expected_shape
        assert torch.allclose(outputs.logits[0, 0, :3]		,			lowerCamelCase_		,			atol=1e-4				)

        # verify masked token prediction equals "cats"
        lowerCamelCase__					:   Optional[int]      				=	outputs.logits[0, 4, :].argmax(-1				).item()
        assert tokenizer.decode([predicted_id]				) == "cats"
    elif vqa_model:
        lowerCamelCase__					:   Any      				=	torch.Size([1, 3129]				)
        lowerCamelCase__					:   int      				=	torch.tensor([-15.9_495, -18.1_472, -10.3_041]				)
        assert torch.allclose(outputs.logits[0, :3]		,			lowerCamelCase_		,			atol=1e-4				)
        assert outputs.logits.shape == expected_shape
        assert torch.allclose(outputs.logits[0, 0, :3]		,			lowerCamelCase_		,			atol=1e-4				)

        # verify vqa prediction equals "2"
        lowerCamelCase__					:   Dict      				=	outputs.logits.argmax(-1				).item()
        assert model.config.idalabel[predicted_idx] == "2"
    elif nlvr_model:
        lowerCamelCase__					:   Dict      				=	torch.Size([1, 2]				)
        lowerCamelCase__					:   List[str]      				=	torch.tensor([-2.8_721, 2.1_291]				)
        assert torch.allclose(outputs.logits[0, :3]		,			lowerCamelCase_		,			atol=1e-4				)
        assert outputs.logits.shape == expected_shape

    Path(lowerCamelCase_				).mkdir(exist_ok=lowerCamelCase_				)
    print(F"""Saving model and processor to {pytorch_dump_folder_path}"""				)
    model.save_pretrained(lowerCamelCase_				)
    processor.save_pretrained(lowerCamelCase_				)


if __name__ == "__main__":
    _UpperCAmelCase       : Optional[Any]			     =  argparse.ArgumentParser()
    # Required parameters
    parser.add_argument(
        """--checkpoint_url""",
        default="""https://github.com/dandelin/ViLT/releases/download/200k/vilt_200k_mlm_itm.ckpt""",
        type=str,
        help="""URL of the checkpoint you\'d like to convert.""",
    )
    parser.add_argument(
        """--pytorch_dump_folder_path""", default=None, type=str, help="""Path to the output PyTorch model directory."""
    )

    _UpperCAmelCase       : Tuple			     =  parser.parse_args()
    convert_vilt_checkpoint(args.checkpoint_url, args.pytorch_dump_folder_path)












from typing import Optional, Tuple, Union

import flax
import flax.linen as nn
import jax
import jax.numpy as jnp
from flax.core.frozen_dict import FrozenDict

from ..configuration_utils import ConfigMixin, flax_register_to_config
from ..utils import BaseOutput
from .embeddings_flax import FlaxTimestepEmbedding, FlaxTimesteps
from .modeling_flax_utils import FlaxModelMixin
from .unet_ad_blocks_flax import (
    FlaxCrossAttnDownBlockaD,
    FlaxCrossAttnUpBlockaD,
    FlaxDownBlockaD,
    FlaxUNetMidBlockaDCrossAttn,
    FlaxUpBlockaD,
)

@flax.struct.dataclass
class      lowerCAmelCase							(      __UpperCamelCase    ):
 UpperCAmelCase__									=	42


@flax_register_to_config
class      lowerCAmelCase							(      nn.Module,						__UpperCamelCase,						__UpperCamelCase    ):
 UpperCAmelCase__									=	32
 UpperCAmelCase__									=	4
 UpperCAmelCase__									=	4
 UpperCAmelCase__									=	(
     "CrossAttnDownBlock2D",
     "CrossAttnDownBlock2D",
     "CrossAttnDownBlock2D",
     "DownBlock2D",
 )
 UpperCAmelCase__									=	("UpBlock2D", "CrossAttnUpBlock2D", "CrossAttnUpBlock2D", "CrossAttnUpBlock2D")
 UpperCAmelCase__									=	False
 UpperCAmelCase__									=	(3_20, 6_40, 12_80, 12_80)
 UpperCAmelCase__									=	2
 UpperCAmelCase__									=	8
 UpperCAmelCase__									=	None
 UpperCAmelCase__									=	12_80
 UpperCAmelCase__									=	0.0
 UpperCAmelCase__									=	False
 UpperCAmelCase__									=	jnp.floataa
 UpperCAmelCase__									=	True
 UpperCAmelCase__									=	0
 UpperCAmelCase__									=	False
 def   A_			(   self						:   Tuple    ,  UpperCAmelCase						:   jax.random.KeyArray			)    ->			FrozenDict:
     # init input tensors
     lowerCamelCase__					:   int      				=	(1, self.in_channels, self.sample_size, self.sample_size)
     lowerCamelCase__					:   List[str]      				=	jnp.zeros(UpperCAmelCase    ,  dtype=jnp.floataa			)
     lowerCamelCase__					:   Tuple      				=	jnp.ones((1,)    ,  dtype=jnp.intaa			)
     lowerCamelCase__					:   Dict      				=	jnp.zeros((1, 1, self.cross_attention_dim)    ,  dtype=jnp.floataa			)

     lowerCamelCase__				,							lowerCamelCase__					:   Union[str, Any]      				=	jax.random.split(UpperCAmelCase			)
     lowerCamelCase__					:   Dict      				=	{'params': params_rng, 'dropout': dropout_rng}

     return self.init(UpperCAmelCase    ,  UpperCAmelCase    ,  UpperCAmelCase    ,  UpperCAmelCase			)["params"]
 def   A_			(   self						:   Tuple			)    ->			Optional[int]:
     lowerCamelCase__					:   Any      				=	self.block_out_channels
     lowerCamelCase__					:   int      				=	block_out_channels[0] * 4

     if self.num_attention_heads is not None:
         raise ValueError(
             'At the moment it is not possible to define the number of attention heads via `num_attention_heads` because of a naming issue as described in https://github.com/huggingface/diffusers/issues/2011#issuecomment-1547958131. Passing `num_attention_heads` will only be supported in diffusers v0.19.'			)

     # If `num_attention_heads` is not defined (which is the case for most models)
     # it will default to `attention_head_dim`. This looks weird upon first reading it and it is.
     # The reason for this behavior is to correct for incorrectly named variables that were introduced
     # when this library was created. The incorrect naming was only discovered much later in https://github.com/huggingface/diffusers/issues/2011#issuecomment-1547958131
     # Changing `attention_head_dim` to `num_attention_heads` for 40,000+ configurations is too backwards breaking
     # which is why we correct for the naming here.
     lowerCamelCase__					:   Tuple      				=	self.num_attention_heads or self.attention_head_dim

     # input
     lowerCamelCase__					:   Optional[Any]      				=	nn.Conv(
         block_out_channels[0]    ,  kernel_size=(3, 3)    ,  strides=(1, 1)    ,  padding=((1, 1), (1, 1))    ,  dtype=self.dtype    ,  )

     # time
     lowerCamelCase__					:   Optional[int]      				=	FlaxTimesteps(
         block_out_channels[0]    ,  flip_sin_to_cos=self.flip_sin_to_cos    ,  freq_shift=self.config.freq_shift			)
     lowerCamelCase__					:   int      				=	FlaxTimestepEmbedding(UpperCAmelCase    ,  dtype=self.dtype			)

     lowerCamelCase__					:   Optional[int]      				=	self.only_cross_attention
     if isinstance(UpperCAmelCase    ,  UpperCAmelCase			):
         lowerCamelCase__					:   str      				=	(only_cross_attention,) * len(self.down_block_types			)

     if isinstance(UpperCAmelCase    ,  UpperCAmelCase			):
         lowerCamelCase__					:   List[Any]      				=	(num_attention_heads,) * len(self.down_block_types			)

     # down
     lowerCamelCase__					:   List[Any]      				=	[]
     lowerCamelCase__					:   Dict      				=	block_out_channels[0]
     for i, down_block_type in enumerate(self.down_block_types			):
         lowerCamelCase__					:   Dict      				=	output_channel
         lowerCamelCase__					:   Optional[int]      				=	block_out_channels[i]
         lowerCamelCase__					:   List[Any]      				=	i == len(UpperCAmelCase			) - 1

         if down_block_type == "CrossAttnDownBlock2D":
             lowerCamelCase__					:   Tuple      				=	FlaxCrossAttnDownBlockaD(
                 in_channels=UpperCAmelCase    ,  out_channels=UpperCAmelCase    ,  dropout=self.dropout    ,  num_layers=self.layers_per_block    ,  num_attention_heads=num_attention_heads[i]    ,  add_downsample=not is_final_block    ,  use_linear_projection=self.use_linear_projection    ,  only_cross_attention=only_cross_attention[i]    ,  use_memory_efficient_attention=self.use_memory_efficient_attention    ,  dtype=self.dtype    ,  )
         else:
             lowerCamelCase__					:   str      				=	FlaxDownBlockaD(
                 in_channels=UpperCAmelCase    ,  out_channels=UpperCAmelCase    ,  dropout=self.dropout    ,  num_layers=self.layers_per_block    ,  add_downsample=not is_final_block    ,  dtype=self.dtype    ,  )

         down_blocks.append(UpperCAmelCase			)
     lowerCamelCase__					:   List[Any]      				=	down_blocks

     # mid
     lowerCamelCase__					:   Dict      				=	FlaxUNetMidBlockaDCrossAttn(
         in_channels=block_out_channels[-1]    ,  dropout=self.dropout    ,  num_attention_heads=num_attention_heads[-1]    ,  use_linear_projection=self.use_linear_projection    ,  use_memory_efficient_attention=self.use_memory_efficient_attention    ,  dtype=self.dtype    ,  )

     # up
     lowerCamelCase__					:   Any      				=	[]
     lowerCamelCase__					:   Optional[int]      				=	list(reversed(UpperCAmelCase			)			)
     lowerCamelCase__					:   Any      				=	list(reversed(UpperCAmelCase			)			)
     lowerCamelCase__					:   int      				=	list(reversed(UpperCAmelCase			)			)
     lowerCamelCase__					:   Tuple      				=	reversed_block_out_channels[0]
     for i, up_block_type in enumerate(self.up_block_types			):
         lowerCamelCase__					:   str      				=	output_channel
         lowerCamelCase__					:   int      				=	reversed_block_out_channels[i]
         lowerCamelCase__					:   int      				=	reversed_block_out_channels[min(i + 1    ,  len(UpperCAmelCase			) - 1			)]

         lowerCamelCase__					:   Optional[Any]      				=	i == len(UpperCAmelCase			) - 1

         if up_block_type == "CrossAttnUpBlock2D":
             lowerCamelCase__					:   Tuple      				=	FlaxCrossAttnUpBlockaD(
                 in_channels=UpperCAmelCase    ,  out_channels=UpperCAmelCase    ,  prev_output_channel=UpperCAmelCase    ,  num_layers=self.layers_per_block + 1    ,  num_attention_heads=reversed_num_attention_heads[i]    ,  add_upsample=not is_final_block    ,  dropout=self.dropout    ,  use_linear_projection=self.use_linear_projection    ,  only_cross_attention=only_cross_attention[i]    ,  use_memory_efficient_attention=self.use_memory_efficient_attention    ,  dtype=self.dtype    ,  )
         else:
             lowerCamelCase__					:   Optional[Any]      				=	FlaxUpBlockaD(
                 in_channels=UpperCAmelCase    ,  out_channels=UpperCAmelCase    ,  prev_output_channel=UpperCAmelCase    ,  num_layers=self.layers_per_block + 1    ,  add_upsample=not is_final_block    ,  dropout=self.dropout    ,  dtype=self.dtype    ,  )

         up_blocks.append(UpperCAmelCase			)
         lowerCamelCase__					:   Tuple      				=	output_channel
     lowerCamelCase__					:   Tuple      				=	up_blocks

     # out
     lowerCamelCase__					:   Optional[int]      				=	nn.GroupNorm(num_groups=32    ,  epsilon=1e-5			)
     lowerCamelCase__					:   Any      				=	nn.Conv(
         self.out_channels    ,  kernel_size=(3, 3)    ,  strides=(1, 1)    ,  padding=((1, 1), (1, 1))    ,  dtype=self.dtype    ,  )







 def __call__(   self						:   int    ,  UpperCAmelCase						:   Any    ,  UpperCAmelCase						:   Tuple    ,  UpperCAmelCase						:   Optional[Any]    ,  UpperCAmelCase						:   Optional[Any]=None    ,  UpperCAmelCase						:   Dict=None    ,  UpperCAmelCase						:   bool = True    ,  UpperCAmelCase						:   bool = False    ,  )    ->			Union[FlaxUNetaDConditionOutput, Tuple]:
     # 1. time
     if not isinstance(UpperCAmelCase    ,  jnp.ndarray			):
         lowerCamelCase__					:   List[str]      				=	jnp.array([timesteps]    ,  dtype=jnp.intaa			)
     elif isinstance(UpperCAmelCase    ,  jnp.ndarray			) and len(timesteps.shape			) == 0:
         lowerCamelCase__					:   List[Any]      				=	timesteps.astype(dtype=jnp.floataa			)
         lowerCamelCase__					:   Any      				=	jnp.expand_dims(UpperCAmelCase    ,  0			)

     lowerCamelCase__					:   List[str]      				=	self.time_proj(UpperCAmelCase			)
     lowerCamelCase__					:   Optional[Any]      				=	self.time_embedding(UpperCAmelCase			)

     # 2. pre-process
     lowerCamelCase__					:   Dict      				=	jnp.transpose(UpperCAmelCase    ,  (0, 2, 3, 1)			)
     lowerCamelCase__					:   Optional[Any]      				=	self.conv_in(UpperCAmelCase			)

     # 3. down
     lowerCamelCase__					:   Any      				=	(sample,)
     for down_block in self.down_blocks:
         if isinstance(UpperCAmelCase    ,  UpperCAmelCase			):
             lowerCamelCase__				,							lowerCamelCase__					:   Optional[Any]      				=	down_block(UpperCAmelCase    ,  UpperCAmelCase    ,  UpperCAmelCase    ,  deterministic=not train			)
         else:
             lowerCamelCase__				,							lowerCamelCase__					:   Any      				=	down_block(UpperCAmelCase    ,  UpperCAmelCase    ,  deterministic=not train			)
         down_block_res_samples += res_samples

     if down_block_additional_residuals is not None:
         lowerCamelCase__					:   Union[str, Any]      				=	()

         for down_block_res_sample, down_block_additional_residual in zip(
             UpperCAmelCase    ,  UpperCAmelCase			):
             down_block_res_sample += down_block_additional_residual
             new_down_block_res_samples += (down_block_res_sample,)

         lowerCamelCase__					:   str      				=	new_down_block_res_samples

     # 4. mid
     lowerCamelCase__					:   List[Any]      				=	self.mid_block(UpperCAmelCase    ,  UpperCAmelCase    ,  UpperCAmelCase    ,  deterministic=not train			)

     if mid_block_additional_residual is not None:
         sample += mid_block_additional_residual

     # 5. up
     for up_block in self.up_blocks:
         lowerCamelCase__					:   str      				=	down_block_res_samples[-(self.layers_per_block + 1) :]
         lowerCamelCase__					:   List[str]      				=	down_block_res_samples[: -(self.layers_per_block + 1)]
         if isinstance(UpperCAmelCase    ,  UpperCAmelCase			):
             lowerCamelCase__					:   List[Any]      				=	up_block(
                 UpperCAmelCase    ,  temb=UpperCAmelCase    ,  encoder_hidden_states=UpperCAmelCase    ,  res_hidden_states_tuple=UpperCAmelCase    ,  deterministic=not train    ,  )
         else:
             lowerCamelCase__					:   int      				=	up_block(UpperCAmelCase    ,  temb=UpperCAmelCase    ,  res_hidden_states_tuple=UpperCAmelCase    ,  deterministic=not train			)

        # 6. post-process
     lowerCamelCase__					:   str      				=	self.conv_norm_out(UpperCAmelCase			)
     lowerCamelCase__					:   Optional[Any]      				=	nn.silu(UpperCAmelCase			)
     lowerCamelCase__					:   Any      				=	self.conv_out(UpperCAmelCase			)
     lowerCamelCase__					:   Optional[Any]      				=	jnp.transpose(UpperCAmelCase    ,  (0, 3, 1, 2)			)

     if not return_dict:
         return (sample,)

     return FlaxUNetaDConditionOutput(sample=UpperCAmelCase			)





