




from typing import Optional, Tuple, Union

import flax
import flax.linen as nn
import jax
import jax.numpy as jnp
from flax.core.frozen_dict import FrozenDict

from ..configuration_utils import ConfigMixin, flax_register_to_config
from ..utils import BaseOutput
from .embeddings_flax import FlaxTimestepEmbedding, FlaxTimesteps
from .modeling_flax_utils import FlaxModelMixin
from .unet_ad_blocks_flax import (
    FlaxCrossAttnDownBlockaD,
    FlaxDownBlockaD,
    FlaxUNetMidBlockaDCrossAttn,
)


@flax.struct.dataclass
class 							UpperCamelCase_       (					UpperCAmelCase__	):

    '''simple docstring'''






    UpperCAmelCase__				    = 42
    UpperCAmelCase__				    = 42


class 							UpperCamelCase_       (					nn.Module	):

    '''simple docstring'''






    UpperCAmelCase__				    = 42
    UpperCAmelCase__				    = (16, 32, 96, 256)
    UpperCAmelCase__				    = jnp.floataa


    def 			SCREAMING_SNAKE_CASE				(     self :      List[str])		->Tuple:





           '''simple docstring'''

           A__ 					=       nn.Conv(
               self.block_out_channels[0]   ,					kernel_size=(3, 3)   ,					padding=((1, 1), (1, 1))   ,					dtype=self.dtype   ,					)

           A__ 					=       []
           for i in range(len(self.block_out_channels) - 1):
                  A__ 					=       self.block_out_channels[i]
                  A__ 					=       self.block_out_channels[i + 1]
                  A__ 					=       nn.Conv(
                      UpperCAmelCase__   ,					kernel_size=(3, 3)   ,					padding=((1, 1), (1, 1))   ,					dtype=self.dtype   ,					)
                  blocks.append(UpperCAmelCase__)
                  A__ 					=       nn.Conv(
                      UpperCAmelCase__   ,					kernel_size=(3, 3)   ,					strides=(2, 2)   ,					padding=((1, 1), (1, 1))   ,					dtype=self.dtype   ,					)
                  blocks.append(UpperCAmelCase__)
           A__ 					=       blocks

           A__ 					=       nn.Conv(
               self.conditioning_embedding_channels   ,					kernel_size=(3, 3)   ,					padding=((1, 1), (1, 1))   ,					kernel_init=nn.initializers.zeros_init()   ,					bias_init=nn.initializers.zeros_init()   ,					dtype=self.dtype   ,					)





    def __call__(     self :      int   ,					UpperCAmelCase__ :      str)		->Union[str, Any]:





           '''simple docstring'''

           A__ 					=       self.conv_in(UpperCAmelCase__)
           A__ 					=       nn.silu(UpperCAmelCase__)

           for block in self.blocks:
                  A__ 					=       block(UpperCAmelCase__)
                  A__ 					=       nn.silu(UpperCAmelCase__)

           A__ 					=       self.conv_out(UpperCAmelCase__)

           return embedding


@flax_register_to_config
class 							UpperCamelCase_       (					nn.Module						,   UpperCAmelCase__						,   UpperCAmelCase__	):

    '''simple docstring'''






    UpperCAmelCase__				    = 32
    UpperCAmelCase__				    = 4
    UpperCAmelCase__				    = (
        "CrossAttnDownBlock2D",
        "CrossAttnDownBlock2D",
        "CrossAttnDownBlock2D",
        "DownBlock2D",
    )
    UpperCAmelCase__				    = False
    UpperCAmelCase__				    = (320, 640, 1280, 1280)
    UpperCAmelCase__				    = 2
    UpperCAmelCase__				    = 8
    UpperCAmelCase__				    = None
    UpperCAmelCase__				    = 1280
    UpperCAmelCase__				    = 0.0
    UpperCAmelCase__				    = False
    UpperCAmelCase__				    = jnp.floataa
    UpperCAmelCase__				    = True
    UpperCAmelCase__				    = 0
    UpperCAmelCase__				    = "rgb"
    UpperCAmelCase__				    = (16, 32, 96, 256)


    def 			SCREAMING_SNAKE_CASE				(     self :      Optional[Any]   ,					UpperCAmelCase__ :      jax.random.KeyArray)		->FrozenDict:





           '''simple docstring'''

           A__ 					=       (1, self.in_channels, self.sample_size, self.sample_size)
           A__ 					=       jnp.zeros(UpperCAmelCase__   ,					dtype=jnp.floataa)
           A__ 					=       jnp.ones((1,)   ,					dtype=jnp.intaa)
           A__ 					=       jnp.zeros((1, 1, self.cross_attention_dim)   ,					dtype=jnp.floataa)
           A__ 					=       (1, 3, self.sample_size * 8, self.sample_size * 8)
           A__ 					=       jnp.zeros(UpperCAmelCase__   ,					dtype=jnp.floataa)

           A__					,				A__ 					=       jax.random.split(UpperCAmelCase__)
           A__ 					=       {'''params''': params_rng, '''dropout''': dropout_rng}

           return self.init(UpperCAmelCase__   ,					UpperCAmelCase__   ,					UpperCAmelCase__   ,					UpperCAmelCase__   ,					UpperCAmelCase__)["params"]


    def 			SCREAMING_SNAKE_CASE				(     self :      Optional[int])		->str:





           '''simple docstring'''

           A__ 					=       self.block_out_channels
           A__ 					=       block_out_channels[0] * 4

           # If `num_attention_heads` is not defined (which is the case for most models)
           # it will default to `attention_head_dim`. This looks weird upon first reading it and it is.
           # The reason for this behavior is to correct for incorrectly named variables that were introduced
           # when this library was created. The incorrect naming was only discovered much later in https://github.com/huggingface/diffusers/issues/2011#issuecomment-1547958131
           # Changing `attention_head_dim` to `num_attention_heads` for 40,000+ configurations is too backwards breaking
           # which is why we correct for the naming here.
           A__ 					=       self.num_attention_heads or self.attention_head_dim

           # input
           A__ 					=       nn.Conv(
               block_out_channels[0]   ,					kernel_size=(3, 3)   ,					strides=(1, 1)   ,					padding=((1, 1), (1, 1))   ,					dtype=self.dtype   ,					)

           # time
           A__ 					=       FlaxTimesteps(
               block_out_channels[0]   ,					flip_sin_to_cos=self.flip_sin_to_cos   ,					freq_shift=self.config.freq_shift)
           A__ 					=       FlaxTimestepEmbedding(UpperCAmelCase__   ,					dtype=self.dtype)

           A__ 					=       FlaxControlNetConditioningEmbedding(
               conditioning_embedding_channels=block_out_channels[0]   ,					block_out_channels=self.conditioning_embedding_out_channels   ,					)

           A__ 					=       self.only_cross_attention
           if isinstance(UpperCAmelCase__   ,					UpperCAmelCase__):
                  A__ 					=       (only_cross_attention,) * len(self.down_block_types)

           if isinstance(UpperCAmelCase__   ,					UpperCAmelCase__):
                  A__ 					=       (num_attention_heads,) * len(self.down_block_types)

           # down
           A__ 					=       []
           A__ 					=       []

           A__ 					=       block_out_channels[0]

           A__ 					=       nn.Conv(
               UpperCAmelCase__   ,					kernel_size=(1, 1)   ,					padding='''VALID'''   ,					kernel_init=nn.initializers.zeros_init()   ,					bias_init=nn.initializers.zeros_init()   ,					dtype=self.dtype   ,					)
           controlnet_down_blocks.append(UpperCAmelCase__)

           for i, down_block_type in enumerate(self.down_block_types):
                  A__ 					=       output_channel
                  A__ 					=       block_out_channels[i]
                  A__ 					=       i == len(UpperCAmelCase__) - 1

                  if down_block_type == "CrossAttnDownBlock2D":
                         A__ 					=       FlaxCrossAttnDownBlockaD(
                             in_channels=UpperCAmelCase__   ,					out_channels=UpperCAmelCase__   ,					dropout=self.dropout   ,					num_layers=self.layers_per_block   ,					num_attention_heads=num_attention_heads[i]   ,					add_downsample=not is_final_block   ,					use_linear_projection=self.use_linear_projection   ,					only_cross_attention=only_cross_attention[i]   ,					dtype=self.dtype   ,					)
                  else:
                         A__ 					=       FlaxDownBlockaD(
                             in_channels=UpperCAmelCase__   ,					out_channels=UpperCAmelCase__   ,					dropout=self.dropout   ,					num_layers=self.layers_per_block   ,					add_downsample=not is_final_block   ,					dtype=self.dtype   ,					)

                  down_blocks.append(UpperCAmelCase__)

                  for _ in range(self.layers_per_block):
                         A__ 					=       nn.Conv(
                             UpperCAmelCase__   ,					kernel_size=(1, 1)   ,					padding='''VALID'''   ,					kernel_init=nn.initializers.zeros_init()   ,					bias_init=nn.initializers.zeros_init()   ,					dtype=self.dtype   ,					)
                         controlnet_down_blocks.append(UpperCAmelCase__)

                  if not is_final_block:
                         A__ 					=       nn.Conv(
                             UpperCAmelCase__   ,					kernel_size=(1, 1)   ,					padding='''VALID'''   ,					kernel_init=nn.initializers.zeros_init()   ,					bias_init=nn.initializers.zeros_init()   ,					dtype=self.dtype   ,					)
                         controlnet_down_blocks.append(UpperCAmelCase__)

           A__ 					=       down_blocks
           A__ 					=       controlnet_down_blocks

           # mid
           A__ 					=       block_out_channels[-1]
           A__ 					=       FlaxUNetMidBlockaDCrossAttn(
               in_channels=UpperCAmelCase__   ,					dropout=self.dropout   ,					num_attention_heads=num_attention_heads[-1]   ,					use_linear_projection=self.use_linear_projection   ,					dtype=self.dtype   ,					)

           A__ 					=       nn.Conv(
               UpperCAmelCase__   ,					kernel_size=(1, 1)   ,					padding='''VALID'''   ,					kernel_init=nn.initializers.zeros_init()   ,					bias_init=nn.initializers.zeros_init()   ,					dtype=self.dtype   ,					)





    def __call__(     self :      List[str]   ,					UpperCAmelCase__ :      Tuple   ,					UpperCAmelCase__ :      Optional[int]   ,					UpperCAmelCase__ :      Any   ,					UpperCAmelCase__ :      int   ,					UpperCAmelCase__ :      float = 1.0   ,					UpperCAmelCase__ :      bool = True   ,					UpperCAmelCase__ :      bool = False   ,					)		->Union[FlaxControlNetOutput, Tuple]:





           '''simple docstring'''

           A__ 					=       self.controlnet_conditioning_channel_order
           if channel_order == "bgr":
                  A__ 					=       jnp.flip(UpperCAmelCase__   ,					axis=1)

           # 1. time
           if not isinstance(UpperCAmelCase__   ,					jnp.ndarray):
                  A__ 					=       jnp.array([timesteps]   ,					dtype=jnp.intaa)
           elif isinstance(UpperCAmelCase__   ,					jnp.ndarray) and len(timesteps.shape) == 0:
                  A__ 					=       timesteps.astype(dtype=jnp.floataa)
                  A__ 					=       jnp.expand_dims(UpperCAmelCase__   ,					0)

           A__ 					=       self.time_proj(UpperCAmelCase__)
           A__ 					=       self.time_embedding(UpperCAmelCase__)

           # 2. pre-process
           A__ 					=       jnp.transpose(UpperCAmelCase__   ,					(0, 2, 3, 1))
           A__ 					=       self.conv_in(UpperCAmelCase__)

           A__ 					=       jnp.transpose(UpperCAmelCase__   ,					(0, 2, 3, 1))
           A__ 					=       self.controlnet_cond_embedding(UpperCAmelCase__)
           sample += controlnet_cond

           # 3. down
           A__ 					=       (sample,)
           for down_block in self.down_blocks:
                  if isinstance(UpperCAmelCase__   ,					UpperCAmelCase__):
                         A__					,				A__ 					=       down_block(UpperCAmelCase__   ,					UpperCAmelCase__   ,					UpperCAmelCase__   ,					deterministic=not train)
                  else:
                         A__					,				A__ 					=       down_block(UpperCAmelCase__   ,					UpperCAmelCase__   ,					deterministic=not train)
                  down_block_res_samples += res_samples

           # 4. mid
           A__ 					=       self.mid_block(UpperCAmelCase__   ,					UpperCAmelCase__   ,					UpperCAmelCase__   ,					deterministic=not train)

           # 5. contronet blocks
           A__ 					=       ()
           for down_block_res_sample, controlnet_block in zip(UpperCAmelCase__   ,					self.controlnet_down_blocks):
                  A__ 					=       controlnet_block(UpperCAmelCase__)
                  controlnet_down_block_res_samples += (down_block_res_sample,)

           A__ 					=       controlnet_down_block_res_samples

           A__ 					=       self.controlnet_mid_block(UpperCAmelCase__)

           # 6. scaling
           A__ 					=       [sample * conditioning_scale for sample in down_block_res_samples]
           mid_block_res_sample *= conditioning_scale

           if not return_dict:
                  return (down_block_res_samples, mid_block_res_sample)

           return FlaxControlNetOutput(
               down_block_res_samples=UpperCAmelCase__   ,					mid_block_res_sample=UpperCAmelCase__)



import json
import logging
import os
import sys
from pathlib import Path

import finetune_rag

from transformers.file_utils import is_apex_available
from transformers.testing_utils import (
    TestCasePlus,
    execute_subprocess_async,
    require_ray,
    require_torch_gpu,
    require_torch_multi_gpu,
)


logging.basicConfig(level=logging.DEBUG)
_SCREAMING_SNAKE_CASE 		=      logging.getLogger()

_SCREAMING_SNAKE_CASE 		=      logging.StreamHandler(sys.stdout)
logger.addHandler(stream_handler)
class SCREAMING_SNAKE_CASE_		(      __lowerCAmelCase		):







   def 						lowerCamelCase_	(     self						:						Union[str, Any]   ,   lowerCamelCase_						:						List[str]				):





    """simple docstring"""


    os.makedirs(lowerCamelCase_   ,   exist_ok=lowerCamelCase_				)
    UpperCamelCase			  =      {"""source""": """What is love ?""", """target""": """life"""}
    UpperCamelCase			  =      {"""train""": 12, """val""": 2, """test""": 2}
    for split in ["train", "test", "val"]:
     for field in ["source", "target"]:
      UpperCamelCase			  =      """\n""".join([contents[field]] * n_lines[split]				)
      with open(os.path.join(lowerCamelCase_   ,   f"""{split}.{field}"""				)   ,   """w"""				) as f:
       f.write(lowerCamelCase_				)







   def 						lowerCamelCase_	(     self						:						Tuple   ,   lowerCamelCase_						:						int   ,   lowerCamelCase_						:						str = "pytorch"				):





    """simple docstring"""


    UpperCamelCase			  =      self.get_auto_remove_tmp_dir()
    UpperCamelCase			  =      os.path.join(lowerCamelCase_   ,   """output"""				)
    UpperCamelCase			  =      os.path.join(lowerCamelCase_   ,   """data"""				)
    self._create_dummy_data(data_dir=lowerCamelCase_				)

    UpperCamelCase			  =      f"""
                --data_dir {data_dir} \
                --output_dir {output_dir} \
                --model_name_or_path facebook/rag-sequence-base \
                --model_type rag_sequence \
                --do_train \
                --do_predict \
                --n_val -1 \
                --val_check_interval 1.0 \
                --train_batch_size 2 \
                --eval_batch_size 1 \
                --max_source_length 25 \
                --max_target_length 25 \
                --val_max_target_length 25 \
                --test_max_target_length 25 \
                --label_smoothing 0.1 \
                --dropout 0.1 \
                --attention_dropout 0.1 \
                --weight_decay 0.001 \
                --adam_epsilon 1e-08 \
                --max_grad_norm 0.1 \
                --lr_scheduler polynomial \
                --learning_rate 3e-04 \
                --num_train_epochs 1 \
                --warmup_steps 4 \
                --gradient_accumulation_steps 1 \
                --distributed-port 8787 \
                --use_dummy_dataset 1 \
                --distributed_retriever {distributed_retriever} \
            """.split()

    if gpus > 0:
     testargs.append(f"""--gpus={gpus}"""				)
     if is_apex_available():
      testargs.append("""--fp16"""				)
    else:
     testargs.append("""--gpus=0"""				)
     testargs.append("""--distributed_backend=ddp_cpu"""				)
     testargs.append("""--num_processes=2"""				)

    UpperCamelCase			  =      [sys.executable, str(Path(finetune_rag.__file__				).resolve()				)] + testargs
    execute_subprocess_async(lowerCamelCase_   ,   env=self.get_env()				)

    UpperCamelCase			  =      os.path.join(lowerCamelCase_   ,   """metrics.json"""				)
    with open(lowerCamelCase_				) as f:
     UpperCamelCase			  =      json.load(lowerCamelCase_				)
    return result







   @require_torch_gpu
   def 						lowerCamelCase_	(     self						:						Tuple				):





    """simple docstring"""


    UpperCamelCase			  =      self._run_finetune(gpus=1				)
    self.assertGreaterEqual(result["""test"""][0]["""test_avg_em"""]   ,   0.2				)







   @require_torch_multi_gpu
   def 						lowerCamelCase_	(     self						:						List[Any]				):





    """simple docstring"""


    UpperCamelCase			  =      self._run_finetune(gpus=2				)
    self.assertGreaterEqual(result["""test"""][0]["""test_avg_em"""]   ,   0.2				)







   @require_torch_gpu
   @require_ray
   def 						lowerCamelCase_	(     self						:						Union[str, Any]				):





    """simple docstring"""


    UpperCamelCase			  =      self._run_finetune(gpus=1   ,   distributed_retriever="""ray"""				)
    self.assertGreaterEqual(result["""test"""][0]["""test_avg_em"""]   ,   0.2				)







   @require_torch_multi_gpu
   @require_ray
   def 						lowerCamelCase_	(     self						:						int				):





    """simple docstring"""


    UpperCamelCase			  =      self._run_finetune(gpus=1   ,   distributed_retriever="""ray"""				)
    self.assertGreaterEqual(result["""test"""][0]["""test_avg_em"""]   ,   0.2				)
