





import flax.linen as nn
import jax
import jax.numpy as jnp


class        __magic_name__	(				nn.Module				):





     """simple docstring"""



     __UpperCamelCase            =  42
     __UpperCamelCase            =  jnp.floataa



     def     SCREAMING_SNAKE_CASE		(				self							:Optional[int]	):





        '''simple docstring'''




        A_       :					Optional[Any]        =      nn.Conv(
            self.out_channels      ,   kernel_size=(3, 3)      ,   strides=(1, 1)      ,   padding=((1, 1), (1, 1))      ,   dtype=self.dtype      ,   )





     def __call__(				self							:Any      ,   snake_case							:Optional[int]	):





        '''simple docstring'''




        A_   ,    A_   ,    A_   ,    A_       :					int        =      hidden_states.shape
        A_       :					Any        =      jax.image.resize(
            snake_case      ,   shape=(batch, height * 2, width * 2, channels)      ,   method="nearest"      ,   )
        A_       :					Tuple        =      self.conv(snake_case	)
        return hidden_states




class        __magic_name__	(				nn.Module				):





     """simple docstring"""



     __UpperCamelCase            =  42
     __UpperCamelCase            =  jnp.floataa



     def     SCREAMING_SNAKE_CASE		(				self							:Optional[int]	):





        '''simple docstring'''




        A_       :					Optional[Any]        =      nn.Conv(
            self.out_channels      ,   kernel_size=(3, 3)      ,   strides=(2, 2)      ,   padding=((1, 1), (1, 1))      ,   dtype=self.dtype      ,   )





     def __call__(				self							:List[str]      ,   snake_case							:Union[str, Any]	):





        '''simple docstring'''




        A_       :					int        =      self.conv(snake_case	)
        return hidden_states




class        __magic_name__	(				nn.Module				):





     """simple docstring"""



     __UpperCamelCase            =  42
     __UpperCamelCase            =  None
     __UpperCamelCase            =  0.0
     __UpperCamelCase            =  None
     __UpperCamelCase            =  jnp.floataa



     def     SCREAMING_SNAKE_CASE		(				self							:Optional[Any]	):





        '''simple docstring'''




        A_       :					str        =      self.in_channels if self.out_channels is None else self.out_channels

        A_       :					int        =      nn.GroupNorm(num_groups=32      ,   epsilon=1e-5	)
        A_       :					Tuple        =      nn.Conv(
            snake_case      ,   kernel_size=(3, 3)      ,   strides=(1, 1)      ,   padding=((1, 1), (1, 1))      ,   dtype=self.dtype      ,   )

        A_       :					Dict        =      nn.Dense(snake_case      ,   dtype=self.dtype	)

        A_       :					int        =      nn.GroupNorm(num_groups=32      ,   epsilon=1e-5	)
        A_       :					int        =      nn.Dropout(self.dropout_prob	)
        A_       :					str        =      nn.Conv(
            snake_case      ,   kernel_size=(3, 3)      ,   strides=(1, 1)      ,   padding=((1, 1), (1, 1))      ,   dtype=self.dtype      ,   )

        A_       :					Tuple        =      self.in_channels != out_channels if self.use_nin_shortcut is None else self.use_nin_shortcut

        A_       :					List[str]        =      None
        if use_nin_shortcut:
           A_       :					str        =      nn.Conv(
               snake_case      ,   kernel_size=(1, 1)      ,   strides=(1, 1)      ,   padding="VALID"      ,   dtype=self.dtype      ,   )





     def __call__(				self							:List[Any]      ,   snake_case							:Optional[Any]      ,   snake_case							:Optional[Any]      ,   snake_case							:List[Any]=True	):





        '''simple docstring'''




        A_       :					Optional[int]        =      hidden_states
        A_       :					Optional[int]        =      self.norma(snake_case	)
        A_       :					List[Any]        =      nn.swish(snake_case	)
        A_       :					Any        =      self.conva(snake_case	)

        A_       :					Optional[Any]        =      self.time_emb_proj(nn.swish(snake_case	)	)
        A_       :					Optional[int]        =      jnp.expand_dims(jnp.expand_dims(snake_case      ,   1	)      ,   1	)
        A_       :					List[str]        =      hidden_states + temb

        A_       :					str        =      self.norma(snake_case	)
        A_       :					List[str]        =      nn.swish(snake_case	)
        A_       :					Union[str, Any]        =      self.dropout(snake_case      ,   snake_case	)
        A_       :					Tuple        =      self.conva(snake_case	)

        if self.conv_shortcut is not None:
           A_       :					str        =      self.conv_shortcut(snake_case	)

        return hidden_states + residual







# Copyright 2023 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import torch

from accelerate import PartialState
from accelerate.utils.operations import broadcast, gather, gather_object, pad_across_processes, reduce

def   __snake_case    (       _lowerCAmelCase			:			Optional[int]						)			->    Dict:
   return (torch.arange(state.num_processes						) + 1.0 + (state.num_processes * state.process_index)).to(state.device						)

def   __snake_case    (       _lowerCAmelCase			:			Tuple						)			->    Optional[Any]:
   A_       :					Optional[Any]        =      create_tensor(_lowerCAmelCase						)
   A_       :					Dict        =      gather(_lowerCAmelCase						)
   assert gathered_tensor.tolist() == list(range(1					,					state.num_processes**2 + 1						)						)

def   __snake_case    (       _lowerCAmelCase			:			List[Any]						)			->    Any:
   A_       :					int        =      [state.process_index]
   A_       :					Union[str, Any]        =      gather_object(_lowerCAmelCase						)
   assert len(_lowerCAmelCase						) == state.num_processes, f"{gathered_obj}, {len(_lowerCAmelCase						)} != {state.num_processes}"
   assert gathered_obj == list(range(state.num_processes						)						), f"{gathered_obj} != {list(range(state.num_processes						)						)}"

def   __snake_case    (       _lowerCAmelCase			:			Union[str, Any]						)			->    Optional[Any]:
   A_       :					List[str]        =      create_tensor(_lowerCAmelCase						)
   A_       :					Optional[Any]        =      broadcast(_lowerCAmelCase						)
   assert broadcasted_tensor.shape == torch.Size([state.num_processes]						)
   assert broadcasted_tensor.tolist() == list(range(1					,					state.num_processes + 1						)						)

def   __snake_case    (       _lowerCAmelCase			:			Dict						)			->    str:
   # We need to pad the tensor with one more element if we are the main process
   # to ensure that we can pad
   if state.is_main_process:
      A_       :					Tuple        =      torch.arange(state.num_processes + 1						).to(state.device						)
   else:
      A_       :					List[str]        =      torch.arange(state.num_processes						).to(state.device						)
   A_       :					Any        =      pad_across_processes(_lowerCAmelCase						)
   assert padded_tensor.shape == torch.Size([state.num_processes + 1]						)
   if not state.is_main_process:
      assert padded_tensor.tolist() == list(range(0					,					state.num_processes						)						) + [0]

def   __snake_case    (       _lowerCAmelCase			:			List[Any]						)			->    Tuple:
   # For now runs on only two processes
   if state.num_processes != 2:
      return
   A_       :					str        =      create_tensor(_lowerCAmelCase						)
   A_       :					int        =      reduce(_lowerCAmelCase					,					"sum"						)
   A_       :					Optional[Any]        =      torch.tensor([4.0, 6]						).to(state.device						)
   assert torch.allclose(_lowerCAmelCase					,					_lowerCAmelCase						), f"{reduced_tensor} != {truth_tensor}"

def   __snake_case    (       _lowerCAmelCase			:			Union[str, Any]						)			->    Dict:
   # For now runs on only two processes
   if state.num_processes != 2:
      return
   A_       :					List[str]        =      create_tensor(_lowerCAmelCase						)
   A_       :					Tuple        =      reduce(_lowerCAmelCase					,					"mean"						)
   A_       :					List[str]        =      torch.tensor([2.0, 3]						).to(state.device						)
   assert torch.allclose(_lowerCAmelCase					,					_lowerCAmelCase						), f"{reduced_tensor} != {truth_tensor}"

def   __snake_case    (       _lowerCAmelCase			:			List[str]						)			->    Dict:
   # For xla_spawn (TPUs)
   main()

def   __snake_case    (       )			->    List[str]:
   A_       :					Tuple        =      PartialState()
   state.print(f"State: {state}"						)
   state.print("testing gather"						)
   test_gather(_lowerCAmelCase						)
   state.print("testing gather_object"						)
   test_gather_object(_lowerCAmelCase						)
   state.print("testing broadcast"						)
   test_broadcast(_lowerCAmelCase						)
   state.print("testing pad_across_processes"						)
   test_pad_across_processes(_lowerCAmelCase						)
   state.print("testing reduce_sum"						)
   test_reduce_sum(_lowerCAmelCase						)
   state.print("testing reduce_mean"						)
   test_reduce_mean(_lowerCAmelCase						)


if __name__ == "__main__":
 main()
