






import flax.linen as nn
import jax.numpy as jnp

from .attention_flax import FlaxTransformeraDModel
from .resnet_flax import FlaxDownsampleaD, FlaxResnetBlockaD, FlaxUpsampleaD
class 		SCREAMING_SNAKE_CASE__	(  nn.Module     ):

  '''simple docstring'''







  _lowerCamelCase    =  42
  _lowerCamelCase    =  42
  _lowerCamelCase    =  0.0
  _lowerCamelCase    =  1
  _lowerCamelCase    =  1
  _lowerCamelCase    =  True
  _lowerCamelCase    =  False
  _lowerCamelCase    =  False
  _lowerCamelCase    =  False
  _lowerCamelCase    =  jnp.floataa


  def  lowerCamelCase       (      self		:			Optional[int]							)						->	List[Any]:


         """simple docstring"""
         _UpperCAmelCase							=    []
         _UpperCAmelCase							=    []

         for i in range(self.num_layers							):
                _UpperCAmelCase							=    self.in_channels if i == 0 else self.out_channels

                _UpperCAmelCase							=    FlaxResnetBlockaD(
                    in_channels=lowerCamelCase       ,				out_channels=self.out_channels       ,				dropout_prob=self.dropout       ,				dtype=self.dtype       ,				)
                resnets.append(lowerCamelCase							)

                _UpperCAmelCase							=    FlaxTransformeraDModel(
                    in_channels=self.out_channels       ,				n_heads=self.num_attention_heads       ,				d_head=self.out_channels // self.num_attention_heads       ,				depth=1       ,				use_linear_projection=self.use_linear_projection       ,				only_cross_attention=self.only_cross_attention       ,				use_memory_efficient_attention=self.use_memory_efficient_attention       ,				dtype=self.dtype       ,				)
                attentions.append(lowerCamelCase							)

         _UpperCAmelCase							=    resnets
         _UpperCAmelCase							=    attentions

         if self.add_downsample:
                _UpperCAmelCase							=    FlaxDownsampleaD(self.out_channels       ,				dtype=self.dtype							)





  def __call__(      self		:			Any       ,				lowerCamelCase		:			List[Any]       ,				lowerCamelCase		:			List[Any]       ,				lowerCamelCase		:			List[Any]       ,				lowerCamelCase		:			str=True							)						->	List[str]:


         """simple docstring"""
         _UpperCAmelCase							=    ()

         for resnet, attn in zip(self.resnets       ,				self.attentions							):
                _UpperCAmelCase							=    resnet(lowerCamelCase       ,				lowerCamelCase       ,				deterministic=lowerCamelCase							)
                _UpperCAmelCase							=    attn(lowerCamelCase       ,				lowerCamelCase       ,				deterministic=lowerCamelCase							)
                output_states += (hidden_states,)

         if self.add_downsample:
                _UpperCAmelCase							=    self.downsamplers_a(lowerCamelCase							)
                output_states += (hidden_states,)

         return hidden_states, output_states



class 		SCREAMING_SNAKE_CASE__	(  nn.Module     ):

  '''simple docstring'''







  _lowerCamelCase    =  42
  _lowerCamelCase    =  42
  _lowerCamelCase    =  0.0
  _lowerCamelCase    =  1
  _lowerCamelCase    =  True
  _lowerCamelCase    =  jnp.floataa


  def  lowerCamelCase       (      self		:			Optional[Any]							)						->	Union[str, Any]:


         """simple docstring"""
         _UpperCAmelCase							=    []

         for i in range(self.num_layers							):
                _UpperCAmelCase							=    self.in_channels if i == 0 else self.out_channels

                _UpperCAmelCase							=    FlaxResnetBlockaD(
                    in_channels=lowerCamelCase       ,				out_channels=self.out_channels       ,				dropout_prob=self.dropout       ,				dtype=self.dtype       ,				)
                resnets.append(lowerCamelCase							)
         _UpperCAmelCase							=    resnets

         if self.add_downsample:
                _UpperCAmelCase							=    FlaxDownsampleaD(self.out_channels       ,				dtype=self.dtype							)





  def __call__(      self		:			List[Any]       ,				lowerCamelCase		:			Optional[int]       ,				lowerCamelCase		:			Any       ,				lowerCamelCase		:			str=True							)						->	List[Any]:


         """simple docstring"""
         _UpperCAmelCase							=    ()

         for resnet in self.resnets:
                _UpperCAmelCase							=    resnet(lowerCamelCase       ,				lowerCamelCase       ,				deterministic=lowerCamelCase							)
                output_states += (hidden_states,)

         if self.add_downsample:
                _UpperCAmelCase							=    self.downsamplers_a(lowerCamelCase							)
                output_states += (hidden_states,)

         return hidden_states, output_states



class 		SCREAMING_SNAKE_CASE__	(  nn.Module     ):

  '''simple docstring'''







  _lowerCamelCase    =  42
  _lowerCamelCase    =  42
  _lowerCamelCase    =  42
  _lowerCamelCase    =  0.0
  _lowerCamelCase    =  1
  _lowerCamelCase    =  1
  _lowerCamelCase    =  True
  _lowerCamelCase    =  False
  _lowerCamelCase    =  False
  _lowerCamelCase    =  False
  _lowerCamelCase    =  jnp.floataa


  def  lowerCamelCase       (      self		:			Any							)						->	Union[str, Any]:


         """simple docstring"""
         _UpperCAmelCase							=    []
         _UpperCAmelCase							=    []

         for i in range(self.num_layers							):
                _UpperCAmelCase							=    self.in_channels if (i == self.num_layers - 1) else self.out_channels
                _UpperCAmelCase							=    self.prev_output_channel if i == 0 else self.out_channels

                _UpperCAmelCase							=    FlaxResnetBlockaD(
                    in_channels=resnet_in_channels + res_skip_channels       ,				out_channels=self.out_channels       ,				dropout_prob=self.dropout       ,				dtype=self.dtype       ,				)
                resnets.append(lowerCamelCase							)

                _UpperCAmelCase							=    FlaxTransformeraDModel(
                    in_channels=self.out_channels       ,				n_heads=self.num_attention_heads       ,				d_head=self.out_channels // self.num_attention_heads       ,				depth=1       ,				use_linear_projection=self.use_linear_projection       ,				only_cross_attention=self.only_cross_attention       ,				use_memory_efficient_attention=self.use_memory_efficient_attention       ,				dtype=self.dtype       ,				)
                attentions.append(lowerCamelCase							)

         _UpperCAmelCase							=    resnets
         _UpperCAmelCase							=    attentions

         if self.add_upsample:
                _UpperCAmelCase							=    FlaxUpsampleaD(self.out_channels       ,				dtype=self.dtype							)





  def __call__(      self		:			Any       ,				lowerCamelCase		:			Tuple       ,				lowerCamelCase		:			List[Any]       ,				lowerCamelCase		:			Any       ,				lowerCamelCase		:			List[str]       ,				lowerCamelCase		:			Union[str, Any]=True							)						->	str:


         """simple docstring"""
         for resnet, attn in zip(self.resnets       ,				self.attentions							):
                # pop res hidden states
                _UpperCAmelCase							=    res_hidden_states_tuple[-1]
                _UpperCAmelCase							=    res_hidden_states_tuple[:-1]
                _UpperCAmelCase							=    jnp.concatenate((hidden_states, res_hidden_states)       ,				axis=-1							)

                _UpperCAmelCase							=    resnet(lowerCamelCase       ,				lowerCamelCase       ,				deterministic=lowerCamelCase							)
                _UpperCAmelCase							=    attn(lowerCamelCase       ,				lowerCamelCase       ,				deterministic=lowerCamelCase							)

         if self.add_upsample:
                _UpperCAmelCase							=    self.upsamplers_a(lowerCamelCase							)

         return hidden_states



class 		SCREAMING_SNAKE_CASE__	(  nn.Module     ):

  '''simple docstring'''







  _lowerCamelCase    =  42
  _lowerCamelCase    =  42
  _lowerCamelCase    =  42
  _lowerCamelCase    =  0.0
  _lowerCamelCase    =  1
  _lowerCamelCase    =  True
  _lowerCamelCase    =  jnp.floataa


  def  lowerCamelCase       (      self		:			Any							)						->	Optional[int]:


         """simple docstring"""
         _UpperCAmelCase							=    []

         for i in range(self.num_layers							):
                _UpperCAmelCase							=    self.in_channels if (i == self.num_layers - 1) else self.out_channels
                _UpperCAmelCase							=    self.prev_output_channel if i == 0 else self.out_channels

                _UpperCAmelCase							=    FlaxResnetBlockaD(
                    in_channels=resnet_in_channels + res_skip_channels       ,				out_channels=self.out_channels       ,				dropout_prob=self.dropout       ,				dtype=self.dtype       ,				)
                resnets.append(lowerCamelCase							)

         _UpperCAmelCase							=    resnets

         if self.add_upsample:
                _UpperCAmelCase							=    FlaxUpsampleaD(self.out_channels       ,				dtype=self.dtype							)





  def __call__(      self		:			int       ,				lowerCamelCase		:			Dict       ,				lowerCamelCase		:			Union[str, Any]       ,				lowerCamelCase		:			List[str]       ,				lowerCamelCase		:			str=True							)						->	Optional[Any]:


         """simple docstring"""
         for resnet in self.resnets:
                # pop res hidden states
                _UpperCAmelCase							=    res_hidden_states_tuple[-1]
                _UpperCAmelCase							=    res_hidden_states_tuple[:-1]
                _UpperCAmelCase							=    jnp.concatenate((hidden_states, res_hidden_states)       ,				axis=-1							)

                _UpperCAmelCase							=    resnet(lowerCamelCase       ,				lowerCamelCase       ,				deterministic=lowerCamelCase							)

         if self.add_upsample:
                _UpperCAmelCase							=    self.upsamplers_a(lowerCamelCase							)

         return hidden_states



class 		SCREAMING_SNAKE_CASE__	(  nn.Module     ):

  '''simple docstring'''







  _lowerCamelCase    =  42
  _lowerCamelCase    =  0.0
  _lowerCamelCase    =  1
  _lowerCamelCase    =  1
  _lowerCamelCase    =  False
  _lowerCamelCase    =  False
  _lowerCamelCase    =  jnp.floataa


  def  lowerCamelCase       (      self		:			int							)						->	Dict:


         """simple docstring"""
         # there is always at least one resnet
         _UpperCAmelCase							=    [
             FlaxResnetBlockaD(
                 in_channels=self.in_channels       ,				out_channels=self.in_channels       ,				dropout_prob=self.dropout       ,				dtype=self.dtype       ,				)
         ]

         _UpperCAmelCase							=    []

         for _ in range(self.num_layers							):
                _UpperCAmelCase							=    FlaxTransformeraDModel(
                    in_channels=self.in_channels       ,				n_heads=self.num_attention_heads       ,				d_head=self.in_channels // self.num_attention_heads       ,				depth=1       ,				use_linear_projection=self.use_linear_projection       ,				use_memory_efficient_attention=self.use_memory_efficient_attention       ,				dtype=self.dtype       ,				)
                attentions.append(lowerCamelCase							)

                _UpperCAmelCase							=    FlaxResnetBlockaD(
                    in_channels=self.in_channels       ,				out_channels=self.in_channels       ,				dropout_prob=self.dropout       ,				dtype=self.dtype       ,				)
                resnets.append(lowerCamelCase							)

         _UpperCAmelCase							=    resnets
         _UpperCAmelCase							=    attentions





  def __call__(      self		:			Dict       ,				lowerCamelCase		:			Tuple       ,				lowerCamelCase		:			List[Any]       ,				lowerCamelCase		:			Optional[Any]       ,				lowerCamelCase		:			List[Any]=True							)						->	str:


         """simple docstring"""
         _UpperCAmelCase							=    self.resnets[0](lowerCamelCase       ,				lowerCamelCase							)
         for attn, resnet in zip(self.attentions       ,				self.resnets[1:]							):
                _UpperCAmelCase							=    attn(lowerCamelCase       ,				lowerCamelCase       ,				deterministic=lowerCamelCase							)
                _UpperCAmelCase							=    resnet(lowerCamelCase       ,				lowerCamelCase       ,				deterministic=lowerCamelCase							)

         return hidden_states







import gc
import threading
import time

import psutil
import torch
class 		SCREAMING_SNAKE_CASE__	:

  '''simple docstring'''







  def __init__(      self		:			Union[str, Any]							)						->	str:


         """simple docstring"""
         _UpperCAmelCase							=    psutil.Process()
         _UpperCAmelCase							=    False


  def  lowerCamelCase       (      self		:			List[Any]							)						->	int:


         """simple docstring"""
         _UpperCAmelCase							=    -1

         while True:
                _UpperCAmelCase							=    max(self.process.memory_info().rss       ,				self.cpu_memory_peak							)

                # can't sleep or will not catch the peak right (this comment is here on purpose)
                if not self.peak_monitoring:
                       break


  def  lowerCamelCase       (      self		:			Optional[Any]							)						->	List[Any]:


         """simple docstring"""
         _UpperCAmelCase							=    True
         _UpperCAmelCase							=    threading.Thread(target=self.peak_monitor							)
         _UpperCAmelCase							=    True
         self.thread.start()





  def  lowerCamelCase       (      self		:			Optional[int]							)						->	List[str]:


         """simple docstring"""
         _UpperCAmelCase							=    False
         self.thread.join()
         return self.cpu_memory_peak



__a:     Union[str, Any]   					=					PeakCPUMemory()




def       _SCREAMING_SNAKE_CASE (							)				->						List[Any]:
       # Time
       _UpperCAmelCase							=    {"""time""": time.time()}

       gc.collect()
       torch.cuda.empty_cache()

       # CPU mem
       _UpperCAmelCase							=    psutil.Process().memory_info().rss
       cpu_peak_tracker.start()

       # GPU mem
       for i in range(torch.cuda.device_count() ):
              _UpperCAmelCase							=    torch.cuda.memory_allocated(__snake_case )
       torch.cuda.reset_peak_memory_stats()

       return measures




def       _SCREAMING_SNAKE_CASE (							__snake_case )				->						Union[str, Any]:
       # Time
       _UpperCAmelCase							=    {"""time""": time.time() - start_measures["""time"""]}

       gc.collect()
       torch.cuda.empty_cache()

       # CPU mem
       _UpperCAmelCase							=    (psutil.Process().memory_info().rss - start_measures["""cpu"""]) / 2**2_0
       _UpperCAmelCase							=    (cpu_peak_tracker.stop() - start_measures["""cpu"""]) / 2**2_0

       # GPU mem
       for i in range(torch.cuda.device_count() ):
              _UpperCAmelCase							=    (torch.cuda.memory_allocated(__snake_case ) - start_measures[str(__snake_case )]) / 2**2_0
              _UpperCAmelCase							=    (torch.cuda.max_memory_allocated(__snake_case ) - start_measures[str(__snake_case )]) / 2**2_0

       return measures





def       _SCREAMING_SNAKE_CASE (							__snake_case   ,      __snake_case )				->						str:
       print(f"""{description}:""" )
       print(f"""- Time: {measures["time"]:.2f}s""" )
       for i in range(torch.cuda.device_count() ):
              print(f"""- GPU {i} allocated: {measures[str(__snake_case )]:.2f}MiB""" )
              _UpperCAmelCase							=    measures[f"""{i}-peak"""]
              print(f"""- GPU {i} peak: {peak:.2f}MiB""" )
       print(f"""- CPU RAM allocated: {measures["cpu"]:.2f}MiB""" )
       print(f"""- CPU RAM peak: {measures["cpu-peak"]:.2f}MiB""" )