'''simple docstring'''


import functools
import math

import flax.linen as nn
import jax
import jax.numpy as jnp


def 					_lowerCamelCase    (      lowerCamelCase_					:							Union[str, Any]	,     lowerCamelCase_					:							List[str]	,     lowerCamelCase_					:							Optional[int]	,     lowerCamelCase_					:							Tuple	,     lowerCamelCase_					:							int = 4096   ):



     """simple docstring"""





     UpperCAmelCase_   ,			UpperCAmelCase_   ,			UpperCAmelCase_							:   Dict   =      key.shape[-3:]
     UpperCAmelCase_							:   Union[str, Any]   =      value.shape[-1]
     UpperCAmelCase_							:   Union[str, Any]   =      min(lowerCamelCase_	,     lowerCamelCase_   )
     UpperCAmelCase_							:   Tuple   =      query / jnp.sqrt(lowerCamelCase_   )

     @functools.partial(jax.checkpoint	,     prevent_cse=lowerCamelCase_   )
     def summarize_chunk(lowerCamelCase_					:							Optional[Any]	,     lowerCamelCase_					:							Any	,     lowerCamelCase_					:							Tuple   ):
          UpperCAmelCase_							:   Dict   =      jnp.einsum('...qhd,...khd->...qhk'	,     lowerCamelCase_	,     lowerCamelCase_	,     precision=lowerCamelCase_   )

          UpperCAmelCase_							:   Optional[Any]   =      jnp.max(lowerCamelCase_	,     axis=-1	,     keepdims=lowerCamelCase_   )
          UpperCAmelCase_							:   Any   =      jax.lax.stop_gradient(lowerCamelCase_   )
          UpperCAmelCase_							:   List[str]   =      jnp.exp(attn_weights - max_score   )

          UpperCAmelCase_							:   str   =      jnp.einsum('...vhf,...qhv->...qhf'	,     lowerCamelCase_	,     lowerCamelCase_	,     precision=lowerCamelCase_   )
          UpperCAmelCase_							:   List[str]   =      jnp.einsum('...qhk->...qh'	,     lowerCamelCase_   )

          return (exp_values, exp_weights.sum(axis=-1   ), max_score)

     def chunk_scanner(lowerCamelCase_					:							Union[str, Any]   ):
          # julienne key array
          UpperCAmelCase_							:   Any   =      jax.lax.dynamic_slice(
              operand=lowerCamelCase_	,     start_indices=[0] * (key.ndim - 3) + [chunk_idx, 0, 0]	,     slice_sizes=list(key.shape[:-3]   ) + [key_chunk_size, num_heads, k_features]	,     )

          # julienne value array
          UpperCAmelCase_							:   Union[str, Any]   =      jax.lax.dynamic_slice(
              operand=lowerCamelCase_	,     start_indices=[0] * (value.ndim - 3) + [chunk_idx, 0, 0]	,     slice_sizes=list(value.shape[:-3]   ) + [key_chunk_size, num_heads, v_features]	,     )

          return summarize_chunk(lowerCamelCase_	,     lowerCamelCase_	,     lowerCamelCase_   )

     UpperCAmelCase_   ,			UpperCAmelCase_   ,			UpperCAmelCase_							:   List[Any]   =      jax.lax.map(f=lowerCamelCase_	,     xs=jnp.arange(0	,     lowerCamelCase_	,     lowerCamelCase_   )   )

     UpperCAmelCase_							:   List[str]   =      jnp.max(lowerCamelCase_	,     axis=0	,     keepdims=lowerCamelCase_   )
     UpperCAmelCase_							:   str   =      jnp.exp(chunk_max - global_max   )

     chunk_values *= jnp.expand_dims(lowerCamelCase_	,     axis=-1   )
     chunk_weights *= max_diffs

     UpperCAmelCase_							:   List[Any]   =      chunk_values.sum(axis=0   )
     UpperCAmelCase_							:   Optional[int]   =      jnp.expand_dims(lowerCamelCase_	,     -1   ).sum(axis=0   )

     return all_values / all_weights


def 					_lowerCamelCase    (      lowerCamelCase_					:							Any	,     lowerCamelCase_					:							List[Any]	,     lowerCamelCase_					:							Any	,     lowerCamelCase_					:							Tuple=jax.lax.Precision.HIGHEST	,     lowerCamelCase_					:							int = 1024	,     lowerCamelCase_					:							int = 4096   ):



     """simple docstring"""





     UpperCAmelCase_   ,			UpperCAmelCase_   ,			UpperCAmelCase_							:   Any   =      query.shape[-3:]

     def chunk_scanner(lowerCamelCase_					:							List[str]	,     lowerCamelCase_					:							Tuple   ):
          # julienne query array
          UpperCAmelCase_							:   Union[str, Any]   =      jax.lax.dynamic_slice(
              operand=lowerCamelCase_	,     start_indices=([0] * (query.ndim - 3)) + [chunk_idx, 0, 0]	,     slice_sizes=list(query.shape[:-3]   ) + [min(lowerCamelCase_	,     lowerCamelCase_   ), num_heads, q_features]	,     )

          return (
              chunk_idx + query_chunk_size,  # unused ignore it
              _query_chunk_attention(
                  query=lowerCamelCase_	,     key=lowerCamelCase_	,     value=lowerCamelCase_	,     precision=lowerCamelCase_	,     key_chunk_size=lowerCamelCase_   ),
          )

     UpperCAmelCase_   ,			UpperCAmelCase_							:   Any   =      jax.lax.scan(
         f=lowerCamelCase_	,     init=0	,     xs=lowerCamelCase_	,     length=math.ceil(num_q / query_chunk_size   )   )

     return jnp.concatenate(lowerCamelCase_	,     axis=-3   )  # fuse the chunked result back






class      __SCREAMING_SNAKE_CASE						(       nn.Module       ):







  '''simple docstring'''






  lowerCamelCase_		:int
  lowerCamelCase_		:int						    =		8
  lowerCamelCase_		:int						    =		64
  lowerCamelCase_		:float						    =		0.0
  lowerCamelCase_		:bool						    =		False
  lowerCamelCase_		:jnp.dtype						    =		jnp.floataa


  def _UpperCamelCase     ( self						):




       '''simple docstring'''

       UpperCAmelCase_							:   int   =      self.dim_head * self.heads
       UpperCAmelCase_							:   Optional[int]   =      self.dim_head**-0.5

       # Weights were exported with old names {to_q, to_k, to_v, to_out}
       UpperCAmelCase_							:   List[str]   =      nn.Dense(snake_case_      ,		use_bias=snake_case_      ,		dtype=self.dtype      ,		name='to_q'						)
       UpperCAmelCase_							:   List[Any]   =      nn.Dense(snake_case_      ,		use_bias=snake_case_      ,		dtype=self.dtype      ,		name='to_k'						)
       UpperCAmelCase_							:   List[Any]   =      nn.Dense(snake_case_      ,		use_bias=snake_case_      ,		dtype=self.dtype      ,		name='to_v'						)

       UpperCAmelCase_							:   Optional[Any]   =      nn.Dense(self.query_dim      ,		dtype=self.dtype      ,		name='to_out_0'						)
       UpperCAmelCase_							:   List[str]   =      nn.Dropout(rate=self.dropout						)


  def _UpperCamelCase     ( self      ,		snake_case_						):




       '''simple docstring'''

       UpperCAmelCase_   ,			UpperCAmelCase_   ,			UpperCAmelCase_							:   str   =      tensor.shape
       UpperCAmelCase_							:   Optional[int]   =      self.heads
       UpperCAmelCase_							:   Tuple   =      tensor.reshape(snake_case_      ,		snake_case_      ,		snake_case_      ,		dim // head_size						)
       UpperCAmelCase_							:   List[str]   =      jnp.transpose(snake_case_      ,		(0, 2, 1, 3)						)
       UpperCAmelCase_							:   Tuple   =      tensor.reshape(batch_size * head_size      ,		snake_case_      ,		dim // head_size						)
       return tensor


  def _UpperCamelCase     ( self      ,		snake_case_						):




       '''simple docstring'''

       UpperCAmelCase_   ,			UpperCAmelCase_   ,			UpperCAmelCase_							:   Optional[Any]   =      tensor.shape
       UpperCAmelCase_							:   Union[str, Any]   =      self.heads
       UpperCAmelCase_							:   Union[str, Any]   =      tensor.reshape(batch_size // head_size      ,		snake_case_      ,		snake_case_      ,		snake_case_						)
       UpperCAmelCase_							:   Optional[Any]   =      jnp.transpose(snake_case_      ,		(0, 2, 1, 3)						)
       UpperCAmelCase_							:   Union[str, Any]   =      tensor.reshape(batch_size // head_size      ,		snake_case_      ,		dim * head_size						)
       return tensor






  def __call__( self      ,		snake_case_      ,		snake_case_=None      ,		snake_case_=True						):




       '''simple docstring'''

       UpperCAmelCase_							:   Dict   =      hidden_states if context is None else context

       UpperCAmelCase_							:   str   =      self.query(snake_case_						)
       UpperCAmelCase_							:   Optional[int]   =      self.key(snake_case_						)
       UpperCAmelCase_							:   Optional[int]   =      self.value(snake_case_						)

       UpperCAmelCase_							:   Dict   =      self.reshape_heads_to_batch_dim(snake_case_						)
       UpperCAmelCase_							:   str   =      self.reshape_heads_to_batch_dim(snake_case_						)
       UpperCAmelCase_							:   Tuple   =      self.reshape_heads_to_batch_dim(snake_case_						)

       if self.use_memory_efficient_attention:
            UpperCAmelCase_							:   Dict   =      query_states.transpose(1      ,		0      ,		2						)
            UpperCAmelCase_							:   List[str]   =      key_states.transpose(1      ,		0      ,		2						)
            UpperCAmelCase_							:   Union[str, Any]   =      value_states.transpose(1      ,		0      ,		2						)

            # this if statement create a chunk size for each layer of the unet
            # the chunk size is equal to the query_length dimension of the deepest layer of the unet

            UpperCAmelCase_							:   List[str]   =      query_states.shape[-3]
            if flatten_latent_dim % 6_4 == 0:
                 UpperCAmelCase_							:   Dict   =      int(flatten_latent_dim / 6_4						)
            elif flatten_latent_dim % 1_6 == 0:
                 UpperCAmelCase_							:   List[Any]   =      int(flatten_latent_dim / 1_6						)
            elif flatten_latent_dim % 4 == 0:
                 UpperCAmelCase_							:   int   =      int(flatten_latent_dim / 4						)
            else:
                 UpperCAmelCase_							:   List[str]   =      int(snake_case_						)

            UpperCAmelCase_							:   List[Any]   =      jax_memory_efficient_attention(
                snake_case_      ,		snake_case_      ,		snake_case_      ,		query_chunk_size=snake_case_      ,		key_chunk_size=4_0_9_6 * 4						)

            UpperCAmelCase_							:   Any   =      hidden_states.transpose(1      ,		0      ,		2						)
       else:
            # compute attentions
            UpperCAmelCase_							:   str   =      jnp.einsum('b i d, b j d->b i j'      ,		snake_case_      ,		snake_case_						)
            UpperCAmelCase_							:   List[Any]   =      attention_scores * self.scale
            UpperCAmelCase_							:   Optional[Any]   =      nn.softmax(snake_case_      ,		axis=2						)

            # attend to values
            UpperCAmelCase_							:   Optional[Any]   =      jnp.einsum('b i j, b j d -> b i d'      ,		snake_case_      ,		snake_case_						)

       UpperCAmelCase_							:   Optional[Any]   =      self.reshape_batch_dim_to_heads(snake_case_						)
       UpperCAmelCase_							:   List[Any]   =      self.proj_attn(snake_case_						)
       return self.dropout_layer(snake_case_      ,		deterministic=snake_case_						)







class      __SCREAMING_SNAKE_CASE						(       nn.Module       ):







  '''simple docstring'''






  lowerCamelCase_		:int
  lowerCamelCase_		:int
  lowerCamelCase_		:int
  lowerCamelCase_		:float						    =		0.0
  lowerCamelCase_		:bool						    =		False
  lowerCamelCase_		:jnp.dtype						    =		jnp.floataa
  lowerCamelCase_		:bool						    =		False


  def _UpperCamelCase     ( self						):




       '''simple docstring'''

       UpperCAmelCase_							:   str   =      FlaxAttention(
           self.dim      ,		self.n_heads      ,		self.d_head      ,		self.dropout      ,		self.use_memory_efficient_attention      ,		dtype=self.dtype						)
       # cross attention
       UpperCAmelCase_							:   Union[str, Any]   =      FlaxAttention(
           self.dim      ,		self.n_heads      ,		self.d_head      ,		self.dropout      ,		self.use_memory_efficient_attention      ,		dtype=self.dtype						)
       UpperCAmelCase_							:   Optional[Any]   =      FlaxFeedForward(dim=self.dim      ,		dropout=self.dropout      ,		dtype=self.dtype						)
       UpperCAmelCase_							:   Tuple   =      nn.LayerNorm(epsilon=1E-5      ,		dtype=self.dtype						)
       UpperCAmelCase_							:   int   =      nn.LayerNorm(epsilon=1E-5      ,		dtype=self.dtype						)
       UpperCAmelCase_							:   Tuple   =      nn.LayerNorm(epsilon=1E-5      ,		dtype=self.dtype						)
       UpperCAmelCase_							:   List[str]   =      nn.Dropout(rate=self.dropout						)






  def __call__( self      ,		snake_case_      ,		snake_case_      ,		snake_case_=True						):




       '''simple docstring'''

       UpperCAmelCase_							:   List[str]   =      hidden_states
       if self.only_cross_attention:
            UpperCAmelCase_							:   Optional[int]   =      self.attna(self.norma(snake_case_						)      ,		snake_case_      ,		deterministic=snake_case_						)
       else:
            UpperCAmelCase_							:   Union[str, Any]   =      self.attna(self.norma(snake_case_						)      ,		deterministic=snake_case_						)
       UpperCAmelCase_							:   Dict   =      hidden_states + residual

       # cross attention
       UpperCAmelCase_							:   Tuple   =      hidden_states
       UpperCAmelCase_							:   Union[str, Any]   =      self.attna(self.norma(snake_case_						)      ,		snake_case_      ,		deterministic=snake_case_						)
       UpperCAmelCase_							:   Optional[Any]   =      hidden_states + residual

       # feed forward
       UpperCAmelCase_							:   Dict   =      hidden_states
       UpperCAmelCase_							:   Tuple   =      self.ff(self.norma(snake_case_						)      ,		deterministic=snake_case_						)
       UpperCAmelCase_							:   Optional[Any]   =      hidden_states + residual

       return self.dropout_layer(snake_case_      ,		deterministic=snake_case_						)







class      __SCREAMING_SNAKE_CASE						(       nn.Module       ):







  '''simple docstring'''






  lowerCamelCase_		:int
  lowerCamelCase_		:int
  lowerCamelCase_		:int
  lowerCamelCase_		:int						    =		1
  lowerCamelCase_		:float						    =		0.0
  lowerCamelCase_		:bool						    =		False
  lowerCamelCase_		:bool						    =		False
  lowerCamelCase_		:jnp.dtype						    =		jnp.floataa
  lowerCamelCase_		:bool						    =		False


  def _UpperCamelCase     ( self						):




       '''simple docstring'''

       UpperCAmelCase_							:   int   =      nn.GroupNorm(num_groups=3_2      ,		epsilon=1E-5						)

       UpperCAmelCase_							:   Dict   =      self.n_heads * self.d_head
       if self.use_linear_projection:
            UpperCAmelCase_							:   List[str]   =      nn.Dense(snake_case_      ,		dtype=self.dtype						)
       else:
            UpperCAmelCase_							:   List[Any]   =      nn.Conv(
                snake_case_      ,		kernel_size=(1, 1)      ,		strides=(1, 1)      ,		padding='VALID'      ,		dtype=self.dtype      ,		)

       UpperCAmelCase_							:   int   =      [
           FlaxBasicTransformerBlock(
               snake_case_      ,		self.n_heads      ,		self.d_head      ,		dropout=self.dropout      ,		only_cross_attention=self.only_cross_attention      ,		dtype=self.dtype      ,		use_memory_efficient_attention=self.use_memory_efficient_attention      ,		)
           for _ in range(self.depth						)
       ]

       if self.use_linear_projection:
            UpperCAmelCase_							:   int   =      nn.Dense(snake_case_      ,		dtype=self.dtype						)
       else:
            UpperCAmelCase_							:   str   =      nn.Conv(
                snake_case_      ,		kernel_size=(1, 1)      ,		strides=(1, 1)      ,		padding='VALID'      ,		dtype=self.dtype      ,		)

       UpperCAmelCase_							:   Union[str, Any]   =      nn.Dropout(rate=self.dropout						)






  def __call__( self      ,		snake_case_      ,		snake_case_      ,		snake_case_=True						):




       '''simple docstring'''

       UpperCAmelCase_   ,			UpperCAmelCase_   ,			UpperCAmelCase_   ,			UpperCAmelCase_							:   int   =      hidden_states.shape
       UpperCAmelCase_							:   int   =      hidden_states
       UpperCAmelCase_							:   Tuple   =      self.norm(snake_case_						)
       if self.use_linear_projection:
            UpperCAmelCase_							:   List[str]   =      hidden_states.reshape(snake_case_      ,		height * width      ,		snake_case_						)
            UpperCAmelCase_							:   Dict   =      self.proj_in(snake_case_						)
       else:
            UpperCAmelCase_							:   Optional[Any]   =      self.proj_in(snake_case_						)
            UpperCAmelCase_							:   Optional[Any]   =      hidden_states.reshape(snake_case_      ,		height * width      ,		snake_case_						)

       for transformer_block in self.transformer_blocks:
            UpperCAmelCase_							:   List[Any]   =      transformer_block(snake_case_      ,		snake_case_      ,		deterministic=snake_case_						)

       if self.use_linear_projection:
            UpperCAmelCase_							:   int   =      self.proj_out(snake_case_						)
            UpperCAmelCase_							:   Optional[int]   =      hidden_states.reshape(snake_case_      ,		snake_case_      ,		snake_case_      ,		snake_case_						)
       else:
            UpperCAmelCase_							:   Optional[Any]   =      hidden_states.reshape(snake_case_      ,		snake_case_      ,		snake_case_      ,		snake_case_						)
            UpperCAmelCase_							:   List[str]   =      self.proj_out(snake_case_						)

       UpperCAmelCase_							:   Any   =      hidden_states + residual
       return self.dropout_layer(snake_case_      ,		deterministic=snake_case_						)







class      __SCREAMING_SNAKE_CASE						(       nn.Module       ):







  '''simple docstring'''






  lowerCamelCase_		:int
  lowerCamelCase_		:float						    =		0.0
  lowerCamelCase_		:jnp.dtype						    =		jnp.floataa


  def _UpperCamelCase     ( self						):




       '''simple docstring'''

       UpperCAmelCase_							:   Any   =      FlaxGEGLU(self.dim      ,		self.dropout      ,		self.dtype						)
       UpperCAmelCase_							:   Union[str, Any]   =      nn.Dense(self.dim      ,		dtype=self.dtype						)






  def __call__( self      ,		snake_case_      ,		snake_case_=True						):




       '''simple docstring'''

       UpperCAmelCase_							:   Optional[Any]   =      self.net_a(snake_case_      ,		deterministic=snake_case_						)
       UpperCAmelCase_							:   int   =      self.net_a(snake_case_						)
       return hidden_states







class      __SCREAMING_SNAKE_CASE						(       nn.Module       ):







  '''simple docstring'''






  lowerCamelCase_		:int
  lowerCamelCase_		:float						    =		0.0
  lowerCamelCase_		:jnp.dtype						    =		jnp.floataa


  def _UpperCamelCase     ( self						):




       '''simple docstring'''

       UpperCAmelCase_							:   List[Any]   =      self.dim * 4
       UpperCAmelCase_							:   Dict   =      nn.Dense(inner_dim * 2      ,		dtype=self.dtype						)
       UpperCAmelCase_							:   List[Any]   =      nn.Dropout(rate=self.dropout						)






  def __call__( self      ,		snake_case_      ,		snake_case_=True						):




       '''simple docstring'''

       UpperCAmelCase_							:   Optional[Any]   =      self.proj(snake_case_						)
       UpperCAmelCase_   ,			UpperCAmelCase_							:   int   =      jnp.split(snake_case_      ,		2      ,		axis=2						)
       return self.dropout_layer(hidden_linear * nn.gelu(snake_case_						)      ,		deterministic=snake_case_						)







'''simple docstring'''


def 					_lowerCamelCase    (      lowerCamelCase_					:							str   ):



     """simple docstring"""





     UpperCAmelCase_							:   str   =      str(lowerCamelCase_   ).strip()
     if not oct_string:
          raise ValueError('Empty string was passed to the function'   )
     UpperCAmelCase_							:   Tuple   =      oct_string[0] == '-'
     if is_negative:
          UpperCAmelCase_							:   Tuple   =      oct_string[1:]
     if not oct_string.isdigit() or not all(0 <= int(lowerCamelCase_   ) <= 7 for char in oct_string   ):
          raise ValueError('Non-octal value was passed to the function'   )
     UpperCAmelCase_							:   str   =      0
     for char in oct_string:
          UpperCAmelCase_							:   List[Any]   =      8 * decimal_number + int(lowerCamelCase_   )
     if is_negative:
          UpperCAmelCase_							:   List[Any]   =      -decimal_number
     return decimal_number


if __name__ == "__main__":
 from doctest import testmod

 testmod()






