
from dataclasses import dataclass
from typing import Optional

import numpy as np
import torch
import torch.nn as nn

from ..utils import BaseOutput, is_torch_version, randn_tensor
from .attention_processor import SpatialNorm
from .unet_ad_blocks import UNetMidBlockaD, get_down_block, get_up_block


@dataclass
class 						__lowerCAmelCase	(	_lowercase      ):



      """simple docstring"""







      A__	:			torch.FloatTensor






class 						__lowerCAmelCase	(	nn.Module      ):



      """simple docstring"""







      def __init__(      self		:       List[str]				,       _snake_case		:       Dict=3				,       _snake_case		:       int=3				,       _snake_case		:       str=("DownEncoderBlock2D",)				,       _snake_case		:       Dict=(64,)				,       _snake_case		:       str=2				,       _snake_case		:       Union[str, Any]=32				,       _snake_case		:       Optional[int]="silu"				,       _snake_case		:       str=True				,       ):
          super().__init__()
          __lowercase		:			Optional[Any]	    =       layers_per_block

          __lowercase		:			int	    =       torch.nn.Convad(
              A_				,       block_out_channels[0]				,       kernel_size=3				,       stride=1				,       padding=1				,       )

          __lowercase		:			int	    =       None
          __lowercase		:			Optional[Any]	    =       nn.ModuleList([]				)

          # down
          __lowercase		:			Any	    =       block_out_channels[0]
          for i, down_block_type in enumerate(A_				):
              __lowercase		:			int	    =       output_channel
              __lowercase		:			Any	    =       block_out_channels[i]
              __lowercase		:			List[str]	    =       i == len(A_				) - 1

              __lowercase		:			Tuple	    =       get_down_block(
                  A_				,       num_layers=self.layers_per_block				,       in_channels=A_				,       out_channels=A_				,       add_downsample=not is_final_block				,       resnet_eps=1E-6				,       downsample_padding=0				,       resnet_act_fn=A_				,       resnet_groups=A_				,       attention_head_dim=A_				,       temb_channels=A_				,       )
              self.down_blocks.append(A_				)

          # mid
          __lowercase		:			Optional[int]	    =       UNetMidBlockaD(
              in_channels=block_out_channels[-1]				,       resnet_eps=1E-6				,       resnet_act_fn=A_				,       output_scale_factor=1				,       resnet_time_scale_shift='''default'''				,       attention_head_dim=block_out_channels[-1]				,       resnet_groups=A_				,       temb_channels=A_				,       )

          # out
          __lowercase		:			Optional[Any]	    =       nn.GroupNorm(num_channels=block_out_channels[-1]				,       num_groups=A_				,       eps=1E-6				)
          __lowercase		:			Any	    =       nn.SiLU()

          __lowercase		:			Optional[Any]	    =       2 * out_channels if double_z else out_channels
          __lowercase		:			List[Any]	    =       nn.Convad(block_out_channels[-1]				,       A_				,       3				,       padding=1				)

          __lowercase		:			List[str]	    =       False




      def 		snake_case_			(      self		:       Union[str, Any]				,       _snake_case		:       Optional[int]				):
          __lowercase		:			Optional[Any]	    =       x
          __lowercase		:			str	    =       self.conv_in(A_				)

          if self.training and self.gradient_checkpointing:

              def create_custom_forward(_snake_case		:       Dict				):
                  def custom_forward(*_snake_case		:       Tuple				):
                      return module(*A_				)

                  return custom_forward

              # down
              if is_torch_version('''>='''				,       '''1.11.0'''				):
                  for down_block in self.down_blocks:
                      __lowercase		:			Optional[int]	    =       torch.utils.checkpoint.checkpoint(
                          create_custom_forward(A_				)				,       A_				,       use_reentrant=A_				)
                  # middle
                  __lowercase		:			List[Any]	    =       torch.utils.checkpoint.checkpoint(
                      create_custom_forward(self.mid_block				)				,       A_				,       use_reentrant=A_				)
              else:
                  for down_block in self.down_blocks:
                      __lowercase		:			List[str]	    =       torch.utils.checkpoint.checkpoint(create_custom_forward(A_				)				,       A_				)
                  # middle
                  __lowercase		:			Dict	    =       torch.utils.checkpoint.checkpoint(create_custom_forward(self.mid_block				)				,       A_				)

          else:
              # down
              for down_block in self.down_blocks:
                  __lowercase		:			Any	    =       down_block(A_				)

              # middle
              __lowercase		:			Any	    =       self.mid_block(A_				)

          # post-process
          __lowercase		:			Any	    =       self.conv_norm_out(A_				)
          __lowercase		:			str	    =       self.conv_act(A_				)
          __lowercase		:			Dict	    =       self.conv_out(A_				)

          return sample






class 						__lowerCAmelCase	(	nn.Module      ):



      """simple docstring"""







      def __init__(      self		:       List[Any]				,       _snake_case		:       Optional[Any]=3				,       _snake_case		:       Optional[int]=3				,       _snake_case		:       str=("UpDecoderBlock2D",)				,       _snake_case		:       Any=(64,)				,       _snake_case		:       Optional[int]=2				,       _snake_case		:       Optional[int]=32				,       _snake_case		:       Tuple="silu"				,       _snake_case		:       Optional[int]="group"				,       ):
          super().__init__()
          __lowercase		:			Optional[int]	    =       layers_per_block

          __lowercase		:			List[str]	    =       nn.Convad(
              A_				,       block_out_channels[-1]				,       kernel_size=3				,       stride=1				,       padding=1				,       )

          __lowercase		:			int	    =       None
          __lowercase		:			int	    =       nn.ModuleList([]				)

          __lowercase		:			Optional[int]	    =       in_channels if norm_type == '''spatial''' else None

          # mid
          __lowercase		:			Dict	    =       UNetMidBlockaD(
              in_channels=block_out_channels[-1]				,       resnet_eps=1E-6				,       resnet_act_fn=A_				,       output_scale_factor=1				,       resnet_time_scale_shift='''default''' if norm_type == '''group''' else norm_type				,       attention_head_dim=block_out_channels[-1]				,       resnet_groups=A_				,       temb_channels=A_				,       )

          # up
          __lowercase		:			List[str]	    =       list(reversed(A_				)				)
          __lowercase		:			Any	    =       reversed_block_out_channels[0]
          for i, up_block_type in enumerate(A_				):
              __lowercase		:			List[Any]	    =       output_channel
              __lowercase		:			Any	    =       reversed_block_out_channels[i]

              __lowercase		:			Union[str, Any]	    =       i == len(A_				) - 1

              __lowercase		:			Union[str, Any]	    =       get_up_block(
                  A_				,       num_layers=self.layers_per_block + 1				,       in_channels=A_				,       out_channels=A_				,       prev_output_channel=A_				,       add_upsample=not is_final_block				,       resnet_eps=1E-6				,       resnet_act_fn=A_				,       resnet_groups=A_				,       attention_head_dim=A_				,       temb_channels=A_				,       resnet_time_scale_shift=A_				,       )
              self.up_blocks.append(A_				)
              __lowercase		:			List[Any]	    =       output_channel

          # out
          if norm_type == "spatial":
              __lowercase		:			int	    =       SpatialNorm(block_out_channels[0]				,       A_				)
          else:
              __lowercase		:			str	    =       nn.GroupNorm(num_channels=block_out_channels[0]				,       num_groups=A_				,       eps=1E-6				)
          __lowercase		:			Optional[int]	    =       nn.SiLU()
          __lowercase		:			List[str]	    =       nn.Convad(block_out_channels[0]				,       A_				,       3				,       padding=1				)

          __lowercase		:			List[Any]	    =       False




      def 		snake_case_			(      self		:       Union[str, Any]				,       _snake_case		:       str				,       _snake_case		:       Union[str, Any]=None				):
          __lowercase		:			int	    =       z
          __lowercase		:			List[str]	    =       self.conv_in(A_				)

          __lowercase		:			Tuple	    =       next(iter(self.up_blocks.parameters()				)				).dtype
          if self.training and self.gradient_checkpointing:

              def create_custom_forward(_snake_case		:       List[Any]				):
                  def custom_forward(*_snake_case		:       Tuple				):
                      return module(*A_				)

                  return custom_forward

              if is_torch_version('''>='''				,       '''1.11.0'''				):
                  # middle
                  __lowercase		:			List[Any]	    =       torch.utils.checkpoint.checkpoint(
                      create_custom_forward(self.mid_block				)				,       A_				,       A_				,       use_reentrant=A_				)
                  __lowercase		:			Union[str, Any]	    =       sample.to(A_				)

                  # up
                  for up_block in self.up_blocks:
                      __lowercase		:			str	    =       torch.utils.checkpoint.checkpoint(
                          create_custom_forward(A_				)				,       A_				,       A_				,       use_reentrant=A_				)
              else:
                  # middle
                  __lowercase		:			Union[str, Any]	    =       torch.utils.checkpoint.checkpoint(
                      create_custom_forward(self.mid_block				)				,       A_				,       A_				)
                  __lowercase		:			Tuple	    =       sample.to(A_				)

                  # up
                  for up_block in self.up_blocks:
                      __lowercase		:			List[str]	    =       torch.utils.checkpoint.checkpoint(create_custom_forward(A_				)				,       A_				,       A_				)
          else:
              # middle
              __lowercase		:			int	    =       self.mid_block(A_				,       A_				)
              __lowercase		:			Optional[int]	    =       sample.to(A_				)

              # up
              for up_block in self.up_blocks:
                  __lowercase		:			int	    =       up_block(A_				,       A_				)

        # post-process
          if latent_embeds is None:
              __lowercase		:			Dict	    =       self.conv_norm_out(A_				)
          else:
              __lowercase		:			Dict	    =       self.conv_norm_out(A_				,       A_				)
          __lowercase		:			Optional[Any]	    =       self.conv_act(A_				)
          __lowercase		:			Union[str, Any]	    =       self.conv_out(A_				)

          return sample






class 						__lowerCAmelCase	(	nn.Module      ):



      """simple docstring"""







      def __init__(      self		:       Optional[int]				,       _snake_case		:       Optional[int]				,       _snake_case		:       Any				,       _snake_case		:       str				,       _snake_case		:       Dict=None				,       _snake_case		:       List[Any]="random"				,       _snake_case		:       Optional[int]=False				,       _snake_case		:       str=True				):
          super().__init__()
          __lowercase		:			Tuple	    =       n_e
          __lowercase		:			Tuple	    =       vq_embed_dim
          __lowercase		:			Tuple	    =       beta
          __lowercase		:			List[str]	    =       legacy

          __lowercase		:			Union[str, Any]	    =       nn.Embedding(self.n_e				,       self.vq_embed_dim				)
          self.embedding.weight.data.uniform_(-1.0 / self.n_e				,       1.0 / self.n_e				)

          __lowercase		:			Union[str, Any]	    =       remap
          if self.remap is not None:
              self.register_buffer('''used'''				,       torch.tensor(np.load(self.remap				)				)				)
              __lowercase		:			List[Any]	    =       self.used.shape[0]
              __lowercase		:			Optional[Any]	    =       unknown_index  # "random" or "extra" or integer
              if self.unknown_index == "extra":
                  __lowercase		:			List[str]	    =       self.re_embed
                  __lowercase		:			List[Any]	    =       self.re_embed + 1
              print(
                  F'Remapping {self.n_e} indices to {self.re_embed} indices. '
                  F'Using {self.unknown_index} for unknown indices.'				)
          else:
              __lowercase		:			Tuple	    =       n_e

          __lowercase		:			List[str]	    =       sane_index_shape




      def 		snake_case_			(      self		:       List[str]				,       _snake_case		:       Optional[int]				):
          __lowercase		:			List[Any]	    =       inds.shape
          assert len(A_				) > 1
          __lowercase		:			Union[str, Any]	    =       inds.reshape(ishape[0]				,       -1				)
          __lowercase		:			Any	    =       self.used.to(A_				)
          __lowercase		:			List[Any]	    =       (inds[:, :, None] == used[None, None, ...]).long()
          __lowercase		:			Tuple	    =       match.argmax(-1				)
          __lowercase		:			Optional[int]	    =       match.sum(2				) < 1
          if self.unknown_index == "random":
              __lowercase		:			Optional[int]	    =       torch.randint(0				,       self.re_embed				,       size=new[unknown].shape				).to(device=new.device				)
          else:
              __lowercase		:			Any	    =       self.unknown_index
          return new.reshape(A_				)




      def 		snake_case_			(      self		:       List[str]				,       _snake_case		:       Optional[Any]				):
          __lowercase		:			Union[str, Any]	    =       inds.shape
          assert len(A_				) > 1
          __lowercase		:			Dict	    =       inds.reshape(ishape[0]				,       -1				)
          __lowercase		:			Optional[Any]	    =       self.used.to(A_				)
          if self.re_embed > self.used.shape[0]:  # extra token
              __lowercase		:			Any	    =       0  # simply set to zero
          __lowercase		:			List[str]	    =       torch.gather(used[None, :][inds.shape[0] * [0], :]				,       1				,       A_				)
          return back.reshape(A_				)




      def 		snake_case_			(      self		:       Optional[int]				,       _snake_case		:       List[Any]				):
          # reshape z -> (batch, height, width, channel) and flatten
          __lowercase		:			Optional[int]	    =       z.permute(0				,       2				,       3				,       1				).contiguous()
          __lowercase		:			Union[str, Any]	    =       z.view(-1				,       self.vq_embed_dim				)

          # distances from z to embeddings e_j (z - e)^2 = z^2 + e^2 - 2 e * z
          __lowercase		:			Tuple	    =       torch.argmin(torch.cdist(A_				,       self.embedding.weight				)				,       dim=1				)

          __lowercase		:			List[str]	    =       self.embedding(A_				).view(z.shape				)
          __lowercase		:			List[str]	    =       None
          __lowercase		:			Optional[Any]	    =       None

          # compute loss for embedding
          if not self.legacy:
              __lowercase		:			List[str]	    =       self.beta * torch.mean((z_q.detach() - z) ** 2				) + torch.mean((z_q - z.detach()) ** 2				)
          else:
              __lowercase		:			Optional[int]	    =       torch.mean((z_q.detach() - z) ** 2				) + self.beta * torch.mean((z_q - z.detach()) ** 2				)

          # preserve gradients
          __lowercase		:			int	    =       z + (z_q - z).detach()

          # reshape back to match original input shape
          __lowercase		:			Tuple	    =       z_q.permute(0				,       3				,       1				,       2				).contiguous()

          if self.remap is not None:
              __lowercase		:			Tuple	    =       min_encoding_indices.reshape(z.shape[0]				,       -1				)  # add batch axis
              __lowercase		:			Optional[int]	    =       self.remap_to_used(A_				)
              __lowercase		:			Any	    =       min_encoding_indices.reshape(-1				,       1				)  # flatten

          if self.sane_index_shape:
              __lowercase		:			Tuple	    =       min_encoding_indices.reshape(z_q.shape[0]				,       z_q.shape[2]				,       z_q.shape[3]				)

          return z_q, loss, (perplexity, min_encodings, min_encoding_indices)




      def 		snake_case_			(      self		:       Dict				,       _snake_case		:       Dict				,       _snake_case		:       str				):
          # shape specifying (batch, height, width, channel)
          if self.remap is not None:
              __lowercase		:			str	    =       indices.reshape(shape[0]				,       -1				)  # add batch axis
              __lowercase		:			Any	    =       self.unmap_to_all(A_				)
              __lowercase		:			Dict	    =       indices.reshape(-1				)  # flatten again

          # get quantized latent vectors
          __lowercase		:			Any	    =       self.embedding(A_				)

          if shape is not None:
              __lowercase		:			Union[str, Any]	    =       z_q.view(A_				)
              # reshape back to match original input shape
              __lowercase		:			int	    =       z_q.permute(0				,       3				,       1				,       2				).contiguous()

          return z_q






class 						__lowerCAmelCase	(	_lowercase      ):



      """simple docstring"""







      def __init__(      self		:       str				,       _snake_case		:       Tuple				,       _snake_case		:       Dict=False				):
          __lowercase		:			Optional[Any]	    =       parameters
          __lowercase    ,      __lowercase		:			Optional[Any]	    =       torch.chunk(A_				,       2				,       dim=1				)
          __lowercase		:			List[str]	    =       torch.clamp(self.logvar				,       -30.0				,       20.0				)
          __lowercase		:			str	    =       deterministic
          __lowercase		:			Any	    =       torch.exp(0.5 * self.logvar				)
          __lowercase		:			Optional[Any]	    =       torch.exp(self.logvar				)
          if self.deterministic:
              __lowercase		:			Optional[Any]	    =       torch.zeros_like(
                  self.mean				,       device=self.parameters.device				,       dtype=self.parameters.dtype				)




      def 		snake_case_			(      self		:       Any				,       _snake_case		:       Optional[torch.Generator] = None				):
          # make sure sample is on the same device as the parameters and has same dtype
          __lowercase		:			List[Any]	    =       randn_tensor(
              self.mean.shape				,       generator=A_				,       device=self.parameters.device				,       dtype=self.parameters.dtype				)
          __lowercase		:			List[Any]	    =       self.mean + self.std * sample
          return x




      def 		snake_case_			(      self		:       Any				,       _snake_case		:       Tuple=None				):
          if self.deterministic:
              return torch.Tensor([0.0]				)
          else:
              if other is None:
                  return 0.5 * torch.sum(torch.pow(self.mean				,       2				) + self.var - 1.0 - self.logvar				,       dim=[1, 2, 3]				)
              else:
                  return 0.5 * torch.sum(
                      torch.pow(self.mean - other.mean				,       2				) / other.var
                      + self.var / other.var
                      - 1.0
                      - self.logvar
                      + other.logvar				,       dim=[1, 2, 3]				,       )




      def 		snake_case_			(      self		:       List[str]				,       _snake_case		:       List[str]				,       _snake_case		:       Union[str, Any]=[1, 2, 3]				):
          if self.deterministic:
              return torch.Tensor([0.0]				)
          __lowercase		:			List[Any]	    =       np.log(2.0 * np.pi				)
          return 0.5 * torch.sum(logtwopi + self.logvar + torch.pow(sample - self.mean				,       2				) / self.var				,       dim=A_				)




      def 		snake_case_			(      self		:       Dict				):
          return self.mean


import unittest

from transformers import is_torch_available
from transformers.testing_utils import require_torch


if is_torch_available():
 import torch

 from transformers.activations import gelu_new, gelu_python, get_activation


@require_torch
class 						__lowerCAmelCase	(	unittest.TestCase      ):



      """simple docstring"""







      def 		snake_case_			(      self		:       Optional[Any]				):
          __lowercase		:			Union[str, Any]	    =       torch.tensor([-100, -1, -0.1, 0, 0.1, 1.0, 100]				)
          __lowercase		:			Any	    =       get_activation('''gelu'''				)
          self.assertTrue(torch.allclose(gelu_python(_snake_case				)				,       torch_builtin(_snake_case				)				)				)
          self.assertFalse(torch.allclose(gelu_python(_snake_case				)				,       gelu_new(_snake_case				)				)				)




      def 		snake_case_			(      self		:       Dict				):
          __lowercase		:			Any	    =       torch.tensor([-100, -1, -0.1, 0, 0.1, 1.0, 100]				)
          __lowercase		:			List[str]	    =       get_activation('''gelu'''				)
          __lowercase		:			Optional[Any]	    =       get_activation('''gelu_10'''				)

          __lowercase		:			Tuple	    =       torch_builtin(_snake_case				)
          __lowercase		:			List[Any]	    =       geluaa(_snake_case				)

          __lowercase		:			Tuple	    =       torch.where(y_gelu_aa < 10.0				,       1				,       0				)

          self.assertTrue(torch.max(_snake_case				).item() == 10.0				)
          self.assertTrue(torch.allclose(y_gelu * clipped_mask				,       y_gelu_aa * clipped_mask				)				)




      def 		snake_case_			(      self		:       Any				):
          get_activation('''gelu'''				)
          get_activation('''gelu_10'''				)
          get_activation('''gelu_fast'''				)
          get_activation('''gelu_new'''				)
          get_activation('''gelu_python'''				)
          get_activation('''gelu_pytorch_tanh'''				)
          get_activation('''linear'''				)
          get_activation('''mish'''				)
          get_activation('''quick_gelu'''				)
          get_activation('''relu'''				)
          get_activation('''sigmoid'''				)
          get_activation('''silu'''				)
          get_activation('''swish'''				)
          get_activation('''tanh'''				)
          with self.assertRaises(_snake_case				):
              get_activation('''bogus'''				)
          with self.assertRaises(_snake_case				):
              get_activation(_snake_case				)




      def 		snake_case_			(      self		:       Dict				):
          __lowercase		:			Union[str, Any]	    =       get_activation('''gelu'''				)
          __lowercase		:			List[str]	    =       1
          __lowercase		:			Union[str, Any]	    =       get_activation('''gelu'''				)
          self.assertEqual(acta.a				,       1				)
          with self.assertRaises(_snake_case				):
              __lowercase		:			Tuple	    =       acta.a


