





"""simple docstring"""





from __future__ import annotations

import os
import tempfile
import unittest

from transformers import ConvBertConfig, is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
  import tensorflow as tf

  from transformers import (
      TFConvBertForMaskedLM,
      TFConvBertForMultipleChoice,
      TFConvBertForQuestionAnswering,
      TFConvBertForSequenceClassification,
      TFConvBertForTokenClassification,
      TFConvBertModel,
  )




class 							snake_case							:

      '''simple docstring'''



      def __init__(							self       :      List[Any]						,				__lowercase       :      Optional[int]						,				__lowercase       :      Optional[Any]=13						,				__lowercase       :      Any=7						,				__lowercase       :      str=True						,				__lowercase       :      Optional[int]=True						,				__lowercase       :      Dict=True						,				__lowercase       :      Optional[int]=True						,				__lowercase       :      List[str]=99						,				__lowercase       :      Optional[Any]=32						,				__lowercase       :      Union[str, Any]=2						,				__lowercase       :      int=4						,				__lowercase       :      Any=37						,				__lowercase       :      Any="gelu"						,				__lowercase       :      Optional[int]=0.1						,				__lowercase       :      Optional[Any]=0.1						,				__lowercase       :      Any=512						,				__lowercase       :      str=16						,				__lowercase       :      Any=2						,				__lowercase       :      int=0.0_2						,				__lowercase       :      List[str]=3						,				__lowercase       :      Dict=4						,				__lowercase       :      Optional[Any]=None						,				):


        '''simple docstring'''




        __UpperCAmelCase    :						str       =						parent
        __UpperCAmelCase    :						List[Any]       =						13
        __UpperCAmelCase    :						Any       =						7
        __UpperCAmelCase    :						Union[str, Any]       =						True
        __UpperCAmelCase    :						str       =						True
        __UpperCAmelCase    :						Optional[int]       =						True
        __UpperCAmelCase    :						str       =						True
        __UpperCAmelCase    :						List[str]       =						99
        __UpperCAmelCase    :						int       =						384
        __UpperCAmelCase    :						Optional[int]       =						2
        __UpperCAmelCase    :						Optional[Any]       =						4
        __UpperCAmelCase    :						Optional[int]       =						37
        __UpperCAmelCase    :						int       =						"gelu"
        __UpperCAmelCase    :						Any       =						0.1
        __UpperCAmelCase    :						Any       =						0.1
        __UpperCAmelCase    :						Tuple       =						512
        __UpperCAmelCase    :						Optional[int]       =						16
        __UpperCAmelCase    :						Tuple       =						2
        __UpperCAmelCase    :						Tuple       =						0.0_2
        __UpperCAmelCase    :						str       =						3
        __UpperCAmelCase    :						Any       =						4
        __UpperCAmelCase    :						str       =						128
        __UpperCAmelCase    :						Union[str, Any]       =						2
        __UpperCAmelCase    :						Optional[int]       =						9
        __UpperCAmelCase    :						List[str]       =						1
        __UpperCAmelCase    :						List[str]       =						None

      def       A_					(							self       :      List[Any] ):


        '''simple docstring'''




        __UpperCAmelCase    :						List[Any]       =						ids_tensor([self.batch_size, self.seq_length]						,				self.vocab_size )

        __UpperCAmelCase    :						int       =						None
        if self.use_input_mask:
          __UpperCAmelCase    :						List[str]       =						random_attention_mask([self.batch_size, self.seq_length] )

        __UpperCAmelCase    :						Union[str, Any]       =						None
        if self.use_token_type_ids:
          __UpperCAmelCase    :						Dict       =						ids_tensor([self.batch_size, self.seq_length]						,				self.type_vocab_size )

        __UpperCAmelCase    :						List[str]       =						None
        __UpperCAmelCase    :						Any       =						None
        __UpperCAmelCase    :						Optional[int]       =						None
        if self.use_labels:
          __UpperCAmelCase    :						Dict       =						ids_tensor([self.batch_size]						,				self.type_sequence_label_size )
          __UpperCAmelCase    :						Union[str, Any]       =						ids_tensor([self.batch_size, self.seq_length]						,				self.num_labels )
          __UpperCAmelCase    :						Optional[Any]       =						ids_tensor([self.batch_size]						,				self.num_choices )

        __UpperCAmelCase    :						Optional[int]       =						ConvBertConfig(
            vocab_size=self.vocab_size						,				hidden_size=self.hidden_size						,				num_hidden_layers=self.num_hidden_layers						,				num_attention_heads=self.num_attention_heads						,				intermediate_size=self.intermediate_size						,				hidden_act=self.hidden_act						,				hidden_dropout_prob=self.hidden_dropout_prob						,				attention_probs_dropout_prob=self.attention_probs_dropout_prob						,				max_position_embeddings=self.max_position_embeddings						,				type_vocab_size=self.type_vocab_size						,				initializer_range=self.initializer_range						,				return_dict=snake_case_						,				)

        return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels

      def       A_					(							self       :      str						,				__lowercase       :      Optional[int]						,				__lowercase       :      Union[str, Any]						,				__lowercase       :      Any						,				__lowercase       :      Tuple						,				__lowercase       :      int						,				__lowercase       :      Optional[int]						,				__lowercase       :      Tuple ):


        '''simple docstring'''




        __UpperCAmelCase    :						Dict       =						TFConvBertModel(config=snake_case_ )
        __UpperCAmelCase    :						List[Any]       =						{"input_ids": input_ids, "attention_mask": input_mask, "token_type_ids": token_type_ids}

        __UpperCAmelCase    :						Union[str, Any]       =						[input_ids, input_mask]
        __UpperCAmelCase    :						Dict       =						model(snake_case_ )

        __UpperCAmelCase    :						Optional[Any]       =						model(snake_case_ )

        self.parent.assertEqual(result.last_hidden_state.shape						,				(self.batch_size, self.seq_length, self.hidden_size) )

      def       A_					(							self       :      List[Any]						,				__lowercase       :      Optional[Any]						,				__lowercase       :      Any						,				__lowercase       :      Union[str, Any]						,				__lowercase       :      int						,				__lowercase       :      Optional[Any]						,				__lowercase       :      Union[str, Any]						,				__lowercase       :      List[str] ):


        '''simple docstring'''




        __UpperCAmelCase    :						List[Any]       =						TFConvBertForMaskedLM(config=snake_case_ )
        __UpperCAmelCase    :						str       =						{
            "input_ids": input_ids,
            "attention_mask": input_mask,
            "token_type_ids": token_type_ids,
        }
        __UpperCAmelCase    :						int       =						model(snake_case_ )
        self.parent.assertEqual(result.logits.shape						,				(self.batch_size, self.seq_length, self.vocab_size) )

      def       A_					(							self       :      str						,				__lowercase       :      str						,				__lowercase       :      Union[str, Any]						,				__lowercase       :      Dict						,				__lowercase       :      List[str]						,				__lowercase       :      str						,				__lowercase       :      Tuple						,				__lowercase       :      int ):


        '''simple docstring'''




        __UpperCAmelCase    :						Optional[Any]       =						self.num_labels
        __UpperCAmelCase    :						Any       =						TFConvBertForSequenceClassification(config=snake_case_ )
        __UpperCAmelCase    :						int       =						{
            "input_ids": input_ids,
            "attention_mask": input_mask,
            "token_type_ids": token_type_ids,
        }

        __UpperCAmelCase    :						int       =						model(snake_case_ )
        self.parent.assertEqual(result.logits.shape						,				(self.batch_size, self.num_labels) )

      def       A_					(							self       :      Optional[Any]						,				__lowercase       :      List[str]						,				__lowercase       :      Tuple						,				__lowercase       :      List[Any]						,				__lowercase       :      Any						,				__lowercase       :      Optional[Any]						,				__lowercase       :      Any						,				__lowercase       :      List[Any] ):


        '''simple docstring'''




        __UpperCAmelCase    :						Dict       =						self.num_choices
        __UpperCAmelCase    :						int       =						TFConvBertForMultipleChoice(config=snake_case_ )
        __UpperCAmelCase    :						Any       =						tf.tile(tf.expand_dims(snake_case_						,				1 )						,				(1, self.num_choices, 1) )
        __UpperCAmelCase    :						int       =						tf.tile(tf.expand_dims(snake_case_						,				1 )						,				(1, self.num_choices, 1) )
        __UpperCAmelCase    :						List[str]       =						tf.tile(tf.expand_dims(snake_case_						,				1 )						,				(1, self.num_choices, 1) )
        __UpperCAmelCase    :						Union[str, Any]       =						{
            "input_ids": multiple_choice_inputs_ids,
            "attention_mask": multiple_choice_input_mask,
            "token_type_ids": multiple_choice_token_type_ids,
        }
        __UpperCAmelCase    :						Optional[Any]       =						model(snake_case_ )
        self.parent.assertEqual(result.logits.shape						,				(self.batch_size, self.num_choices) )

      def       A_					(							self       :      Union[str, Any]						,				__lowercase       :      Optional[Any]						,				__lowercase       :      int						,				__lowercase       :      str						,				__lowercase       :      Tuple						,				__lowercase       :      List[Any]						,				__lowercase       :      Tuple						,				__lowercase       :      Any ):


        '''simple docstring'''




        __UpperCAmelCase    :						Optional[Any]       =						self.num_labels
        __UpperCAmelCase    :						Any       =						TFConvBertForTokenClassification(config=snake_case_ )
        __UpperCAmelCase    :						List[str]       =						{
            "input_ids": input_ids,
            "attention_mask": input_mask,
            "token_type_ids": token_type_ids,
        }
        __UpperCAmelCase    :						List[str]       =						model(snake_case_ )
        self.parent.assertEqual(result.logits.shape						,				(self.batch_size, self.seq_length, self.num_labels) )

      def       A_					(							self       :      Union[str, Any]						,				__lowercase       :      Union[str, Any]						,				__lowercase       :      Optional[Any]						,				__lowercase       :      Optional[Any]						,				__lowercase       :      Tuple						,				__lowercase       :      int						,				__lowercase       :      Optional[int]						,				__lowercase       :      Any ):


        '''simple docstring'''




        __UpperCAmelCase    :						List[str]       =						TFConvBertForQuestionAnswering(config=snake_case_ )
        __UpperCAmelCase    :						Optional[Any]       =						{
            "input_ids": input_ids,
            "attention_mask": input_mask,
            "token_type_ids": token_type_ids,
        }

        __UpperCAmelCase    :						int       =						model(snake_case_ )
        self.parent.assertEqual(result.start_logits.shape						,				(self.batch_size, self.seq_length) )
        self.parent.assertEqual(result.end_logits.shape						,				(self.batch_size, self.seq_length) )







      def       A_					(							self       :      Optional[int] ):


        '''simple docstring'''




        __UpperCAmelCase    :						Union[str, Any]       =						self.prepare_config_and_inputs()
        (
            __UpperCAmelCase					
        )    :						Tuple       =						config_and_inputs
        __UpperCAmelCase    :						Optional[int]       =						{"input_ids": input_ids, "token_type_ids": token_type_ids, "attention_mask": input_mask}
        return config, inputs_dict




@require_tf
class 							snake_case							(			_snake_case      ,  _snake_case      ,  unittest.TestCase ):

      '''simple docstring'''



      _A     :       Optional[int]         =       (
          (
              TFConvBertModel,
              TFConvBertForMaskedLM,
              TFConvBertForQuestionAnswering,
              TFConvBertForSequenceClassification,
              TFConvBertForTokenClassification,
              TFConvBertForMultipleChoice,
          )
          if is_tf_available()
          else ()
      )
      _A     :       List[Any]         =       (
          {
              """feature-extraction""": TFConvBertModel,
              """fill-mask""": TFConvBertForMaskedLM,
              """question-answering""": TFConvBertForQuestionAnswering,
              """text-classification""": TFConvBertForSequenceClassification,
              """token-classification""": TFConvBertForTokenClassification,
              """zero-shot""": TFConvBertForSequenceClassification,
          }
          if is_tf_available()
          else {}
      )
      _A     :       Optional[int]         =       False
      _A     :       Tuple         =       False
      _A     :       Optional[Any]         =       False

      def       A_					(							self       :      Dict ):


        '''simple docstring'''




        __UpperCAmelCase    :						List[Any]       =						TFConvBertModelTester(self )
        __UpperCAmelCase    :						Dict       =						ConfigTester(self						,				config_class=snake_case_						,				hidden_size=37 )

      def       A_					(							self       :      str ):


        '''simple docstring'''




        self.config_tester.run_common_tests()

      def       A_					(							self       :      str ):


        '''simple docstring'''




        __UpperCAmelCase    :						Union[str, Any]       =						self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_model(*snake_case_ )

      def       A_					(							self       :      Tuple ):


        '''simple docstring'''




        __UpperCAmelCase    :						Any       =						self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_for_masked_lm(*snake_case_ )

      def       A_					(							self       :      Any ):


        '''simple docstring'''




        __UpperCAmelCase    :						Dict       =						self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_for_multiple_choice(*snake_case_ )

      def       A_					(							self       :      Any ):


        '''simple docstring'''




        __UpperCAmelCase    :						List[str]       =						self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_for_question_answering(*snake_case_ )

      def       A_					(							self       :      Tuple ):


        '''simple docstring'''




        __UpperCAmelCase    :						Any       =						self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_for_sequence_classification(*snake_case_ )

      def       A_					(							self       :      Optional[Any] ):


        '''simple docstring'''




        __UpperCAmelCase    :						Optional[int]       =						self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_for_token_classification(*snake_case_ )

      @slow
      def       A_					(							self       :      int ):


        '''simple docstring'''




        __UpperCAmelCase    :						List[str]       =						self.model_tester.prepare_config_and_inputs_for_common()
        __UpperCAmelCase    :						List[Any]       =						True
        __UpperCAmelCase    :						Dict       =						True

        if hasattr(snake_case_						,				'''use_cache''' ):
          __UpperCAmelCase    :						Dict       =						True

        __UpperCAmelCase    :						str       =						getattr(self.model_tester						,				'''encoder_seq_length'''						,				self.model_tester.seq_length )
        __UpperCAmelCase    :						List[Any]       =						getattr(self.model_tester						,				'''key_length'''						,				snake_case_ )

        for model_class in self.all_model_classes:
          __UpperCAmelCase    :						Optional[Any]       =						self._prepare_for_class(snake_case_						,				snake_case_ )
          __UpperCAmelCase    :						str       =						model_class(snake_case_ )
          __UpperCAmelCase    :						List[str]       =						len(model(snake_case_ ) )

          with tempfile.TemporaryDirectory() as tmpdirname:
            model.save_pretrained(snake_case_						,				saved_model=snake_case_ )
            __UpperCAmelCase    :						List[str]       =						os.path.join(snake_case_						,				'''saved_model'''						,				'''1''' )
            __UpperCAmelCase    :						Dict       =						tf.keras.models.load_model(snake_case_ )
            __UpperCAmelCase    :						str       =						model(snake_case_ )

            if self.is_encoder_decoder:
              __UpperCAmelCase    :						str       =						outputs["encoder_hidden_states"]
              __UpperCAmelCase    :						List[Any]       =						outputs["encoder_attentions"]
            else:
              __UpperCAmelCase    :						int       =						outputs["hidden_states"]
              __UpperCAmelCase    :						List[Any]       =						outputs["attentions"]

            self.assertEqual(len(snake_case_ )						,				snake_case_ )

            __UpperCAmelCase    :						Dict       =						getattr(
                self.model_tester						,				'''expected_num_hidden_layers'''						,				self.model_tester.num_hidden_layers + 1 )

            self.assertEqual(len(snake_case_ )						,				snake_case_ )
            self.assertListEqual(
                list(output_hidden_states[0].shape[-2:] )						,				[self.model_tester.seq_length, self.model_tester.hidden_size]						,				)

            self.assertEqual(len(snake_case_ )						,				self.model_tester.num_hidden_layers )
            self.assertListEqual(
                list(output_attentions[0].shape[-3:] )						,				[self.model_tester.num_attention_heads / 2, encoder_seq_length, encoder_key_length]						,				)

      @slow
      def       A_					(							self       :      str ):


        '''simple docstring'''




        __UpperCAmelCase    :						Any       =						TFConvBertModel.from_pretrained('''YituTech/conv-bert-base''' )
        self.assertIsNotNone(snake_case_ )







      def       A_					(							self       :      Optional[int] ):


        '''simple docstring'''




        __UpperCAmelCase    :						Optional[Any]       =						self.model_tester.prepare_config_and_inputs_for_common()
        __UpperCAmelCase    :						Optional[Any]       =						True
        __UpperCAmelCase    :						Tuple       =						getattr(self.model_tester						,				'''decoder_seq_length'''						,				self.model_tester.seq_length )
        __UpperCAmelCase    :						Any       =						getattr(self.model_tester						,				'''encoder_seq_length'''						,				self.model_tester.seq_length )
        __UpperCAmelCase    :						List[str]       =						getattr(self.model_tester						,				'''key_length'''						,				snake_case_ )
        __UpperCAmelCase    :						List[str]       =						getattr(self.model_tester						,				'''key_length'''						,				snake_case_ )

        def check_decoder_attentions_output(__lowercase       :      int ):
          __UpperCAmelCase    :						Optional[int]       =						len(snake_case_ )
          self.assertEqual(out_len % 2						,				0 )
          __UpperCAmelCase    :						List[str]       =						outputs.decoder_attentions
          self.assertEqual(len(snake_case_ )						,				self.model_tester.num_hidden_layers )
          self.assertListEqual(
              list(decoder_attentions[0].shape[-3:] )						,				[self.model_tester.num_attention_heads / 2, decoder_seq_length, decoder_key_length]						,				)

        def check_encoder_attentions_output(__lowercase       :      int ):
          __UpperCAmelCase    :						Optional[Any]       =						[
              t.numpy() for t in (outputs.encoder_attentions if config.is_encoder_decoder else outputs.attentions)
          ]
          self.assertEqual(len(snake_case_ )						,				self.model_tester.num_hidden_layers )
          self.assertListEqual(
              list(attentions[0].shape[-3:] )						,				[self.model_tester.num_attention_heads / 2, encoder_seq_length, encoder_key_length]						,				)

        for model_class in self.all_model_classes:
          __UpperCAmelCase    :						str       =						True
          __UpperCAmelCase    :						Tuple       =						False
          __UpperCAmelCase    :						List[Any]       =						model_class(snake_case_ )
          __UpperCAmelCase    :						List[str]       =						model(self._prepare_for_class(snake_case_						,				snake_case_ ) )
          __UpperCAmelCase    :						Optional[int]       =						len(snake_case_ )
          self.assertEqual(config.output_hidden_states						,				snake_case_ )
          check_encoder_attentions_output(snake_case_ )

          if self.is_encoder_decoder:
            __UpperCAmelCase    :						Optional[Any]       =						model_class(snake_case_ )
            __UpperCAmelCase    :						str       =						model(self._prepare_for_class(snake_case_						,				snake_case_ ) )
            self.assertEqual(config.output_hidden_states						,				snake_case_ )
            check_decoder_attentions_output(snake_case_ )

          # Check that output attentions can also be changed via the config
          del inputs_dict["output_attentions"]
          __UpperCAmelCase    :						List[Any]       =						True
          __UpperCAmelCase    :						Dict       =						model_class(snake_case_ )
          __UpperCAmelCase    :						Tuple       =						model(self._prepare_for_class(snake_case_						,				snake_case_ ) )
          self.assertEqual(config.output_hidden_states						,				snake_case_ )
          check_encoder_attentions_output(snake_case_ )

          # Check attention is always last and order is fine
          __UpperCAmelCase    :						str       =						True
          __UpperCAmelCase    :						int       =						True
          __UpperCAmelCase    :						Optional[int]       =						model_class(snake_case_ )
          __UpperCAmelCase    :						List[str]       =						model(self._prepare_for_class(snake_case_						,				snake_case_ ) )

          self.assertEqual(out_len + (2 if self.is_encoder_decoder else 1)						,				len(snake_case_ ) )
          self.assertEqual(model.config.output_hidden_states						,				snake_case_ )
          check_encoder_attentions_output(snake_case_ )




@require_tf
class 							snake_case							(			unittest.TestCase ):

      '''simple docstring'''



      @slow
      def       A_					(							self       :      Dict ):


        '''simple docstring'''




        __UpperCAmelCase    :						str       =						TFConvBertModel.from_pretrained('''YituTech/conv-bert-base''' )
        __UpperCAmelCase    :						Optional[int]       =						tf.constant([[0, 1, 2, 3, 4, 5]] )
        __UpperCAmelCase    :						int       =						model(snake_case_ )[0]

        __UpperCAmelCase    :						List[str]       =						[1, 6, 768]
        self.assertEqual(output.shape						,				snake_case_ )

        __UpperCAmelCase    :						List[Any]       =						tf.constant(
            [
                [
                    [-0.0_3_4_7_5_4_9_3, -0.4_6_8_6_0_3_4, -0.3_0_6_3_8_8_3_2],
                    [0.2_2_6_3_7_2_4_8, -0.2_6_9_8_8_6_4_6, -0.7_4_2_3_4_2_4],
                    [0.1_0_3_2_4_8_6_8, -0.4_5_0_1_3_5_0_8, -0.5_8_2_8_0_7_8_4],
                ]
            ] )
        tf.debugging.assert_near(output[:, :3, :3]						,				snake_case_						,				atol=1e-4 )






"""simple docstring"""





from dataclasses import dataclass
from typing import Dict, Optional, Union

import torch
import torch.nn.functional as F
from torch import nn

from ..configuration_utils import ConfigMixin, register_to_config
from ..utils import BaseOutput
from .attention import BasicTransformerBlock
from .attention_processor import AttentionProcessor, AttnProcessor
from .embeddings import TimestepEmbedding, Timesteps
from .modeling_utils import ModelMixin




@dataclass
class 							snake_case							(			__UpperCAmelCase ):

      '''simple docstring'''



      _A     :       torch.FloatTensor




class 							snake_case							(			__UpperCAmelCase      ,  __UpperCAmelCase ):

      '''simple docstring'''



      @register_to_config
      def __init__(							self       :      Union[str, Any]						,				__lowercase       :      int = 32						,				__lowercase       :      int = 64						,				__lowercase       :      int = 20						,				__lowercase       :      int = 768						,				__lowercase       :      Optional[int]=77						,				__lowercase       :      Union[str, Any]=4						,				__lowercase       :      float = 0.0						,				__lowercase       :      str = "silu"						,				__lowercase       :      Optional[str] = None						,				__lowercase       :      Optional[str] = None						,				__lowercase       :      Optional[str] = "linear"						,				__lowercase       :      Optional[str] = "prd"						,				__lowercase       :      Optional[int] = None						,				__lowercase       :      Optional[int] = None						,				__lowercase       :      Optional[int] = None						,				):


        '''simple docstring'''




        super().__init__()
        __UpperCAmelCase    :						int       =						num_attention_heads
        __UpperCAmelCase    :						List[Any]       =						attention_head_dim
        __UpperCAmelCase    :						int       =						num_attention_heads * attention_head_dim
        __UpperCAmelCase    :						List[str]       =						additional_embeddings

        __UpperCAmelCase    :						Optional[int]       =						time_embed_dim or inner_dim
        __UpperCAmelCase    :						Tuple       =						embedding_proj_dim or embedding_dim
        __UpperCAmelCase    :						Dict       =						clip_embed_dim or embedding_dim

        __UpperCAmelCase    :						Dict       =						Timesteps(__lowercase						,				__lowercase						,				0 )
        __UpperCAmelCase    :						List[str]       =						TimestepEmbedding(__lowercase						,				__lowercase						,				out_dim=__lowercase						,				act_fn=__lowercase )

        __UpperCAmelCase    :						Any       =						nn.Linear(__lowercase						,				__lowercase )

        if embedding_proj_norm_type is None:
          __UpperCAmelCase    :						Dict       =						None
        elif embedding_proj_norm_type == "layer":
          __UpperCAmelCase    :						Any       =						nn.LayerNorm(__lowercase )
        else:
          raise ValueError(f'''unsupported embedding_proj_norm_type: {embedding_proj_norm_type}''' )

        __UpperCAmelCase    :						List[Any]       =						nn.Linear(__lowercase						,				__lowercase )

        if encoder_hid_proj_type is None:
          __UpperCAmelCase    :						int       =						None
        elif encoder_hid_proj_type == "linear":
          __UpperCAmelCase    :						int       =						nn.Linear(__lowercase						,				__lowercase )
        else:
          raise ValueError(f'''unsupported encoder_hid_proj_type: {encoder_hid_proj_type}''' )

        __UpperCAmelCase    :						str       =						nn.Parameter(torch.zeros(1						,				num_embeddings + additional_embeddings						,				__lowercase ) )

        if added_emb_type == "prd":
          __UpperCAmelCase    :						Optional[int]       =						nn.Parameter(torch.zeros(1						,				1						,				__lowercase ) )
        elif added_emb_type is None:
          __UpperCAmelCase    :						str       =						None
        else:
          raise ValueError(
              f'''`added_emb_type`: {added_emb_type} is not supported. Make sure to choose one of `\'prd\'` or `None`.''' )

        __UpperCAmelCase    :						Union[str, Any]       =						nn.ModuleList(
            [
                BasicTransformerBlock(
                    __lowercase						,				__lowercase						,				__lowercase						,				dropout=__lowercase						,				activation_fn='''gelu'''						,				attention_bias=__lowercase						,				)
                for d in range(__lowercase )
            ] )

        if norm_in_type == "layer":
          __UpperCAmelCase    :						Optional[Any]       =						nn.LayerNorm(__lowercase )
        elif norm_in_type is None:
          __UpperCAmelCase    :						List[Any]       =						None
        else:
          raise ValueError(f'''Unsupported norm_in_type: {norm_in_type}.''' )

        __UpperCAmelCase    :						str       =						nn.LayerNorm(__lowercase )

        __UpperCAmelCase    :						List[Any]       =						nn.Linear(__lowercase						,				__lowercase )

        __UpperCAmelCase    :						List[Any]       =						torch.full(
            [num_embeddings + additional_embeddings, num_embeddings + additional_embeddings]						,				-1_0_0_0_0.0 )
        causal_attention_mask.triu_(1 )
        __UpperCAmelCase    :						Optional[Any]       =						causal_attention_mask[None, ...]
        self.register_buffer('''causal_attention_mask'''						,				__lowercase						,				persistent=__lowercase )

        __UpperCAmelCase    :						Any       =						nn.Parameter(torch.zeros(1						,				__lowercase ) )
        __UpperCAmelCase    :						List[str]       =						nn.Parameter(torch.zeros(1						,				__lowercase ) )

      @property
      # Copied from diffusers.models.unet_2d_condition.UNet2DConditionModel.attn_processors
      def       A_					(							self       :      Optional[int] ):


        '''simple docstring'''




        __UpperCAmelCase    :						Optional[int]       =						{}

        def fn_recursive_add_processors(__lowercase       :      str						,				__lowercase       :      torch.nn.Module						,				__lowercase       :      Dict[str, AttentionProcessor] ):
          if hasattr(__lowercase						,				'''set_processor''' ):
            __UpperCAmelCase    :						Optional[Any]       =						module.processor

          for sub_name, child in module.named_children():
            fn_recursive_add_processors(f'''{name}.{sub_name}'''						,				__lowercase						,				__lowercase )

          return processors

        for name, module in self.named_children():
          fn_recursive_add_processors(__lowercase						,				__lowercase						,				__lowercase )

        return processors

      def       A_					(							self       :      Any						,				__lowercase       :      Union[AttentionProcessor, Dict[str, AttentionProcessor]] ):


        '''simple docstring'''




        __UpperCAmelCase    :						Tuple       =						len(self.attn_processors.keys() )

        if isinstance(__lowercase						,				__lowercase ) and len(__lowercase ) != count:
          raise ValueError(
              f'''A dict of processors was passed, but the number of processors {len(__lowercase )} does not match the'''
              f''' number of attention layers: {count}. Please make sure to pass {count} processor classes.''' )

        def fn_recursive_attn_processor(__lowercase       :      str						,				__lowercase       :      torch.nn.Module						,				__lowercase       :      int ):
          if hasattr(__lowercase						,				'''set_processor''' ):
            if not isinstance(__lowercase						,				__lowercase ):
              module.set_processor(__lowercase )
            else:
              module.set_processor(processor.pop(f'''{name}.processor''' ) )

          for sub_name, child in module.named_children():
            fn_recursive_attn_processor(f'''{name}.{sub_name}'''						,				__lowercase						,				__lowercase )

        for name, module in self.named_children():
          fn_recursive_attn_processor(__lowercase						,				__lowercase						,				__lowercase )

      def       A_					(							self       :      Optional[Any] ):


        '''simple docstring'''




        self.set_attn_processor(AttnProcessor() )

      def       A_					(							self       :      Optional[int]						,				__lowercase       :      Dict						,				__lowercase       :      Union[torch.Tensor, float, int]						,				__lowercase       :      torch.FloatTensor						,				__lowercase       :      Optional[torch.FloatTensor] = None						,				__lowercase       :      Optional[torch.BoolTensor] = None						,				__lowercase       :      bool = True						,				):


        '''simple docstring'''




        __UpperCAmelCase    :						List[Any]       =						hidden_states.shape[0]

        __UpperCAmelCase    :						Any       =						timestep
        if not torch.is_tensor(__lowercase ):
          __UpperCAmelCase    :						Union[str, Any]       =						torch.tensor([timesteps]						,				dtype=torch.long						,				device=hidden_states.device )
        elif torch.is_tensor(__lowercase ) and len(timesteps.shape ) == 0:
          __UpperCAmelCase    :						Union[str, Any]       =						timesteps[None].to(hidden_states.device )

        # broadcast to batch dimension in a way that's compatible with ONNX/Core ML
        __UpperCAmelCase    :						List[str]       =						timesteps * torch.ones(__lowercase						,				dtype=timesteps.dtype						,				device=timesteps.device )

        __UpperCAmelCase    :						List[str]       =						self.time_proj(__lowercase )

        # timesteps does not contain any weights and will always return f32 tensors
        # but time_embedding might be fp16, so we need to cast here.
        __UpperCAmelCase    :						Any       =						timesteps_projected.to(dtype=self.dtype )
        __UpperCAmelCase    :						List[str]       =						self.time_embedding(__lowercase )

        if self.embedding_proj_norm is not None:
          __UpperCAmelCase    :						Dict       =						self.embedding_proj_norm(__lowercase )

        __UpperCAmelCase    :						Optional[int]       =						self.embedding_proj(__lowercase )
        if self.encoder_hidden_states_proj is not None and encoder_hidden_states is not None:
          __UpperCAmelCase    :						Dict       =						self.encoder_hidden_states_proj(__lowercase )
        elif self.encoder_hidden_states_proj is not None and encoder_hidden_states is None:
          raise ValueError('''`encoder_hidden_states_proj` requires `encoder_hidden_states` to be set''' )

        __UpperCAmelCase    :						int       =						self.proj_in(__lowercase )

        __UpperCAmelCase    :						Tuple       =						self.positional_embedding.to(hidden_states.dtype )

        __UpperCAmelCase    :						Optional[Any]       =						[]
        __UpperCAmelCase    :						str       =						0

        if encoder_hidden_states is not None:
          additional_embeds.append(__lowercase )
          additional_embeddings_len += encoder_hidden_states.shape[1]

        if len(proj_embeddings.shape ) == 2:
          __UpperCAmelCase    :						str       =						proj_embeddings[:, None, :]

        if len(hidden_states.shape ) == 2:
          __UpperCAmelCase    :						Optional[int]       =						hidden_states[:, None, :]

        __UpperCAmelCase    :						str       =						additional_embeds + [
            proj_embeddings,
            time_embeddings[:, None, :],
            hidden_states,
        ]

        if self.prd_embedding is not None:
          __UpperCAmelCase    :						Dict       =						self.prd_embedding.to(hidden_states.dtype ).expand(__lowercase						,				-1						,				-1 )
          additional_embeds.append(__lowercase )

        __UpperCAmelCase    :						Optional[Any]       =						torch.cat(
            __lowercase						,				dim=1						,				)

        # Allow positional_embedding to not include the `addtional_embeddings` and instead pad it with zeros for these additional tokens
        __UpperCAmelCase    :						Optional[int]       =						additional_embeddings_len + proj_embeddings.shape[1] + 1
        if positional_embeddings.shape[1] < hidden_states.shape[1]:
          __UpperCAmelCase    :						List[str]       =						F.pad(
              __lowercase						,				(
                  0,
                  0,
                  additional_embeddings_len,
                  self.prd_embedding.shape[1] if self.prd_embedding is not None else 0,
              )						,				value=0.0						,				)

        __UpperCAmelCase    :						Dict       =						hidden_states + positional_embeddings

        if attention_mask is not None:
          __UpperCAmelCase    :						Optional[Any]       =						(1 - attention_mask.to(hidden_states.dtype )) * -1_0_0_0_0.0
          __UpperCAmelCase    :						Optional[Any]       =						F.pad(__lowercase						,				(0, self.additional_embeddings)						,				value=0.0 )
          __UpperCAmelCase    :						Dict       =						(attention_mask[:, None, :] + self.causal_attention_mask).to(hidden_states.dtype )
          __UpperCAmelCase    :						Tuple       =						attention_mask.repeat_interleave(self.config.num_attention_heads						,				dim=0 )

        if self.norm_in is not None:
          __UpperCAmelCase    :						str       =						self.norm_in(__lowercase )

        for block in self.transformer_blocks:
          __UpperCAmelCase    :						Dict       =						block(__lowercase						,				attention_mask=__lowercase )

        __UpperCAmelCase    :						Any       =						self.norm_out(__lowercase )

        if self.prd_embedding is not None:
          __UpperCAmelCase    :						List[Any]       =						hidden_states[:, -1]
        else:
          __UpperCAmelCase    :						List[str]       =						hidden_states[:, additional_embeddings_len:]

        __UpperCAmelCase    :						Optional[int]       =						self.proj_to_clip_embeddings(__lowercase )

        if not return_dict:
          return (predicted_image_embedding,)

        return PriorTransformerOutput(predicted_image_embedding=__lowercase )







      def       A_					(							self       :      List[Any]						,				__lowercase       :      Optional[int] ):


        '''simple docstring'''




        __UpperCAmelCase    :						List[str]       =						(prior_latents * self.clip_std) + self.clip_mean
        return prior_latents