



"""simple docstring"""





import math

import torch
from torch import nn

from ..configuration_utils import ConfigMixin, register_to_config
from .attention_processor import Attention
from .embeddings import get_timestep_embedding
from .modeling_utils import ModelMixin



class  _lowerCAmelCase     (					SCREAMING_SNAKE_CASE__	,					SCREAMING_SNAKE_CASE__		):





		"""simple docstring"""






		@register_to_config
		def __init__(       self				,     _lowercase = 1_2_8				,     _lowercase = 2_5_6				,     _lowercase = 2000.0				,     _lowercase = 7_6_8				,     _lowercase = 1_2				,     _lowercase = 1_2				,     _lowercase = 6_4				,     _lowercase = 2_0_4_8				,     _lowercase = 0.1				,     )			->    Dict:







								'''simple docstring'''

								super().__init__()

								snake_case_ :			Optional[Any]            =      nn.Sequential(
								    nn.Linear(_lowercase				,     d_model * 4				,     bias=_lowercase )				,     nn.SiLU()				,     nn.Linear(d_model * 4				,     d_model * 4				,     bias=_lowercase )				,     nn.SiLU()				,     )

								snake_case_ :			Any            =      nn.Embedding(_lowercase				,     _lowercase )
								snake_case_ :			Union[str, Any]            =      False

								snake_case_ :			List[Any]            =      nn.Linear(_lowercase				,     _lowercase				,     bias=_lowercase )

								snake_case_ :			Union[str, Any]            =      nn.Dropout(p=_lowercase )

								snake_case_ :			Tuple            =      nn.ModuleList()
								for lyr_num in range(_lowercase ):
														# FiLM conditional T5 decoder
														snake_case_ :			Union[str, Any]            =      DecoderLayer(d_model=_lowercase				,     d_kv=_lowercase				,     num_heads=_lowercase				,     d_ff=_lowercase				,     dropout_rate=_lowercase )
														self.decoders.append(_lowercase )

								snake_case_ :			List[Any]            =      TaLayerNorm(_lowercase )

								snake_case_ :			Optional[Any]            =      nn.Dropout(p=_lowercase )
								snake_case_ :			List[Any]            =      nn.Linear(_lowercase				,     _lowercase				,     bias=_lowercase )





		def  UpperCAmelCase__				(       self				,     _lowercase				,     _lowercase )			->    Optional[Any]:







								'''simple docstring'''

								snake_case_ :			Optional[int]            =      torch.mul(query_input.unsqueeze(-1 )				,     key_input.unsqueeze(-2 ) )
								return mask.unsqueeze(-3 )





		def  UpperCAmelCase__				(       self				,     _lowercase				,     _lowercase				,     _lowercase )			->    List[Any]:







								'''simple docstring'''

								snake_case_      ,		snake_case_      ,		snake_case_ :			str            =      decoder_input_tokens.shape
								assert decoder_noise_time.shape == (batch,)

								# decoder_noise_time is in [0, 1), so rescale to expected timing range.
								snake_case_ :			Optional[int]            =      get_timestep_embedding(
								    decoder_noise_time * self.config.max_decoder_noise_time				,     embedding_dim=self.config.d_model				,     max_period=self.config.max_decoder_noise_time				,     ).to(dtype=self.dtype )

								snake_case_ :			int            =      self.conditioning_emb(_lowercase ).unsqueeze(1 )

								assert conditioning_emb.shape == (batch, 1, self.config.d_model * 4)

								snake_case_ :			Tuple            =      decoder_input_tokens.shape[1]

								# If we want to use relative positions for audio context, we can just offset
								# this sequence by the length of encodings_and_masks.
								snake_case_ :			Dict            =      torch.broadcast_to(
								    torch.arange(_lowercase				,     device=decoder_input_tokens.device )				,     (batch, seq_length)				,     )

								snake_case_ :			Tuple            =      self.position_encoding(_lowercase )

								snake_case_ :			Optional[Any]            =      self.continuous_inputs_projection(_lowercase )
								inputs += position_encodings
								snake_case_ :			List[Any]            =      self.dropout(_lowercase )

								# decoder: No padding present.
								snake_case_ :			Tuple            =      torch.ones(
								    decoder_input_tokens.shape[:2]				,     device=decoder_input_tokens.device				,     dtype=inputs.dtype )

								# Translate encoding masks to encoder-decoder masks.
								snake_case_ :			int            =      [(x, self.encoder_decoder_mask(_lowercase				,     _lowercase )) for x, y in encodings_and_masks]

								# cross attend style: concat encodings
								snake_case_ :			Optional[Any]            =      torch.cat([x[0] for x in encodings_and_encdec_masks]				,     dim=1 )
								snake_case_ :			str            =      torch.cat([x[1] for x in encodings_and_encdec_masks]				,     dim=-1 )

								for lyr in self.decoders:
														snake_case_ :			int            =      lyr(
														    _lowercase				,     conditioning_emb=_lowercase				,     encoder_hidden_states=_lowercase				,     encoder_attention_mask=_lowercase				,     )[0]

								snake_case_ :			int            =      self.decoder_norm(_lowercase )
								snake_case_ :			Union[str, Any]            =      self.post_dropout(_lowercase )

								snake_case_ :			int            =      self.spec_out(_lowercase )
								return spec_out




class  _lowerCAmelCase     (					nn.Module		):





		"""simple docstring"""






		def __init__(       self				,     _lowercase				,     _lowercase				,     _lowercase				,     _lowercase				,     _lowercase				,     _lowercase=1E-6 )			->    Union[str, Any]:







								'''simple docstring'''

								super().__init__()
								snake_case_ :			Any            =      nn.ModuleList()

								# cond self attention: layer 0
								self.layer.append(
								    TaLayerSelfAttentionCond(d_model=_lowercase				,     d_kv=_lowercase				,     num_heads=_lowercase				,     dropout_rate=_lowercase ) )

								# cross attention: layer 1
								self.layer.append(
								    TaLayerCrossAttention(
								        d_model=_lowercase				,     d_kv=_lowercase				,     num_heads=_lowercase				,     dropout_rate=_lowercase				,     layer_norm_epsilon=_lowercase				,     ) )

								# Film Cond MLP + dropout: last layer
								self.layer.append(
								    TaLayerFFCond(d_model=_lowercase				,     d_ff=_lowercase				,     dropout_rate=_lowercase				,     layer_norm_epsilon=_lowercase ) )





		def  UpperCAmelCase__				(       self				,     _lowercase				,     _lowercase=None				,     _lowercase=None				,     _lowercase=None				,     _lowercase=None				,     _lowercase=None				,     )			->    List[Any]:







								'''simple docstring'''

								snake_case_ :			Tuple            =      self.layer[0](
								    _lowercase				,     conditioning_emb=_lowercase				,     attention_mask=_lowercase				,     )

								if encoder_hidden_states is not None:
														snake_case_ :			Tuple            =      torch.where(encoder_attention_mask > 0				,     0				,     -1E10 ).to(
														    encoder_hidden_states.dtype )

														snake_case_ :			str            =      self.layer[1](
														    _lowercase				,     key_value_states=_lowercase				,     attention_mask=_lowercase				,     )

								# Apply Film Conditional Feed Forward layer
								snake_case_ :			Any            =      self.layer[-1](_lowercase				,     _lowercase )

								return (hidden_states,)




class  _lowerCAmelCase     (					nn.Module		):





		"""simple docstring"""






		def __init__(       self				,     _lowercase				,     _lowercase				,     _lowercase				,     _lowercase )			->    str:







								'''simple docstring'''

								super().__init__()
								snake_case_ :			Any            =      TaLayerNorm(_lowercase )
								snake_case_ :			List[Any]            =      TaFiLMLayer(in_features=d_model * 4				,     out_features=_lowercase )
								snake_case_ :			Union[str, Any]            =      Attention(query_dim=_lowercase				,     heads=_lowercase				,     dim_head=_lowercase				,     out_bias=_lowercase				,     scale_qk=_lowercase )
								snake_case_ :			List[Any]            =      nn.Dropout(_lowercase )





		def  UpperCAmelCase__				(       self				,     _lowercase				,     _lowercase=None				,     _lowercase=None				,     )			->    Optional[Any]:







								'''simple docstring'''

								snake_case_ :			Dict            =      self.layer_norm(_lowercase )

								if conditioning_emb is not None:
														snake_case_ :			str            =      self.FiLMLayer(_lowercase				,     _lowercase )

								# Self-attention block
								snake_case_ :			List[Any]            =      self.attention(_lowercase )

								snake_case_ :			List[str]            =      hidden_states + self.dropout(_lowercase )

								return hidden_states




class  _lowerCAmelCase     (					nn.Module		):





		"""simple docstring"""






		def __init__(       self				,     _lowercase				,     _lowercase				,     _lowercase				,     _lowercase				,     _lowercase )			->    List[Any]:







								'''simple docstring'''

								super().__init__()
								snake_case_ :			List[Any]            =      Attention(query_dim=_lowercase				,     heads=_lowercase				,     dim_head=_lowercase				,     out_bias=_lowercase				,     scale_qk=_lowercase )
								snake_case_ :			Union[str, Any]            =      TaLayerNorm(_lowercase				,     eps=_lowercase )
								snake_case_ :			Optional[Any]            =      nn.Dropout(_lowercase )





		def  UpperCAmelCase__				(       self				,     _lowercase				,     _lowercase=None				,     _lowercase=None				,     )			->    Optional[int]:







								'''simple docstring'''

								snake_case_ :			List[Any]            =      self.layer_norm(_lowercase )
								snake_case_ :			Optional[Any]            =      self.attention(
								    _lowercase				,     encoder_hidden_states=_lowercase				,     attention_mask=attention_mask.squeeze(1 )				,     )
								snake_case_ :			Any            =      hidden_states + self.dropout(_lowercase )
								return layer_output




class  _lowerCAmelCase     (					nn.Module		):





		"""simple docstring"""






		def __init__(       self				,     _lowercase				,     _lowercase				,     _lowercase				,     _lowercase )			->    Dict:







								'''simple docstring'''

								super().__init__()
								snake_case_ :			Tuple            =      TaDenseGatedActDense(d_model=_lowercase				,     d_ff=_lowercase				,     dropout_rate=_lowercase )
								snake_case_ :			List[Any]            =      TaFiLMLayer(in_features=d_model * 4				,     out_features=_lowercase )
								snake_case_ :			Optional[int]            =      TaLayerNorm(_lowercase				,     eps=_lowercase )
								snake_case_ :			Tuple            =      nn.Dropout(_lowercase )





		def  UpperCAmelCase__				(       self				,     _lowercase				,     _lowercase=None )			->    str:







								'''simple docstring'''

								snake_case_ :			List[Any]            =      self.layer_norm(_lowercase )
								if conditioning_emb is not None:
														snake_case_ :			Optional[int]            =      self.film(_lowercase				,     _lowercase )

								snake_case_ :			int            =      self.DenseReluDense(_lowercase )
								snake_case_ :			Optional[Any]            =      hidden_states + self.dropout(_lowercase )
								return hidden_states




class  _lowerCAmelCase     (					nn.Module		):





		"""simple docstring"""






		def __init__(       self				,     _lowercase				,     _lowercase				,     _lowercase )			->    Optional[int]:







								'''simple docstring'''

								super().__init__()
								snake_case_ :			Optional[int]            =      nn.Linear(_lowercase				,     _lowercase				,     bias=_lowercase )
								snake_case_ :			Optional[int]            =      nn.Linear(_lowercase				,     _lowercase				,     bias=_lowercase )
								snake_case_ :			Any            =      nn.Linear(_lowercase				,     _lowercase				,     bias=_lowercase )
								snake_case_ :			int            =      nn.Dropout(_lowercase )
								snake_case_ :			Optional[int]            =      NewGELUActivation()





		def  UpperCAmelCase__				(       self				,     _lowercase )			->    int:







								'''simple docstring'''

								snake_case_ :			str            =      self.act(self.wi_a(_lowercase ) )
								snake_case_ :			Dict            =      self.wi_a(_lowercase )
								snake_case_ :			Any            =      hidden_gelu * hidden_linear
								snake_case_ :			List[Any]            =      self.dropout(_lowercase )

								snake_case_ :			Tuple            =      self.wo(_lowercase )
								return hidden_states




class  _lowerCAmelCase     (					nn.Module		):





		"""simple docstring"""






		def __init__(       self				,     _lowercase				,     _lowercase=1E-6 )			->    str:







								'''simple docstring'''

								super().__init__()
								snake_case_ :			Union[str, Any]            =      nn.Parameter(torch.ones(_lowercase ) )
								snake_case_ :			int            =      eps





		def  UpperCAmelCase__				(       self				,     _lowercase )			->    List[Any]:







								'''simple docstring'''

								snake_case_ :			Tuple            =      hidden_states.to(torch.floataa ).pow(2 ).mean(-1				,     keepdim=_lowercase )
								snake_case_ :			Any            =      hidden_states * torch.rsqrt(variance + self.variance_epsilon )

								# convert into half-precision if necessary
								if self.weight.dtype in [torch.floataa, torch.bfloataa]:
														snake_case_ :			str            =      hidden_states.to(self.weight.dtype )

								return self.weight * hidden_states




class  _lowerCAmelCase     (					nn.Module		):





		"""simple docstring"""






		def  UpperCAmelCase__				(       self				,     _lowercase )			->    torch.Tensor:







								'''simple docstring'''

								return 0.5 * input * (1.0 + torch.tanh(math.sqrt(2.0 / math.pi ) * (input + 0.04_4715 * torch.pow(_lowercase				,     3.0 )) ))




class  _lowerCAmelCase     (					nn.Module		):





		"""simple docstring"""






		def __init__(       self				,     _lowercase				,     _lowercase )			->    Any:







								'''simple docstring'''

								super().__init__()
								snake_case_ :			List[Any]            =      nn.Linear(_lowercase				,     out_features * 2				,     bias=_lowercase )





		def  UpperCAmelCase__				(       self				,     _lowercase				,     _lowercase )			->    List[Any]:







								'''simple docstring'''

								snake_case_ :			List[Any]            =      self.scale_bias(_lowercase )
								snake_case_      ,		snake_case_ :			Any            =      torch.chunk(_lowercase				,     2				,     -1 )
								snake_case_ :			Optional[Any]            =      x * (1 + scale) + shift
								return x








from ...processing_utils import ProcessorMixin
from ...tokenization_utils_base import BatchEncoding




class 	UpperCamelCase_							(  UpperCamelCase__     ):
							lowerCamelCase_							     =       ["image_processor", "tokenizer"]
							lowerCamelCase_							     =       "AutoImageProcessor"
							lowerCamelCase_							     =       "AutoTokenizer"


							def __init__(				self     :Optional[int]				,					__A     :Optional[Any]				,					__A     :Dict		) -> Dict:



									"""simple docstring"""



									super().__init__(__A				,					__A		)
									SCREAMING_SNAKE_CASE__								=   self.image_processor


							def __call__(				self     :int				,					__A     :str=None				,					__A     :int=None				,					__A     :Union[str, Any]=None				,					**__A     :str		) -> Optional[Any]:



									"""simple docstring"""



									if text is None and images is None:
											raise ValueError("""You have to specify either text or images. Both cannot be none."""		)

									if text is not None:
											SCREAMING_SNAKE_CASE__								=   self.tokenizer(__A				,					return_tensors=__A				,					**__A		)

									if images is not None:
											SCREAMING_SNAKE_CASE__								=   self.image_processor(__A				,					return_tensors=__A				,					**__A		)

									if text is not None and images is not None:
											SCREAMING_SNAKE_CASE__								=   image_features.pixel_values
											return encoding
									elif text is not None:
											return encoding
									else:
											return BatchEncoding(data=dict(**__A		)				,					tensor_type=__A		)


							def   _snake_case	(				self     :str				,					*__A     :List[str]				,					**__A     :List[str]		) -> List[Any]:



									"""simple docstring"""



									return self.tokenizer.batch_decode(*__A				,					**__A		)


							def   _snake_case	(				self     :List[str]				,					*__A     :Any				,					**__A     :Any		) -> Tuple:



									"""simple docstring"""



									return self.tokenizer.decode(*__A				,					**__A		)


							@property
							def   _snake_case	(				self     :Dict		) -> List[Any]:



									"""simple docstring"""



									return ["input_ids", "attention_mask", "pixel_values"]