


"""simple docstring"""



import math

import torch
from torch import nn

from ..configuration_utils import ConfigMixin, register_to_config
from .attention_processor import Attention
from .embeddings import get_timestep_embedding
from .modeling_utils import ModelMixin







class _lowerCAmelCase							( snake_case_       ,				snake_case_      ):







   @register_to_config
   def __init__(      self		,      UpperCamelCase__ = 128		,      UpperCamelCase__ = 256		,      UpperCamelCase__ = 2000.0		,      UpperCamelCase__ = 768		,      UpperCamelCase__ = 12		,      UpperCamelCase__ = 12		,      UpperCamelCase__ = 64		,      UpperCamelCase__ = 2048		,      UpperCamelCase__ = 0.1		,      )      ->						Any:

       '''simple docstring'''




       super().__init__()

       snake_case       : Tuple    =       nn.Sequential(
           nn.Linear(UpperCamelCase__		,      d_model * 4		,      bias=UpperCamelCase__		)		,      nn.SiLU()		,      nn.Linear(d_model * 4		,      d_model * 4		,      bias=UpperCamelCase__		)		,      nn.SiLU()		,      )

       snake_case       : Any    =       nn.Embedding(UpperCamelCase__		,      UpperCamelCase__		)
       snake_case       : Dict    =       False

       snake_case       : Optional[Any]    =       nn.Linear(UpperCamelCase__		,      UpperCamelCase__		,      bias=UpperCamelCase__		)

       snake_case       : str    =       nn.Dropout(p=UpperCamelCase__		)

       snake_case       : List[str]    =       nn.ModuleList()
       for lyr_num in range(UpperCamelCase__		):
           # FiLM conditional T5 decoder
           snake_case       : str    =       DecoderLayer(d_model=UpperCamelCase__		,      d_kv=UpperCamelCase__		,      num_heads=UpperCamelCase__		,      d_ff=UpperCamelCase__		,      dropout_rate=UpperCamelCase__		)
           self.decoders.append(UpperCamelCase__		)

       snake_case       : List[str]    =       TaLayerNorm(UpperCamelCase__		)

       snake_case       : Optional[Any]    =       nn.Dropout(p=UpperCamelCase__		)
       snake_case       : List[Any]    =       nn.Linear(UpperCamelCase__		,      UpperCamelCase__		,      bias=UpperCamelCase__		)







   def 					lowerCamelCase						(      self		,      UpperCamelCase__		,      UpperCamelCase__		)      ->						Union[str, Any]:

       '''simple docstring'''




       snake_case       : List[Any]    =       torch.mul(query_input.unsqueeze(-1		)		,      key_input.unsqueeze(-2		)		)
       return mask.unsqueeze(-3		)







   def 					lowerCamelCase						(      self		,      UpperCamelCase__		,      UpperCamelCase__		,      UpperCamelCase__		)      ->						Union[str, Any]:

       '''simple docstring'''




       snake_case   ,snake_case   ,snake_case       : str    =       decoder_input_tokens.shape
       assert decoder_noise_time.shape == (batch,)

       # decoder_noise_time is in [0, 1), so rescale to expected timing range.
       snake_case       : Dict    =       get_timestep_embedding(
           decoder_noise_time * self.config.max_decoder_noise_time		,      embedding_dim=self.config.d_model		,      max_period=self.config.max_decoder_noise_time		,      ).to(dtype=self.dtype		)

       snake_case       : List[Any]    =       self.conditioning_emb(UpperCamelCase__		).unsqueeze(1		)

       assert conditioning_emb.shape == (batch, 1, self.config.d_model * 4)

       snake_case       : Optional[Any]    =       decoder_input_tokens.shape[1]

       # If we want to use relative positions for audio context, we can just offset
       # this sequence by the length of encodings_and_masks.
       snake_case       : int    =       torch.broadcast_to(
           torch.arange(UpperCamelCase__		,      device=decoder_input_tokens.device		)		,      (batch, seq_length)		,      )

       snake_case       : Optional[Any]    =       self.position_encoding(UpperCamelCase__		)

       snake_case       : Union[str, Any]    =       self.continuous_inputs_projection(UpperCamelCase__		)
       inputs += position_encodings
       snake_case       : Any    =       self.dropout(UpperCamelCase__		)

       # decoder: No padding present.
       snake_case       : Union[str, Any]    =       torch.ones(
           decoder_input_tokens.shape[:2]		,      device=decoder_input_tokens.device		,      dtype=inputs.dtype		)

       # Translate encoding masks to encoder-decoder masks.
       snake_case       : List[str]    =       [(x, self.encoder_decoder_mask(UpperCamelCase__		,      UpperCamelCase__		)) for x, y in encodings_and_masks]

       # cross attend style: concat encodings
       snake_case       : Tuple    =       torch.cat([x[0] for x in encodings_and_encdec_masks]		,      dim=1		)
       snake_case       : str    =       torch.cat([x[1] for x in encodings_and_encdec_masks]		,      dim=-1		)

       for lyr in self.decoders:
           snake_case       : Any    =       lyr(
               UpperCamelCase__		,      conditioning_emb=UpperCamelCase__		,      encoder_hidden_states=UpperCamelCase__		,      encoder_attention_mask=UpperCamelCase__		,      )[0]

       snake_case       : List[str]    =       self.decoder_norm(UpperCamelCase__		)
       snake_case       : Optional[Any]    =       self.post_dropout(UpperCamelCase__		)

       snake_case       : Union[str, Any]    =       self.spec_out(UpperCamelCase__		)
       return spec_out







class _lowerCAmelCase							( nn.Module      ):







   def __init__(      self		,      UpperCamelCase__		,      UpperCamelCase__		,      UpperCamelCase__		,      UpperCamelCase__		,      UpperCamelCase__		,      UpperCamelCase__=1e-6		)      ->						str:

       '''simple docstring'''




       super().__init__()
       snake_case       : List[str]    =       nn.ModuleList()

       # cond self attention: layer 0
       self.layer.append(
           TaLayerSelfAttentionCond(d_model=UpperCamelCase__		,      d_kv=UpperCamelCase__		,      num_heads=UpperCamelCase__		,      dropout_rate=UpperCamelCase__		)		)

       # cross attention: layer 1
       self.layer.append(
           TaLayerCrossAttention(
               d_model=UpperCamelCase__		,      d_kv=UpperCamelCase__		,      num_heads=UpperCamelCase__		,      dropout_rate=UpperCamelCase__		,      layer_norm_epsilon=UpperCamelCase__		,      )		)

       # Film Cond MLP + dropout: last layer
       self.layer.append(
           TaLayerFFCond(d_model=UpperCamelCase__		,      d_ff=UpperCamelCase__		,      dropout_rate=UpperCamelCase__		,      layer_norm_epsilon=UpperCamelCase__		)		)







   def 					lowerCamelCase						(      self		,      UpperCamelCase__		,      UpperCamelCase__=None		,      UpperCamelCase__=None		,      UpperCamelCase__=None		,      UpperCamelCase__=None		,      UpperCamelCase__=None		,      )      ->						Dict:

       '''simple docstring'''




       snake_case       : Optional[int]    =       self.layer[0](
           UpperCamelCase__		,      conditioning_emb=UpperCamelCase__		,      attention_mask=UpperCamelCase__		,      )

       if encoder_hidden_states is not None:
           snake_case       : int    =       torch.where(encoder_attention_mask > 0		,      0		,      -1e10		).to(
               encoder_hidden_states.dtype		)

           snake_case       : str    =       self.layer[1](
               UpperCamelCase__		,      key_value_states=UpperCamelCase__		,      attention_mask=UpperCamelCase__		,      )

       # Apply Film Conditional Feed Forward layer
       snake_case       : Dict    =       self.layer[-1](UpperCamelCase__		,      UpperCamelCase__		)

       return (hidden_states,)







class _lowerCAmelCase							( nn.Module      ):







   def __init__(      self		,      UpperCamelCase__		,      UpperCamelCase__		,      UpperCamelCase__		,      UpperCamelCase__		)      ->						List[Any]:

       '''simple docstring'''




       super().__init__()
       snake_case       : int    =       TaLayerNorm(UpperCamelCase__		)
       snake_case       : int    =       TaFiLMLayer(in_features=d_model * 4		,      out_features=UpperCamelCase__		)
       snake_case       : str    =       Attention(query_dim=UpperCamelCase__		,      heads=UpperCamelCase__		,      dim_head=UpperCamelCase__		,      out_bias=UpperCamelCase__		,      scale_qk=UpperCamelCase__		)
       snake_case       : List[Any]    =       nn.Dropout(UpperCamelCase__		)







   def 					lowerCamelCase						(      self		,      UpperCamelCase__		,      UpperCamelCase__=None		,      UpperCamelCase__=None		,      )      ->						List[Any]:

       '''simple docstring'''




       snake_case       : Any    =       self.layer_norm(UpperCamelCase__		)

       if conditioning_emb is not None:
           snake_case       : Any    =       self.FiLMLayer(UpperCamelCase__		,      UpperCamelCase__		)

       # Self-attention block
       snake_case       : Optional[Any]    =       self.attention(UpperCamelCase__		)

       snake_case       : Dict    =       hidden_states + self.dropout(UpperCamelCase__		)

       return hidden_states







class _lowerCAmelCase							( nn.Module      ):







   def __init__(      self		,      UpperCamelCase__		,      UpperCamelCase__		,      UpperCamelCase__		,      UpperCamelCase__		,      UpperCamelCase__		)      ->						int:

       '''simple docstring'''




       super().__init__()
       snake_case       : int    =       Attention(query_dim=UpperCamelCase__		,      heads=UpperCamelCase__		,      dim_head=UpperCamelCase__		,      out_bias=UpperCamelCase__		,      scale_qk=UpperCamelCase__		)
       snake_case       : List[str]    =       TaLayerNorm(UpperCamelCase__		,      eps=UpperCamelCase__		)
       snake_case       : Optional[Any]    =       nn.Dropout(UpperCamelCase__		)







   def 					lowerCamelCase						(      self		,      UpperCamelCase__		,      UpperCamelCase__=None		,      UpperCamelCase__=None		,      )      ->						str:

       '''simple docstring'''




       snake_case       : List[str]    =       self.layer_norm(UpperCamelCase__		)
       snake_case       : Union[str, Any]    =       self.attention(
           UpperCamelCase__		,      encoder_hidden_states=UpperCamelCase__		,      attention_mask=attention_mask.squeeze(1		)		,      )
       snake_case       : Optional[Any]    =       hidden_states + self.dropout(UpperCamelCase__		)
       return layer_output







class _lowerCAmelCase							( nn.Module      ):







   def __init__(      self		,      UpperCamelCase__		,      UpperCamelCase__		,      UpperCamelCase__		,      UpperCamelCase__		)      ->						Union[str, Any]:

       '''simple docstring'''




       super().__init__()
       snake_case       : Optional[Any]    =       TaDenseGatedActDense(d_model=UpperCamelCase__		,      d_ff=UpperCamelCase__		,      dropout_rate=UpperCamelCase__		)
       snake_case       : List[str]    =       TaFiLMLayer(in_features=d_model * 4		,      out_features=UpperCamelCase__		)
       snake_case       : int    =       TaLayerNorm(UpperCamelCase__		,      eps=UpperCamelCase__		)
       snake_case       : int    =       nn.Dropout(UpperCamelCase__		)







   def 					lowerCamelCase						(      self		,      UpperCamelCase__		,      UpperCamelCase__=None		)      ->						Tuple:

       '''simple docstring'''




       snake_case       : str    =       self.layer_norm(UpperCamelCase__		)
       if conditioning_emb is not None:
           snake_case       : Union[str, Any]    =       self.film(UpperCamelCase__		,      UpperCamelCase__		)

       snake_case       : int    =       self.DenseReluDense(UpperCamelCase__		)
       snake_case       : Optional[Any]    =       hidden_states + self.dropout(UpperCamelCase__		)
       return hidden_states







class _lowerCAmelCase							( nn.Module      ):







   def __init__(      self		,      UpperCamelCase__		,      UpperCamelCase__		,      UpperCamelCase__		)      ->						Any:

       '''simple docstring'''




       super().__init__()
       snake_case       : Dict    =       nn.Linear(UpperCamelCase__		,      UpperCamelCase__		,      bias=UpperCamelCase__		)
       snake_case       : Tuple    =       nn.Linear(UpperCamelCase__		,      UpperCamelCase__		,      bias=UpperCamelCase__		)
       snake_case       : Optional[Any]    =       nn.Linear(UpperCamelCase__		,      UpperCamelCase__		,      bias=UpperCamelCase__		)
       snake_case       : Any    =       nn.Dropout(UpperCamelCase__		)
       snake_case       : Tuple    =       NewGELUActivation()







   def 					lowerCamelCase						(      self		,      UpperCamelCase__		)      ->						Optional[int]:

       '''simple docstring'''




       snake_case       : int    =       self.act(self.wi_a(UpperCamelCase__		)		)
       snake_case       : Tuple    =       self.wi_a(UpperCamelCase__		)
       snake_case       : Any    =       hidden_gelu * hidden_linear
       snake_case       : Union[str, Any]    =       self.dropout(UpperCamelCase__		)

       snake_case       : Optional[Any]    =       self.wo(UpperCamelCase__		)
       return hidden_states







class _lowerCAmelCase							( nn.Module      ):







   def __init__(      self		,      UpperCamelCase__		,      UpperCamelCase__=1e-6		)      ->						List[Any]:

       '''simple docstring'''




       super().__init__()
       snake_case       : Dict    =       nn.Parameter(torch.ones(UpperCamelCase__		)		)
       snake_case       : Any    =       eps







   def 					lowerCamelCase						(      self		,      UpperCamelCase__		)      ->						int:

       '''simple docstring'''




       snake_case       : List[str]    =       hidden_states.to(torch.floataa		).pow(2		).mean(-1		,      keepdim=UpperCamelCase__		)
       snake_case       : Optional[int]    =       hidden_states * torch.rsqrt(variance + self.variance_epsilon		)

       # convert into half-precision if necessary
       if self.weight.dtype in [torch.floataa, torch.bfloataa]:
           snake_case       : List[str]    =       hidden_states.to(self.weight.dtype		)

       return self.weight * hidden_states







class _lowerCAmelCase							( nn.Module      ):







   def 					lowerCamelCase						(      self		,      UpperCamelCase__		)      ->						torch.Tensor:

       '''simple docstring'''




       return 0.5 * input * (1.0 + torch.tanh(math.sqrt(2.0 / math.pi		) * (input + 0.044715 * torch.pow(UpperCamelCase__		,      3.0		))		))







class _lowerCAmelCase							( nn.Module      ):







   def __init__(      self		,      UpperCamelCase__		,      UpperCamelCase__		)      ->						Any:

       '''simple docstring'''




       super().__init__()
       snake_case       : str    =       nn.Linear(UpperCamelCase__		,      out_features * 2		,      bias=UpperCamelCase__		)







   def 					lowerCamelCase						(      self		,      UpperCamelCase__		,      UpperCamelCase__		)      ->						List[Any]:

       '''simple docstring'''




       snake_case       : str    =       self.scale_bias(UpperCamelCase__		)
       snake_case   ,snake_case       : int    =       torch.chunk(UpperCamelCase__		,      2		,      -1		)
       snake_case       : List[Any]    =       x * (1 + scale) + shift
       return x









"""simple docstring"""



import warnings

from ...utils import logging
from .image_processing_segformer import SegformerImageProcessor


__snake_case       =       logging.get_logger(__name__)







class _lowerCAmelCase							( snake_case_      ):







   def __init__(      self		,      *UpperCamelCase__		,      **UpperCamelCase__		)      ->						None:

       '''simple docstring'''




       warnings.warn(
           "The class SegformerFeatureExtractor is deprecated and will be removed in version 5 of Transformers."
           " Please use SegformerImageProcessor instead."		,      UpperCamelCase__		,      )
       super().__init__(*UpperCamelCase__		,      **UpperCamelCase__		)





