






import math

import torch
from torch import nn

from ..configuration_utils import ConfigMixin, register_to_config
from .attention_processor import Attention
from .embeddings import get_timestep_embedding
from .modeling_utils import ModelMixin



class 				_UpperCamelCase      (a_				,		a_      ):


      @register_to_config
      def __init__(						self       ,    __UpperCamelCase = 1_2_8       ,    __UpperCamelCase = 2_5_6       ,    __UpperCamelCase = 2_0_0_0.0       ,    __UpperCamelCase = 7_6_8       ,    __UpperCamelCase = 1_2       ,    __UpperCamelCase = 1_2       ,    __UpperCamelCase = 6_4       ,    __UpperCamelCase = 2_0_4_8       ,    __UpperCamelCase = 0.1       ,    )->						Tuple:
           super().__init__()

           __lowerCAmelCase             =       nn.Sequential(
               nn.Linear(__UpperCamelCase       ,    d_model * 4       ,    bias=__UpperCamelCase						)       ,    nn.SiLU()       ,    nn.Linear(d_model * 4       ,    d_model * 4       ,    bias=__UpperCamelCase						)       ,    nn.SiLU()       ,    )

           __lowerCAmelCase             =       nn.Embedding(__UpperCamelCase       ,    __UpperCamelCase						)
           __lowerCAmelCase             =       False

           __lowerCAmelCase             =       nn.Linear(__UpperCamelCase       ,    __UpperCamelCase       ,    bias=__UpperCamelCase						)

           __lowerCAmelCase             =       nn.Dropout(p=__UpperCamelCase						)

           __lowerCAmelCase             =       nn.ModuleList()
           for lyr_num in range(__UpperCamelCase						):
                # FiLM conditional T5 decoder
                __lowerCAmelCase             =       DecoderLayer(d_model=__UpperCamelCase       ,    d_kv=__UpperCamelCase       ,    num_heads=__UpperCamelCase       ,    d_ff=__UpperCamelCase       ,    dropout_rate=__UpperCamelCase						)
                self.decoders.append(__UpperCamelCase						)

           __lowerCAmelCase             =       TaLayerNorm(__UpperCamelCase						)

           __lowerCAmelCase             =       nn.Dropout(p=__UpperCamelCase						)
           __lowerCAmelCase             =       nn.Linear(__UpperCamelCase       ,    __UpperCamelCase       ,    bias=__UpperCamelCase						)


      def 			__UpperCAmelCase		(						self       ,    __UpperCamelCase       ,    __UpperCamelCase						)->						str:
           __lowerCAmelCase             =       torch.mul(query_input.unsqueeze(-1						)       ,    key_input.unsqueeze(-2						)						)
           return mask.unsqueeze(-3						)


      def 			__UpperCAmelCase		(						self       ,    __UpperCamelCase       ,    __UpperCamelCase       ,    __UpperCamelCase						)->						Any:
           __lowerCAmelCase      ,			__lowerCAmelCase      ,			__lowerCAmelCase             =       decoder_input_tokens.shape
           assert decoder_noise_time.shape == (batch,)

           # decoder_noise_time is in [0, 1), so rescale to expected timing range.
           __lowerCAmelCase             =       get_timestep_embedding(
               decoder_noise_time * self.config.max_decoder_noise_time       ,    embedding_dim=self.config.d_model       ,    max_period=self.config.max_decoder_noise_time       ,    ).to(dtype=self.dtype						)

           __lowerCAmelCase             =       self.conditioning_emb(__UpperCamelCase						).unsqueeze(1						)

           assert conditioning_emb.shape == (batch, 1, self.config.d_model * 4)

           __lowerCAmelCase             =       decoder_input_tokens.shape[1]

           # If we want to use relative positions for audio context, we can just offset
           # this sequence by the length of encodings_and_masks.
           __lowerCAmelCase             =       torch.broadcast_to(
               torch.arange(__UpperCamelCase       ,    device=decoder_input_tokens.device						)       ,    (batch, seq_length)       ,    )

           __lowerCAmelCase             =       self.position_encoding(__UpperCamelCase						)

           __lowerCAmelCase             =       self.continuous_inputs_projection(__UpperCamelCase						)
           inputs += position_encodings
           __lowerCAmelCase             =       self.dropout(__UpperCamelCase						)

           # decoder: No padding present.
           __lowerCAmelCase             =       torch.ones(
               decoder_input_tokens.shape[:2]       ,    device=decoder_input_tokens.device       ,    dtype=inputs.dtype						)

           # Translate encoding masks to encoder-decoder masks.
           __lowerCAmelCase             =       [(x, self.encoder_decoder_mask(__UpperCamelCase       ,    __UpperCamelCase						)) for x, y in encodings_and_masks]

           # cross attend style: concat encodings
           __lowerCAmelCase             =       torch.cat([x[0] for x in encodings_and_encdec_masks]       ,    dim=1						)
           __lowerCAmelCase             =       torch.cat([x[1] for x in encodings_and_encdec_masks]       ,    dim=-1						)

           for lyr in self.decoders:
                __lowerCAmelCase             =       lyr(
                    __UpperCamelCase       ,    conditioning_emb=__UpperCamelCase       ,    encoder_hidden_states=__UpperCamelCase       ,    encoder_attention_mask=__UpperCamelCase       ,    )[0]

           __lowerCAmelCase             =       self.decoder_norm(__UpperCamelCase						)
           __lowerCAmelCase             =       self.post_dropout(__UpperCamelCase						)

           __lowerCAmelCase             =       self.spec_out(__UpperCamelCase						)
           return spec_out



class 				_UpperCamelCase      (nn.Module      ):


      def __init__(						self       ,    __UpperCamelCase       ,    __UpperCamelCase       ,    __UpperCamelCase       ,    __UpperCamelCase       ,    __UpperCamelCase       ,    __UpperCamelCase=1e-6						)->						List[Any]:
           super().__init__()
           __lowerCAmelCase             =       nn.ModuleList()

           # cond self attention: layer 0
           self.layer.append(
               TaLayerSelfAttentionCond(d_model=__UpperCamelCase       ,    d_kv=__UpperCamelCase       ,    num_heads=__UpperCamelCase       ,    dropout_rate=__UpperCamelCase						)						)

           # cross attention: layer 1
           self.layer.append(
               TaLayerCrossAttention(
                   d_model=__UpperCamelCase       ,    d_kv=__UpperCamelCase       ,    num_heads=__UpperCamelCase       ,    dropout_rate=__UpperCamelCase       ,    layer_norm_epsilon=__UpperCamelCase       ,    )						)

           # Film Cond MLP + dropout: last layer
           self.layer.append(
               TaLayerFFCond(d_model=__UpperCamelCase       ,    d_ff=__UpperCamelCase       ,    dropout_rate=__UpperCamelCase       ,    layer_norm_epsilon=__UpperCamelCase						)						)


      def 			__UpperCAmelCase		(						self       ,    __UpperCamelCase       ,    __UpperCamelCase=None       ,    __UpperCamelCase=None       ,    __UpperCamelCase=None       ,    __UpperCamelCase=None       ,    __UpperCamelCase=None       ,    )->						List[Any]:
           __lowerCAmelCase             =       self.layer[0](
               __UpperCamelCase       ,    conditioning_emb=__UpperCamelCase       ,    attention_mask=__UpperCamelCase       ,    )

           if encoder_hidden_states is not None:
                __lowerCAmelCase             =       torch.where(encoder_attention_mask > 0       ,    0       ,    -1e10						).to(
                    encoder_hidden_states.dtype						)

                __lowerCAmelCase             =       self.layer[1](
                    __UpperCamelCase       ,    key_value_states=__UpperCamelCase       ,    attention_mask=__UpperCamelCase       ,    )

           # Apply Film Conditional Feed Forward layer
           __lowerCAmelCase             =       self.layer[-1](__UpperCamelCase       ,    __UpperCamelCase						)

           return (hidden_states,)



class 				_UpperCamelCase      (nn.Module      ):


      def __init__(						self       ,    __UpperCamelCase       ,    __UpperCamelCase       ,    __UpperCamelCase       ,    __UpperCamelCase						)->						List[Any]:
           super().__init__()
           __lowerCAmelCase             =       TaLayerNorm(__UpperCamelCase						)
           __lowerCAmelCase             =       TaFiLMLayer(in_features=d_model * 4       ,    out_features=__UpperCamelCase						)
           __lowerCAmelCase             =       Attention(query_dim=__UpperCamelCase       ,    heads=__UpperCamelCase       ,    dim_head=__UpperCamelCase       ,    out_bias=__UpperCamelCase       ,    scale_qk=__UpperCamelCase						)
           __lowerCAmelCase             =       nn.Dropout(__UpperCamelCase						)


      def 			__UpperCAmelCase		(						self       ,    __UpperCamelCase       ,    __UpperCamelCase=None       ,    __UpperCamelCase=None       ,    )->						Tuple:
           # pre_self_attention_layer_norm
           __lowerCAmelCase             =       self.layer_norm(__UpperCamelCase						)

           if conditioning_emb is not None:
                __lowerCAmelCase             =       self.FiLMLayer(__UpperCamelCase       ,    __UpperCamelCase						)

           # Self-attention block
           __lowerCAmelCase             =       self.attention(__UpperCamelCase						)

           __lowerCAmelCase             =       hidden_states + self.dropout(__UpperCamelCase						)

           return hidden_states



class 				_UpperCamelCase      (nn.Module      ):


      def __init__(						self       ,    __UpperCamelCase       ,    __UpperCamelCase       ,    __UpperCamelCase       ,    __UpperCamelCase       ,    __UpperCamelCase						)->						Any:
           super().__init__()
           __lowerCAmelCase             =       Attention(query_dim=__UpperCamelCase       ,    heads=__UpperCamelCase       ,    dim_head=__UpperCamelCase       ,    out_bias=__UpperCamelCase       ,    scale_qk=__UpperCamelCase						)
           __lowerCAmelCase             =       TaLayerNorm(__UpperCamelCase       ,    eps=__UpperCamelCase						)
           __lowerCAmelCase             =       nn.Dropout(__UpperCamelCase						)


      def 			__UpperCAmelCase		(						self       ,    __UpperCamelCase       ,    __UpperCamelCase=None       ,    __UpperCamelCase=None       ,    )->						Optional[Any]:
           __lowerCAmelCase             =       self.layer_norm(__UpperCamelCase						)
           __lowerCAmelCase             =       self.attention(
               __UpperCamelCase       ,    encoder_hidden_states=__UpperCamelCase       ,    attention_mask=attention_mask.squeeze(1						)       ,    )
           __lowerCAmelCase             =       hidden_states + self.dropout(__UpperCamelCase						)
           return layer_output



class 				_UpperCamelCase      (nn.Module      ):


      def __init__(						self       ,    __UpperCamelCase       ,    __UpperCamelCase       ,    __UpperCamelCase       ,    __UpperCamelCase						)->						List[str]:
           super().__init__()
           __lowerCAmelCase             =       TaDenseGatedActDense(d_model=__UpperCamelCase       ,    d_ff=__UpperCamelCase       ,    dropout_rate=__UpperCamelCase						)
           __lowerCAmelCase             =       TaFiLMLayer(in_features=d_model * 4       ,    out_features=__UpperCamelCase						)
           __lowerCAmelCase             =       TaLayerNorm(__UpperCamelCase       ,    eps=__UpperCamelCase						)
           __lowerCAmelCase             =       nn.Dropout(__UpperCamelCase						)


      def 			__UpperCAmelCase		(						self       ,    __UpperCamelCase       ,    __UpperCamelCase=None						)->						Optional[Any]:
           __lowerCAmelCase             =       self.layer_norm(__UpperCamelCase						)
           if conditioning_emb is not None:
                __lowerCAmelCase             =       self.film(__UpperCamelCase       ,    __UpperCamelCase						)

           __lowerCAmelCase             =       self.DenseReluDense(__UpperCamelCase						)
           __lowerCAmelCase             =       hidden_states + self.dropout(__UpperCamelCase						)
           return hidden_states



class 				_UpperCamelCase      (nn.Module      ):


      def __init__(						self       ,    __UpperCamelCase       ,    __UpperCamelCase       ,    __UpperCamelCase						)->						List[str]:
           super().__init__()
           __lowerCAmelCase             =       nn.Linear(__UpperCamelCase       ,    __UpperCamelCase       ,    bias=__UpperCamelCase						)
           __lowerCAmelCase             =       nn.Linear(__UpperCamelCase       ,    __UpperCamelCase       ,    bias=__UpperCamelCase						)
           __lowerCAmelCase             =       nn.Linear(__UpperCamelCase       ,    __UpperCamelCase       ,    bias=__UpperCamelCase						)
           __lowerCAmelCase             =       nn.Dropout(__UpperCamelCase						)
           __lowerCAmelCase             =       NewGELUActivation()


      def 			__UpperCAmelCase		(						self       ,    __UpperCamelCase						)->						Any:
           __lowerCAmelCase             =       self.act(self.wi_a(__UpperCamelCase						)						)
           __lowerCAmelCase             =       self.wi_a(__UpperCamelCase						)
           __lowerCAmelCase             =       hidden_gelu * hidden_linear
           __lowerCAmelCase             =       self.dropout(__UpperCamelCase						)

           __lowerCAmelCase             =       self.wo(__UpperCamelCase						)
           return hidden_states



class 				_UpperCamelCase      (nn.Module      ):


      def __init__(						self       ,    __UpperCamelCase       ,    __UpperCamelCase=1e-6						)->						List[Any]:
           super().__init__()
           __lowerCAmelCase             =       nn.Parameter(torch.ones(__UpperCamelCase						)						)
           __lowerCAmelCase             =       eps


      def 			__UpperCAmelCase		(						self       ,    __UpperCamelCase						)->						List[Any]:
           # T5 uses a layer_norm which only scales and doesn't shift, which is also known as Root Mean
           # Square Layer Normalization https://arxiv.org/abs/1910.07467 thus variance is calculated
           # w/o mean and there is no bias. Additionally we want to make sure that the accumulation for
           # half-precision inputs is done in fp32

           __lowerCAmelCase             =       hidden_states.to(torch.floataa						).pow(2						).mean(-1       ,    keepdim=__UpperCamelCase						)
           __lowerCAmelCase             =       hidden_states * torch.rsqrt(variance + self.variance_epsilon						)

           # convert into half-precision if necessary
           if self.weight.dtype in [torch.floataa, torch.bfloataa]:
                __lowerCAmelCase             =       hidden_states.to(self.weight.dtype						)

           return self.weight * hidden_states



class 				_UpperCamelCase      (nn.Module      ):


      def 			__UpperCAmelCase		(						self       ,    __UpperCamelCase						)->						torch.Tensor:
           return 0.5 * input * (1.0 + torch.tanh(math.sqrt(2.0 / math.pi						) * (input + 0.0_4_4_7_1_5 * torch.pow(__UpperCamelCase       ,    3.0						))						))



class 				_UpperCamelCase      (nn.Module      ):


      def __init__(						self       ,    __UpperCamelCase       ,    __UpperCamelCase						)->						Dict:
           super().__init__()
           __lowerCAmelCase             =       nn.Linear(__UpperCamelCase       ,    out_features * 2       ,    bias=__UpperCamelCase						)


      def 			__UpperCAmelCase		(						self       ,    __UpperCamelCase       ,    __UpperCamelCase						)->						Any:
           __lowerCAmelCase             =       self.scale_bias(__UpperCamelCase						)
           __lowerCAmelCase      ,			__lowerCAmelCase             =       torch.chunk(__UpperCamelCase       ,    2       ,    -1						)
           __lowerCAmelCase             =       x * (1 + scale) + shift
           return x














import itertools
import string
from collections.abc import Generator, Iterable





def    __lowerCAmelCase   (    __snake_case       ,			__snake_case							):
     __lowerCAmelCase             =       iter(__snake_case							)
     while True:
          __lowerCAmelCase             =       tuple(itertools.islice(__snake_case       ,			__snake_case							)							)
          if not chunk:
               return
          yield chunk





def    __lowerCAmelCase   (    __snake_case							):
     __lowerCAmelCase             =       "".join([c.upper() for c in dirty if c in string.ascii_letters]							)
     __lowerCAmelCase             =       ""

     if len(__snake_case							) < 2:
          return dirty

     for i in range(len(__snake_case							) - 1							):
          clean += dirty[i]

          if dirty[i] == dirty[i + 1]:
               clean += "X"

     clean += dirty[-1]

     if len(__snake_case							) & 1:
          clean += "X"

     return clean





def    __lowerCAmelCase   (    __snake_case							):
     # I and J are used interchangeably to allow
     # us to use a 5x5 table (25 letters)
     __lowerCAmelCase             =       "ABCDEFGHIKLMNOPQRSTUVWXYZ"
     # we're using a list instead of a '2d' array because it makes the math
     # for setting up the table and doing the actual encoding/decoding simpler
     __lowerCAmelCase             =       []

     # copy key chars into the table if they are in `alphabet` ignoring duplicates
     for char in key.upper():
          if char not in table and char in alphabet:
               table.append(__snake_case							)

    # fill the rest of the table in with the remaining alphabet chars
     for char in alphabet:
          if char not in table:
               table.append(__snake_case							)

     return table





def    __lowerCAmelCase   (    __snake_case       ,			__snake_case							):
     __lowerCAmelCase             =       generate_table(__snake_case							)
     __lowerCAmelCase             =       prepare_input(__snake_case							)
     __lowerCAmelCase             =       ""

     # https://en.wikipedia.org/wiki/Playfair_cipher#Description
     for chara, chara in chunker(__snake_case       ,			2							):
          __lowerCAmelCase      ,			__lowerCAmelCase             =       divmod(table.index(__snake_case							)       ,			5							)
          __lowerCAmelCase      ,			__lowerCAmelCase             =       divmod(table.index(__snake_case							)       ,			5							)

          if rowa == rowa:
               ciphertext += table[rowa * 5 + (cola + 1) % 5]
               ciphertext += table[rowa * 5 + (cola + 1) % 5]
          elif cola == cola:
               ciphertext += table[((rowa + 1) % 5) * 5 + cola]
               ciphertext += table[((rowa + 1) % 5) * 5 + cola]
          else:  # rectangle
               ciphertext += table[rowa * 5 + cola]
               ciphertext += table[rowa * 5 + cola]

     return ciphertext





def    __lowerCAmelCase   (    __snake_case       ,			__snake_case							):
     __lowerCAmelCase             =       generate_table(__snake_case							)
     __lowerCAmelCase             =       ""

     # https://en.wikipedia.org/wiki/Playfair_cipher#Description
     for chara, chara in chunker(__snake_case       ,			2							):
          __lowerCAmelCase      ,			__lowerCAmelCase             =       divmod(table.index(__snake_case							)       ,			5							)
          __lowerCAmelCase      ,			__lowerCAmelCase             =       divmod(table.index(__snake_case							)       ,			5							)

          if rowa == rowa:
               plaintext += table[rowa * 5 + (cola - 1) % 5]
               plaintext += table[rowa * 5 + (cola - 1) % 5]
          elif cola == cola:
               plaintext += table[((rowa - 1) % 5) * 5 + cola]
               plaintext += table[((rowa - 1) % 5) * 5 + cola]
          else:  # rectangle
               plaintext += table[rowa * 5 + cola]
               plaintext += table[rowa * 5 + cola]

     return plaintext






