






import math

import torch
from torch import nn

from ..configuration_utils import ConfigMixin, register_to_config
from .attention_processor import Attention
from .embeddings import get_timestep_embedding
from .modeling_utils import ModelMixin






class 						UpperCAmelCase__ (							__snake_case     ,				__snake_case							):







		@register_to_config
		def __init__(					self       ,A__ = 128       ,A__ = 256       ,A__ = 20_00.0       ,A__ = 768       ,A__ = 12       ,A__ = 12       ,A__ = 64       ,A__ = 2048       ,A__ = 0.1       ,):
						super().__init__()

						_A	:				List[str]   =      nn.Sequential(
						    nn.Linear(A__       ,d_model * 4       ,bias=A__		)       ,nn.SiLU()       ,nn.Linear(d_model * 4       ,d_model * 4       ,bias=A__		)       ,nn.SiLU()       ,)

						_A	:				List[str]   =      nn.Embedding(A__       ,A__		)
						_A	:				Union[str, Any]   =      False

						_A	:				Optional[Any]   =      nn.Linear(A__       ,A__       ,bias=A__		)

						_A	:				Tuple   =      nn.Dropout(p=A__		)

						_A	:				Tuple   =      nn.ModuleList()
						for lyr_num in range(A__		):
										# FiLM conditional T5 decoder
										_A	:				int   =      DecoderLayer(d_model=A__       ,d_kv=A__       ,num_heads=A__       ,d_ff=A__       ,dropout_rate=A__		)
										self.decoders.append(A__		)

						_A	:				Dict   =      TaLayerNorm(A__		)

						_A	:				Union[str, Any]   =      nn.Dropout(p=A__		)
						_A	:				Optional[int]   =      nn.Linear(A__       ,A__       ,bias=A__		)







		def  A__      (					self       ,A__       ,A__		):
						_A	:				List[Any]   =      torch.mul(query_input.unsqueeze(-1		)       ,key_input.unsqueeze(-2		)		)
						return mask.unsqueeze(-3		)







		def  A__      (					self       ,A__       ,A__       ,A__		):
						_A  ,				_A  ,				_A	:				List[str]   =      decoder_input_tokens.shape
						assert decoder_noise_time.shape == (batch,)

						# decoder_noise_time is in [0, 1), so rescale to expected timing range.
						_A	:				List[str]   =      get_timestep_embedding(
						    decoder_noise_time * self.config.max_decoder_noise_time       ,embedding_dim=self.config.d_model       ,max_period=self.config.max_decoder_noise_time       ,).to(dtype=self.dtype		)

						_A	:				int   =      self.conditioning_emb(A__		).unsqueeze(1		)

						assert conditioning_emb.shape == (batch, 1, self.config.d_model * 4)

						_A	:				int   =      decoder_input_tokens.shape[1]

						# If we want to use relative positions for audio context, we can just offset
						# this sequence by the length of encodings_and_masks.
						_A	:				Optional[int]   =      torch.broadcast_to(
						    torch.arange(A__       ,device=decoder_input_tokens.device		)       ,(batch, seq_length)       ,)

						_A	:				Optional[int]   =      self.position_encoding(A__		)

						_A	:				str   =      self.continuous_inputs_projection(A__		)
						inputs += position_encodings
						_A	:				Optional[int]   =      self.dropout(A__		)

						# decoder: No padding present.
						_A	:				Optional[int]   =      torch.ones(
						    decoder_input_tokens.shape[:2]       ,device=decoder_input_tokens.device       ,dtype=inputs.dtype		)

						# Translate encoding masks to encoder-decoder masks.
						_A	:				int   =      [(x, self.encoder_decoder_mask(A__       ,A__		)) for x, y in encodings_and_masks]

						# cross attend style: concat encodings
						_A	:				List[Any]   =      torch.cat([x[0] for x in encodings_and_encdec_masks]       ,dim=1		)
						_A	:				Union[str, Any]   =      torch.cat([x[1] for x in encodings_and_encdec_masks]       ,dim=-1		)

						for lyr in self.decoders:
										_A	:				Tuple   =      lyr(
										    A__       ,conditioning_emb=A__       ,encoder_hidden_states=A__       ,encoder_attention_mask=A__       ,)[0]

						_A	:				List[Any]   =      self.decoder_norm(A__		)
						_A	:				List[str]   =      self.post_dropout(A__		)

						_A	:				int   =      self.spec_out(A__		)
						return spec_out






class 						UpperCAmelCase__ (							nn.Module							):







		def __init__(					self       ,A__       ,A__       ,A__       ,A__       ,A__       ,A__=1E-6		):
						super().__init__()
						_A	:				List[str]   =      nn.ModuleList()

						# cond self attention: layer 0
						self.layer.append(
						    TaLayerSelfAttentionCond(d_model=A__       ,d_kv=A__       ,num_heads=A__       ,dropout_rate=A__		)		)

						# cross attention: layer 1
						self.layer.append(
						    TaLayerCrossAttention(
						        d_model=A__       ,d_kv=A__       ,num_heads=A__       ,dropout_rate=A__       ,layer_norm_epsilon=A__       ,)		)

						# Film Cond MLP + dropout: last layer
						self.layer.append(
						    TaLayerFFCond(d_model=A__       ,d_ff=A__       ,dropout_rate=A__       ,layer_norm_epsilon=A__		)		)







		def  A__      (					self       ,A__       ,A__=None       ,A__=None       ,A__=None       ,A__=None       ,A__=None       ,):
						_A	:				Dict   =      self.layer[0](
						    A__       ,conditioning_emb=A__       ,attention_mask=A__       ,)

						if encoder_hidden_states is not None:
										_A	:				Dict   =      torch.where(encoder_attention_mask > 0       ,0       ,-1E10		).to(
										    encoder_hidden_states.dtype		)

										_A	:				int   =      self.layer[1](
										    A__       ,key_value_states=A__       ,attention_mask=A__       ,)

						# Apply Film Conditional Feed Forward layer
						_A	:				Optional[int]   =      self.layer[-1](A__       ,A__		)

						return (hidden_states,)






class 						UpperCAmelCase__ (							nn.Module							):







		def __init__(					self       ,A__       ,A__       ,A__       ,A__		):
						super().__init__()
						_A	:				Optional[Any]   =      TaLayerNorm(A__		)
						_A	:				Union[str, Any]   =      TaFiLMLayer(in_features=d_model * 4       ,out_features=A__		)
						_A	:				List[Any]   =      Attention(query_dim=A__       ,heads=A__       ,dim_head=A__       ,out_bias=A__       ,scale_qk=A__		)
						_A	:				Union[str, Any]   =      nn.Dropout(A__		)







		def  A__      (					self       ,A__       ,A__=None       ,A__=None       ,):
						# pre_self_attention_layer_norm
						_A	:				List[str]   =      self.layer_norm(A__		)

						if conditioning_emb is not None:
										_A	:				List[str]   =      self.FiLMLayer(A__       ,A__		)

						# Self-attention block
						_A	:				List[Any]   =      self.attention(A__		)

						_A	:				Optional[Any]   =      hidden_states + self.dropout(A__		)

						return hidden_states






class 						UpperCAmelCase__ (							nn.Module							):







		def __init__(					self       ,A__       ,A__       ,A__       ,A__       ,A__		):
						super().__init__()
						_A	:				List[str]   =      Attention(query_dim=A__       ,heads=A__       ,dim_head=A__       ,out_bias=A__       ,scale_qk=A__		)
						_A	:				Any   =      TaLayerNorm(A__       ,eps=A__		)
						_A	:				Optional[Any]   =      nn.Dropout(A__		)







		def  A__      (					self       ,A__       ,A__=None       ,A__=None       ,):
						_A	:				List[str]   =      self.layer_norm(A__		)
						_A	:				List[Any]   =      self.attention(
						    A__       ,encoder_hidden_states=A__       ,attention_mask=attention_mask.squeeze(1		)       ,)
						_A	:				Dict   =      hidden_states + self.dropout(A__		)
						return layer_output






class 						UpperCAmelCase__ (							nn.Module							):







		def __init__(					self       ,A__       ,A__       ,A__       ,A__		):
						super().__init__()
						_A	:				Dict   =      TaDenseGatedActDense(d_model=A__       ,d_ff=A__       ,dropout_rate=A__		)
						_A	:				str   =      TaFiLMLayer(in_features=d_model * 4       ,out_features=A__		)
						_A	:				int   =      TaLayerNorm(A__       ,eps=A__		)
						_A	:				Tuple   =      nn.Dropout(A__		)







		def  A__      (					self       ,A__       ,A__=None		):
						_A	:				str   =      self.layer_norm(A__		)
						if conditioning_emb is not None:
										_A	:				str   =      self.film(A__       ,A__		)

						_A	:				int   =      self.DenseReluDense(A__		)
						_A	:				str   =      hidden_states + self.dropout(A__		)
						return hidden_states






class 						UpperCAmelCase__ (							nn.Module							):







		def __init__(					self       ,A__       ,A__       ,A__		):
						super().__init__()
						_A	:				List[str]   =      nn.Linear(A__       ,A__       ,bias=A__		)
						_A	:				Union[str, Any]   =      nn.Linear(A__       ,A__       ,bias=A__		)
						_A	:				Any   =      nn.Linear(A__       ,A__       ,bias=A__		)
						_A	:				Optional[int]   =      nn.Dropout(A__		)
						_A	:				str   =      NewGELUActivation()







		def  A__      (					self       ,A__		):
						_A	:				List[str]   =      self.act(self.wi_a(A__		)		)
						_A	:				Optional[int]   =      self.wi_a(A__		)
						_A	:				Tuple   =      hidden_gelu * hidden_linear
						_A	:				Optional[int]   =      self.dropout(A__		)

						_A	:				Optional[int]   =      self.wo(A__		)
						return hidden_states






class 						UpperCAmelCase__ (							nn.Module							):







		def __init__(					self       ,A__       ,A__=1E-6		):
						super().__init__()
						_A	:				List[str]   =      nn.Parameter(torch.ones(A__		)		)
						_A	:				Tuple   =      eps







		def  A__      (					self       ,A__		):
						# T5 uses a layer_norm which only scales and doesn't shift, which is also known as Root Mean
						# Square Layer Normalization https://arxiv.org/abs/1910.07467 thus variance is calculated
						# w/o mean and there is no bias. Additionally we want to make sure that the accumulation for
						# half-precision inputs is done in fp32

						_A	:				Optional[Any]   =      hidden_states.to(torch.floataa		).pow(2		).mean(-1       ,keepdim=A__		)
						_A	:				int   =      hidden_states * torch.rsqrt(variance + self.variance_epsilon		)

						# convert into half-precision if necessary
						if self.weight.dtype in [torch.floataa, torch.bfloataa]:
										_A	:				List[Any]   =      hidden_states.to(self.weight.dtype		)

						return self.weight * hidden_states






class 						UpperCAmelCase__ (							nn.Module							):







		def  A__      (					self       ,A__		):
						return 0.5 * input * (1.0 + torch.tanh(math.sqrt(2.0 / math.pi		) * (input + 0.04_47_15 * torch.pow(A__       ,3.0		))		))






class 						UpperCAmelCase__ (							nn.Module							):







		def __init__(					self       ,A__       ,A__		):
						super().__init__()
						_A	:				str   =      nn.Linear(A__       ,out_features * 2       ,bias=A__		)







		def  A__      (					self       ,A__       ,A__		):
						_A	:				Union[str, Any]   =      self.scale_bias(A__		)
						_A  ,				_A	:				int   =      torch.chunk(A__       ,2       ,-1		)
						_A	:				Dict   =      x * (1 + scale) + shift
						return x











from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


_UpperCamelCase			:	Any								={
    'configuration_megatron_bert': ['MEGATRON_BERT_PRETRAINED_CONFIG_ARCHIVE_MAP', 'MegatronBertConfig'],
}

try:
	if not is_torch_available():
		raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
	pass
else:
	_UpperCamelCase			:	Optional[Any]								=[
	    'MEGATRON_BERT_PRETRAINED_MODEL_ARCHIVE_LIST',
	    'MegatronBertForCausalLM',
	    'MegatronBertForMaskedLM',
	    'MegatronBertForMultipleChoice',
	    'MegatronBertForNextSentencePrediction',
	    'MegatronBertForPreTraining',
	    'MegatronBertForQuestionAnswering',
	    'MegatronBertForSequenceClassification',
	    'MegatronBertForTokenClassification',
	    'MegatronBertModel',
	    'MegatronBertPreTrainedModel',
	]

if TYPE_CHECKING:
	from .configuration_megatron_bert import MEGATRON_BERT_PRETRAINED_CONFIG_ARCHIVE_MAP, MegatronBertConfig

	try:
		if not is_torch_available():
			raise OptionalDependencyNotAvailable()
	except OptionalDependencyNotAvailable:
		pass
	else:
		from .modeling_megatron_bert import (
		    MEGATRON_BERT_PRETRAINED_MODEL_ARCHIVE_LIST,
		    MegatronBertForCausalLM,
		    MegatronBertForMaskedLM,
		    MegatronBertForMultipleChoice,
		    MegatronBertForNextSentencePrediction,
		    MegatronBertForPreTraining,
		    MegatronBertForQuestionAnswering,
		    MegatronBertForSequenceClassification,
		    MegatronBertForTokenClassification,
		    MegatronBertModel,
		    MegatronBertPreTrainedModel,
		)

else:
	import sys

	_UpperCamelCase			:	Optional[int]								=_LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)



