from google.protobuf import descriptor as _descriptor
from google.protobuf import descriptor_pool as _descriptor_pool
from google.protobuf import symbol_database as _symbol_database
from google.protobuf.internal import builder as _builder


# @@protoc_insertion_point(imports)

__A  =				_symbol_database.Default()


__A  =				_descriptor_pool.Default().AddSerializedFile(
    B'''\n\x19sentencepiece_model.proto\x12\rsentencepiece"\x80\x0c\n\x0bTrainerSpec\x12\r\n\x05input\x18\x01 \x03(\t\x12\x14\n\x0cinput_format\x18\x07 \x01(\t\x12\x14\n\x0cmodel_prefix\x18\x02 \x01(\t\x12\x41\n\nmodel_type\x18\x03 \x01(\x0e\x32$.sentencepiece.TrainerSpec.ModelType:\x07UNIGRAM\x12\x18\n\nvocab_size\x18\x04 \x01(\x05:\x04\x38\x30\x30\x30\x12\x17\n\x0f\x61\x63\x63\x65pt_language\x18\x05 \x03(\t\x12 \n\x15self_test_sample_size\x18\x06 \x01(\x05:\x01\x30\x12*\n\x1b\x65nable_differential_privacy\x18\x32 \x01(\x08:\x05\x66\x61lse\x12+\n differential_privacy_noise_level\x18\x33 \x01(\x02:\x01\x30\x12\x32\n\'differential_privacy_clipping_threshold\x18\x34 \x01(\x04:\x01\x30\x12"\n\x12\x63haracter_coverage\x18\n \x01(\x02:\x06\x30.9995\x12\x1e\n\x13input_sentence_size\x18\x0b \x01(\x04:\x01\x30\x12$\n\x16shuffle_input_sentence\x18\x13 \x01(\x08:\x04true\x12 \n\x14mining_sentence_size\x18\x0c \x01(\x05\x42\x02\x18\x01\x12"\n\x16training_sentence_size\x18\r \x01(\x05\x42\x02\x18\x01\x12(\n\x17seed_sentencepiece_size\x18\x0e \x01(\x05:\x07\x31\x30\x30\x30\x30\x30\x30\x12\x1e\n\x10shrinking_factor\x18\x0f \x01(\x02:\x04\x30.75\x12!\n\x13max_sentence_length\x18\x12 \x01(\x05:\x04\x34\x31\x39\x32\x12\x17\n\x0bnum_threads\x18\x10 \x01(\x05:\x02\x31\x36\x12\x1d\n\x12num_sub_iterations\x18\x11 \x01(\x05:\x01\x32\x12$\n\x18max_sentencepiece_length\x18\x14 \x01(\x05:\x02\x31\x36\x12%\n\x17split_by_unicode_script\x18\x15 \x01(\x08:\x04true\x12\x1d\n\x0fsplit_by_number\x18\x17 \x01(\x08:\x04true\x12!\n\x13split_by_whitespace\x18\x16 \x01(\x08:\x04true\x12)\n\x1atreat_whitespace_as_suffix\x18\x18 \x01(\x08:\x05\x66\x61lse\x12+\n\x1c\x61llow_whitespace_only_pieces\x18\x1a \x01(\x08:\x05\x66\x61lse\x12\x1b\n\x0csplit_digits\x18\x19 \x01(\x08:\x05\x66\x61lse\x12#\n\x19pretokenization_delimiter\x18\x35 \x01(\t:\x00\x12\x17\n\x0f\x63ontrol_symbols\x18\x1e \x03(\t\x12\x1c\n\x14user_defined_symbols\x18\x1f \x03(\t\x12\x16\n\x0erequired_chars\x18$ \x01(\t\x12\x1c\n\rbyte_fallback\x18# \x01(\x08:\x05\x66\x61lse\x12+\n\x1dvocabulary_output_piece_score\x18  \x01(\x08:\x04true\x12\x1e\n\x10hard_vocab_limit\x18! \x01(\x08:\x04true\x12\x1c\n\ruse_all_vocab\x18" \x01(\x08:\x05\x66\x61lse\x12\x11\n\x06unk_id\x18( \x01(\x05:\x01\x30\x12\x11\n\x06\x62os_id\x18) \x01(\x05:\x01\x31\x12\x11\n\x06\x65os_id\x18* \x01(\x05:\x01\x32\x12\x12\n\x06pad_id\x18+ \x01(\x05:\x02-1\x12\x18\n\tunk_piece\x18- \x01(\t:\x05<unk>\x12\x16\n\tbos_piece\x18. \x01(\t:\x03<s>\x12\x17\n\teos_piece\x18/ \x01(\t:\x04</s>\x12\x18\n\tpad_piece\x18\x30 \x01(\t:\x05<pad>\x12\x1a\n\x0bunk_surface\x18, \x01(\t:\x05 \xe2\x81\x87 \x12+\n\x1ctrain_extremely_large_corpus\x18\x31 \x01(\x08:\x05\x66\x61lse"5\n\tModelType\x12\x0b\n\x07UNIGRAM\x10\x01\x12\x07\n\x03\x42PE\x10\x02\x12\x08\n\x04WORD\x10\x03\x12\x08\n\x04\x43HAR\x10\x04*\t\x08\xc8\x01\x10\x80\x80\x80\x80\x02"\xd1\x01\n\x0eNormalizerSpec\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x1c\n\x14precompiled_charsmap\x18\x02 \x01(\x0c\x12\x1e\n\x10\x61\x64\x64_dummy_prefix\x18\x03 \x01(\x08:\x04true\x12&\n\x18remove_extra_whitespaces\x18\x04 \x01(\x08:\x04true\x12 \n\x12\x65scape_whitespaces\x18\x05 \x01(\x08:\x04true\x12\x1e\n\x16normalization_rule_tsv\x18\x06 \x01(\t*\t\x08\xc8\x01\x10\x80\x80\x80\x80\x02"y\n\x0cSelfTestData\x12\x33\n\x07samples\x18\x01 \x03(\x0b\x32".sentencepiece.SelfTestData.Sample\x1a)\n\x06Sample\x12\r\n\x05input\x18\x01 \x01(\t\x12\x10\n\x08\x65xpected\x18\x02 \x01(\t*\t\x08\xc8\x01\x10\x80\x80\x80\x80\x02"\xfe\x03\n\nModelProto\x12\x37\n\x06pieces\x18\x01 \x03(\x0b\x32\'.sentencepiece.ModelProto.SentencePiece\x12\x30\n\x0ctrainer_spec\x18\x02 \x01(\x0b\x32\x1a.sentencepiece.TrainerSpec\x12\x36\n\x0fnormalizer_spec\x18\x03 \x01(\x0b\x32\x1d.sentencepiece.NormalizerSpec\x12\x33\n\x0eself_test_data\x18\x04 \x01(\x0b\x32\x1b.sentencepiece.SelfTestData\x12\x38\n\x11\x64\x65normalizer_spec\x18\x05 \x01(\x0b\x32\x1d.sentencepiece.NormalizerSpec\x1a\xd2\x01\n\rSentencePiece\x12\r\n\x05piece\x18\x01 \x01(\t\x12\r\n\x05score\x18\x02 \x01(\x02\x12\x42\n\x04type\x18\x03 \x01(\x0e\x32,.sentencepiece.ModelProto.SentencePiece.Type:\x06NORMAL"T\n\x04Type\x12\n\n\x06NORMAL\x10\x01\x12\x0b\n\x07UNKNOWN\x10\x02\x12\x0b\n\x07\x43ONTROL\x10\x03\x12\x10\n\x0cUSER_DEFINED\x10\x04\x12\x08\n\x04\x42YTE\x10\x06\x12\n\n\x06UNUSED\x10\x05*\t\x08\xc8\x01\x10\x80\x80\x80\x80\x02*\t\x08\xc8\x01\x10\x80\x80\x80\x80\x02\x42\x02H\x03'''
)

__A  =				globals()
_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, '''sentencepiece_model_pb2''', _globals)
if _descriptor._USE_C_DESCRIPTORS is False:
				__A  =				None
				__A  =				B'''H\003'''
				# (generated by protobuf compiler, but `_TRAINERSPEC` is not defined)
				# _TRAINERSPEC.fields_by_name["mining_sentence_size"]._options = None
				# _TRAINERSPEC.fields_by_name["mining_sentence_size"]._serialized_options = b"\030\001"
				# _TRAINERSPEC.fields_by_name["training_sentence_size"]._options = None
				# _TRAINERSPEC.fields_by_name["training_sentence_size"]._serialized_options = b"\030\001"
				__A  =				45
				__A  =				1581
				__A  =				1517
				__A  =				1570
				__A  =				1584
				__A  =				1793
				__A  =				1795
				__A  =				1916
				__A  =				1864
				__A  =				1905
				__A  =				1919
				__A  =				2429
				__A  =				2208
				__A  =				2418
				__A  =				2323
				__A  =				2407
# @@protoc_insertion_point(module_scope)
"""simple docstring"""





import math

import torch
from torch import nn

from ..configuration_utils import ConfigMixin, register_to_config
from .attention_processor import Attention
from .embeddings import get_timestep_embedding
from .modeling_utils import ModelMixin
class  _snake_case     (					a__				, a__		):






						@register_to_config
						def __init__(       self      :			Optional[Any]							,					UpperCAmelCase      :			int = 128							,					UpperCAmelCase      :			int = 256							,					UpperCAmelCase      :			float = 2_0_0_0.0							,					UpperCAmelCase      :			int = 768							,					UpperCAmelCase      :			int = 12							,					UpperCAmelCase      :			int = 12							,					UpperCAmelCase      :			int = 64							,					UpperCAmelCase      :			int = 2048							,					UpperCAmelCase      :			float = 0.1							,					):
													super().__init__()

													__lowerCamelCase					:		List[Any]		   =     nn.Sequential(
													    nn.Linear(UpperCAmelCase							,					d_model * 4							,					bias=UpperCAmelCase    )							,					nn.SiLU()							,					nn.Linear(d_model * 4							,					d_model * 4							,					bias=UpperCAmelCase    )							,					nn.SiLU()							,					)

													__lowerCamelCase					:		str		   =     nn.Embedding(UpperCAmelCase							,					UpperCAmelCase    )
													__lowerCamelCase					:		Optional[Any]		   =     False

													__lowerCamelCase					:		Optional[int]		   =     nn.Linear(UpperCAmelCase							,					UpperCAmelCase							,					bias=UpperCAmelCase    )

													__lowerCamelCase					:		Optional[Any]		   =     nn.Dropout(p=UpperCAmelCase    )

													__lowerCamelCase					:		str		   =     nn.ModuleList()
													for lyr_num in range(UpperCAmelCase    ):
																				# FiLM conditional T5 decoder
																				__lowerCamelCase					:		List[str]		   =     DecoderLayer(d_model=UpperCAmelCase							,					d_kv=UpperCAmelCase							,					num_heads=UpperCAmelCase							,					d_ff=UpperCAmelCase							,					dropout_rate=UpperCAmelCase    )
																				self.decoders.append(UpperCAmelCase    )

													__lowerCamelCase					:		Union[str, Any]		   =     TaLayerNorm(UpperCAmelCase    )

													__lowerCamelCase					:		List[Any]		   =     nn.Dropout(p=UpperCAmelCase    )
													__lowerCamelCase					:		Any		   =     nn.Linear(UpperCAmelCase							,					UpperCAmelCase							,					bias=UpperCAmelCase    )






						def       lowerCamelCase__     (       self      :			List[str]							,					UpperCAmelCase      :			List[str]							,					UpperCAmelCase      :			int    ):
													__lowerCamelCase					:		List[Any]		   =     torch.mul(query_input.unsqueeze(-1    )							,					key_input.unsqueeze(-2    )    )
													return mask.unsqueeze(-3    )







						def       lowerCamelCase__     (       self      :			Optional[Any]							,					UpperCAmelCase      :			Optional[int]							,					UpperCAmelCase      :			List[str]							,					UpperCAmelCase      :			str    ):
													__lowerCamelCase   ,  __lowerCamelCase   ,  __lowerCamelCase					:		Optional[Any]		   =     decoder_input_tokens.shape
													assert decoder_noise_time.shape == (batch,)

													# decoder_noise_time is in [0, 1), so rescale to expected timing range.
													__lowerCamelCase					:		str		   =     get_timestep_embedding(
													    decoder_noise_time * self.config.max_decoder_noise_time							,					embedding_dim=self.config.d_model							,					max_period=self.config.max_decoder_noise_time							,					).to(dtype=self.dtype    )

													__lowerCamelCase					:		Optional[int]		   =     self.conditioning_emb(UpperCAmelCase    ).unsqueeze(1    )

													assert conditioning_emb.shape == (batch, 1, self.config.d_model * 4)

													__lowerCamelCase					:		Tuple		   =     decoder_input_tokens.shape[1]

													# If we want to use relative positions for audio context, we can just offset
													# this sequence by the length of encodings_and_masks.
													__lowerCamelCase					:		int		   =     torch.broadcast_to(
													    torch.arange(UpperCAmelCase							,					device=decoder_input_tokens.device    )							,					(batch, seq_length)							,					)

													__lowerCamelCase					:		Optional[Any]		   =     self.position_encoding(UpperCAmelCase    )

													__lowerCamelCase					:		List[str]		   =     self.continuous_inputs_projection(UpperCAmelCase    )
													inputs += position_encodings
													__lowerCamelCase					:		List[Any]		   =     self.dropout(UpperCAmelCase    )

													# decoder: No padding present.
													__lowerCamelCase					:		Tuple		   =     torch.ones(
													    decoder_input_tokens.shape[:2]							,					device=decoder_input_tokens.device							,					dtype=inputs.dtype    )

													# Translate encoding masks to encoder-decoder masks.
													__lowerCamelCase					:		Optional[Any]		   =     [(x, self.encoder_decoder_mask(UpperCAmelCase							,					UpperCAmelCase    )) for x, y in encodings_and_masks]

													# cross attend style: concat encodings
													__lowerCamelCase					:		Union[str, Any]		   =     torch.cat([x[0] for x in encodings_and_encdec_masks]							,					dim=1    )
													__lowerCamelCase					:		Union[str, Any]		   =     torch.cat([x[1] for x in encodings_and_encdec_masks]							,					dim=-1    )

													for lyr in self.decoders:
																				__lowerCamelCase					:		List[Any]		   =     lyr(
																				    UpperCAmelCase							,					conditioning_emb=UpperCAmelCase							,					encoder_hidden_states=UpperCAmelCase							,					encoder_attention_mask=UpperCAmelCase							,					)[0]

													__lowerCamelCase					:		Dict		   =     self.decoder_norm(UpperCAmelCase    )
													__lowerCamelCase					:		Optional[int]		   =     self.post_dropout(UpperCAmelCase    )

													__lowerCamelCase					:		str		   =     self.spec_out(UpperCAmelCase    )
													return spec_out

class  _snake_case     (					nn.Module		):






						def __init__(       self      :			List[str]							,					UpperCAmelCase      :			Optional[int]							,					UpperCAmelCase      :			Optional[int]							,					UpperCAmelCase      :			Dict							,					UpperCAmelCase      :			Optional[Any]							,					UpperCAmelCase      :			str							,					UpperCAmelCase      :			List[str]=1E-6    ):
													super().__init__()
													__lowerCamelCase					:		Union[str, Any]		   =     nn.ModuleList()

													# cond self attention: layer 0
													self.layer.append(
													    TaLayerSelfAttentionCond(d_model=UpperCAmelCase							,					d_kv=UpperCAmelCase							,					num_heads=UpperCAmelCase							,					dropout_rate=UpperCAmelCase    )    )

													# cross attention: layer 1
													self.layer.append(
													    TaLayerCrossAttention(
													        d_model=UpperCAmelCase							,					d_kv=UpperCAmelCase							,					num_heads=UpperCAmelCase							,					dropout_rate=UpperCAmelCase							,					layer_norm_epsilon=UpperCAmelCase							,					)    )

													# Film Cond MLP + dropout: last layer
													self.layer.append(
													    TaLayerFFCond(d_model=UpperCAmelCase							,					d_ff=UpperCAmelCase							,					dropout_rate=UpperCAmelCase							,					layer_norm_epsilon=UpperCAmelCase    )    )







						def       lowerCamelCase__     (       self      :			Union[str, Any]							,					UpperCAmelCase      :			Tuple							,					UpperCAmelCase      :			Tuple=None							,					UpperCAmelCase      :			Dict=None							,					UpperCAmelCase      :			Optional[Any]=None							,					UpperCAmelCase      :			List[str]=None							,					UpperCAmelCase      :			Dict=None							,					):
													__lowerCamelCase					:		Union[str, Any]		   =     self.layer[0](
													    UpperCAmelCase							,					conditioning_emb=UpperCAmelCase							,					attention_mask=UpperCAmelCase							,					)

													if encoder_hidden_states is not None:
																				__lowerCamelCase					:		Tuple		   =     torch.where(encoder_attention_mask > 0							,					0							,					-1E10    ).to(
																				    encoder_hidden_states.dtype    )

																				__lowerCamelCase					:		Dict		   =     self.layer[1](
																				    UpperCAmelCase							,					key_value_states=UpperCAmelCase							,					attention_mask=UpperCAmelCase							,					)

													# Apply Film Conditional Feed Forward layer
													__lowerCamelCase					:		List[Any]		   =     self.layer[-1](UpperCAmelCase							,					UpperCAmelCase    )

													return (hidden_states,)

class  _snake_case     (					nn.Module		):






						def __init__(       self      :			str							,					UpperCAmelCase      :			int							,					UpperCAmelCase      :			Optional[Any]							,					UpperCAmelCase      :			List[Any]							,					UpperCAmelCase      :			Dict    ):
													super().__init__()
													__lowerCamelCase					:		Union[str, Any]		   =     TaLayerNorm(UpperCAmelCase    )
													__lowerCamelCase					:		Any		   =     TaFiLMLayer(in_features=d_model * 4							,					out_features=UpperCAmelCase    )
													__lowerCamelCase					:		Optional[Any]		   =     Attention(query_dim=UpperCAmelCase							,					heads=UpperCAmelCase							,					dim_head=UpperCAmelCase							,					out_bias=UpperCAmelCase							,					scale_qk=UpperCAmelCase    )
													__lowerCamelCase					:		Tuple		   =     nn.Dropout(UpperCAmelCase    )







						def       lowerCamelCase__     (       self      :			int							,					UpperCAmelCase      :			Optional[int]							,					UpperCAmelCase      :			List[Any]=None							,					UpperCAmelCase      :			str=None							,					):
													# pre_self_attention_layer_norm
													__lowerCamelCase					:		int		   =     self.layer_norm(UpperCAmelCase    )

													if conditioning_emb is not None:
																				__lowerCamelCase					:		Optional[Any]		   =     self.FiLMLayer(UpperCAmelCase							,					UpperCAmelCase    )

													# Self-attention block
													__lowerCamelCase					:		Optional[Any]		   =     self.attention(UpperCAmelCase    )

													__lowerCamelCase					:		Dict		   =     hidden_states + self.dropout(UpperCAmelCase    )

													return hidden_states

class  _snake_case     (					nn.Module		):






						def __init__(       self      :			Optional[int]							,					UpperCAmelCase      :			Optional[Any]							,					UpperCAmelCase      :			Optional[int]							,					UpperCAmelCase      :			str							,					UpperCAmelCase      :			int							,					UpperCAmelCase      :			List[str]    ):
													super().__init__()
													__lowerCamelCase					:		str		   =     Attention(query_dim=UpperCAmelCase							,					heads=UpperCAmelCase							,					dim_head=UpperCAmelCase							,					out_bias=UpperCAmelCase							,					scale_qk=UpperCAmelCase    )
													__lowerCamelCase					:		List[Any]		   =     TaLayerNorm(UpperCAmelCase							,					eps=UpperCAmelCase    )
													__lowerCamelCase					:		str		   =     nn.Dropout(UpperCAmelCase    )







						def       lowerCamelCase__     (       self      :			str							,					UpperCAmelCase      :			List[Any]							,					UpperCAmelCase      :			Union[str, Any]=None							,					UpperCAmelCase      :			Optional[int]=None							,					):
													__lowerCamelCase					:		str		   =     self.layer_norm(UpperCAmelCase    )
													__lowerCamelCase					:		Dict		   =     self.attention(
													    UpperCAmelCase							,					encoder_hidden_states=UpperCAmelCase							,					attention_mask=attention_mask.squeeze(1    )							,					)
													__lowerCamelCase					:		List[str]		   =     hidden_states + self.dropout(UpperCAmelCase    )
													return layer_output

class  _snake_case     (					nn.Module		):






						def __init__(       self      :			Dict							,					UpperCAmelCase      :			Dict							,					UpperCAmelCase      :			str							,					UpperCAmelCase      :			List[str]							,					UpperCAmelCase      :			Union[str, Any]    ):
													super().__init__()
													__lowerCamelCase					:		str		   =     TaDenseGatedActDense(d_model=UpperCAmelCase							,					d_ff=UpperCAmelCase							,					dropout_rate=UpperCAmelCase    )
													__lowerCamelCase					:		Union[str, Any]		   =     TaFiLMLayer(in_features=d_model * 4							,					out_features=UpperCAmelCase    )
													__lowerCamelCase					:		Dict		   =     TaLayerNorm(UpperCAmelCase							,					eps=UpperCAmelCase    )
													__lowerCamelCase					:		Optional[Any]		   =     nn.Dropout(UpperCAmelCase    )







						def       lowerCamelCase__     (       self      :			Optional[int]							,					UpperCAmelCase      :			Optional[Any]							,					UpperCAmelCase      :			Any=None    ):
													__lowerCamelCase					:		int		   =     self.layer_norm(UpperCAmelCase    )
													if conditioning_emb is not None:
																				__lowerCamelCase					:		Union[str, Any]		   =     self.film(UpperCAmelCase							,					UpperCAmelCase    )

													__lowerCamelCase					:		List[str]		   =     self.DenseReluDense(UpperCAmelCase    )
													__lowerCamelCase					:		Optional[int]		   =     hidden_states + self.dropout(UpperCAmelCase    )
													return hidden_states

class  _snake_case     (					nn.Module		):






						def __init__(       self      :			Optional[int]							,					UpperCAmelCase      :			Tuple							,					UpperCAmelCase      :			List[Any]							,					UpperCAmelCase      :			Union[str, Any]    ):
													super().__init__()
													__lowerCamelCase					:		List[Any]		   =     nn.Linear(UpperCAmelCase							,					UpperCAmelCase							,					bias=UpperCAmelCase    )
													__lowerCamelCase					:		List[Any]		   =     nn.Linear(UpperCAmelCase							,					UpperCAmelCase							,					bias=UpperCAmelCase    )
													__lowerCamelCase					:		List[Any]		   =     nn.Linear(UpperCAmelCase							,					UpperCAmelCase							,					bias=UpperCAmelCase    )
													__lowerCamelCase					:		str		   =     nn.Dropout(UpperCAmelCase    )
													__lowerCamelCase					:		Optional[Any]		   =     NewGELUActivation()







						def       lowerCamelCase__     (       self      :			Dict							,					UpperCAmelCase      :			List[Any]    ):
													__lowerCamelCase					:		Union[str, Any]		   =     self.act(self.wi_a(UpperCAmelCase    )    )
													__lowerCamelCase					:		Any		   =     self.wi_a(UpperCAmelCase    )
													__lowerCamelCase					:		Optional[Any]		   =     hidden_gelu * hidden_linear
													__lowerCamelCase					:		Any		   =     self.dropout(UpperCAmelCase    )

													__lowerCamelCase					:		List[Any]		   =     self.wo(UpperCAmelCase    )
													return hidden_states

class  _snake_case     (					nn.Module		):






						def __init__(       self      :			Union[str, Any]							,					UpperCAmelCase      :			int							,					UpperCAmelCase      :			List[str]=1E-6    ):
													super().__init__()
													__lowerCamelCase					:		List[Any]		   =     nn.Parameter(torch.ones(UpperCAmelCase    )    )
													__lowerCamelCase					:		Tuple		   =     eps







						def       lowerCamelCase__     (       self      :			Tuple							,					UpperCAmelCase      :			Any    ):
													# T5 uses a layer_norm which only scales and doesn't shift, which is also known as Root Mean
													# Square Layer Normalization https://arxiv.org/abs/1910.07467 thus variance is calculated
													# w/o mean and there is no bias. Additionally we want to make sure that the accumulation for
													# half-precision inputs is done in fp32

													__lowerCamelCase					:		int		   =     hidden_states.to(torch.floataa    ).pow(2    ).mean(-1							,					keepdim=UpperCAmelCase    )
													__lowerCamelCase					:		Dict		   =     hidden_states * torch.rsqrt(variance + self.variance_epsilon    )

													# convert into half-precision if necessary
													if self.weight.dtype in [torch.floataa, torch.bfloataa]:
																				__lowerCamelCase					:		Union[str, Any]		   =     hidden_states.to(self.weight.dtype    )

													return self.weight * hidden_states

class  _snake_case     (					nn.Module		):






						def       lowerCamelCase__     (       self      :			str							,					UpperCAmelCase      :			torch.Tensor    ):
													return 0.5 * input * (1.0 + torch.tanh(math.sqrt(2.0 / math.pi    ) * (input + 0.0_4_4_7_1_5 * torch.pow(UpperCAmelCase							,					3.0    ))    ))

class  _snake_case     (					nn.Module		):






						def __init__(       self      :			Tuple							,					UpperCAmelCase      :			Any							,					UpperCAmelCase      :			Optional[Any]    ):
													super().__init__()
													__lowerCamelCase					:		Any		   =     nn.Linear(UpperCAmelCase							,					out_features * 2							,					bias=UpperCAmelCase    )







						def       lowerCamelCase__     (       self      :			List[str]							,					UpperCAmelCase      :			Optional[Any]							,					UpperCAmelCase      :			Any    ):
													__lowerCamelCase					:		Optional[Any]		   =     self.scale_bias(UpperCAmelCase    )
													__lowerCamelCase   ,  __lowerCamelCase					:		Dict		   =     torch.chunk(UpperCAmelCase							,					2							,					-1    )
													__lowerCamelCase					:		List[Any]		   =     x * (1 + scale) + shift
													return x