




import flax.linen as nn
import jax
import jax.numpy as jnp




class       _A				(		nn.Module     ):
		__a					    =       42
		__a					    =       jnp.floataa






		def 				UpperCAmelCase (    self			):
								_UpperCAmelCase      =			nn.Conv(
								    self.out_channels						,		kernel_size=(3, 3)						,		strides=(1, 1)						,		padding=((1, 1), (1, 1))						,		dtype=self.dtype						,		)






		def __call__(    self						,		_SCREAMING_SNAKE_CASE			):
								_UpperCAmelCase      =			hidden_states.shape
								_UpperCAmelCase      =			jax.image.resize(
								    lowercase__						,		shape=(batch, height * 2, width * 2, channels)						,		method="""nearest"""						,		)
								_UpperCAmelCase      =			self.conv(lowercase__			)
								return hidden_states




class       _A				(		nn.Module     ):
		__a					    =       42
		__a					    =       jnp.floataa






		def 				UpperCAmelCase (    self			):
								_UpperCAmelCase      =			nn.Conv(
								    self.out_channels						,		kernel_size=(3, 3)						,		strides=(2, 2)						,		padding=((1, 1), (1, 1))						,		dtype=self.dtype						,		)






		def __call__(    self						,		_SCREAMING_SNAKE_CASE			):
								_UpperCAmelCase      =			self.conv(lowercase__			)
								return hidden_states




class       _A				(		nn.Module     ):
		__a					    =       42
		__a					    =       None
		__a					    =       0.0
		__a					    =       None
		__a					    =       jnp.floataa






		def 				UpperCAmelCase (    self			):
								_UpperCAmelCase      =			self.in_channels if self.out_channels is None else self.out_channels

								_UpperCAmelCase      =			nn.GroupNorm(num_groups=32						,		epsilon=1e-5			)
								_UpperCAmelCase      =			nn.Conv(
								    lowercase__						,		kernel_size=(3, 3)						,		strides=(1, 1)						,		padding=((1, 1), (1, 1))						,		dtype=self.dtype						,		)

								_UpperCAmelCase      =			nn.Dense(lowercase__						,		dtype=self.dtype			)

								_UpperCAmelCase      =			nn.GroupNorm(num_groups=32						,		epsilon=1e-5			)
								_UpperCAmelCase      =			nn.Dropout(self.dropout_prob			)
								_UpperCAmelCase      =			nn.Conv(
								    lowercase__						,		kernel_size=(3, 3)						,		strides=(1, 1)						,		padding=((1, 1), (1, 1))						,		dtype=self.dtype						,		)

								_UpperCAmelCase      =			self.in_channels != out_channels if self.use_nin_shortcut is None else self.use_nin_shortcut

								_UpperCAmelCase      =			None
								if use_nin_shortcut:
														_UpperCAmelCase      =			nn.Conv(
														    lowercase__						,		kernel_size=(1, 1)						,		strides=(1, 1)						,		padding="""VALID"""						,		dtype=self.dtype						,		)






		def __call__(    self						,		_SCREAMING_SNAKE_CASE						,		_SCREAMING_SNAKE_CASE						,		_SCREAMING_SNAKE_CASE=True			):
								_UpperCAmelCase      =			hidden_states
								_UpperCAmelCase      =			self.norma(lowercase__			)
								_UpperCAmelCase      =			nn.swish(lowercase__			)
								_UpperCAmelCase      =			self.conva(lowercase__			)

								_UpperCAmelCase      =			self.time_emb_proj(nn.swish(lowercase__			)			)
								_UpperCAmelCase      =			jnp.expand_dims(jnp.expand_dims(lowercase__						,		1			)						,		1			)
								_UpperCAmelCase      =			hidden_states + temb

								_UpperCAmelCase      =			self.norma(lowercase__			)
								_UpperCAmelCase      =			nn.swish(lowercase__			)
								_UpperCAmelCase      =			self.dropout(lowercase__						,		lowercase__			)
								_UpperCAmelCase      =			self.conva(lowercase__			)

								if self.conv_shortcut is not None:
														_UpperCAmelCase      =			self.conv_shortcut(lowercase__			)

								return hidden_states + residual





import logging

from transformers.configuration_utils import PretrainedConfig


a													=			logging.getLogger(__name__)




class       _A				(		__lowercase     ):
		__a					    =       """masked_bert"""






		def __init__(    self						,		_SCREAMING_SNAKE_CASE=3_0522						,		_SCREAMING_SNAKE_CASE=768						,		_SCREAMING_SNAKE_CASE=12						,		_SCREAMING_SNAKE_CASE=12						,		_SCREAMING_SNAKE_CASE=3072						,		_SCREAMING_SNAKE_CASE="gelu"						,		_SCREAMING_SNAKE_CASE=0.1						,		_SCREAMING_SNAKE_CASE=0.1						,		_SCREAMING_SNAKE_CASE=512						,		_SCREAMING_SNAKE_CASE=2						,		_SCREAMING_SNAKE_CASE=0.02						,		_SCREAMING_SNAKE_CASE=1e-12						,		_SCREAMING_SNAKE_CASE=0						,		_SCREAMING_SNAKE_CASE="topK"						,		_SCREAMING_SNAKE_CASE="constant"						,		_SCREAMING_SNAKE_CASE=0.0						,		**_SCREAMING_SNAKE_CASE						,		):
								super().__init__(pad_token_id=_SCREAMING_SNAKE_CASE						,		**_SCREAMING_SNAKE_CASE			)

								_UpperCAmelCase      =			vocab_size
								_UpperCAmelCase      =			hidden_size
								_UpperCAmelCase      =			num_hidden_layers
								_UpperCAmelCase      =			num_attention_heads
								_UpperCAmelCase      =			hidden_act
								_UpperCAmelCase      =			intermediate_size
								_UpperCAmelCase      =			hidden_dropout_prob
								_UpperCAmelCase      =			attention_probs_dropout_prob
								_UpperCAmelCase      =			max_position_embeddings
								_UpperCAmelCase      =			type_vocab_size
								_UpperCAmelCase      =			initializer_range
								_UpperCAmelCase      =			layer_norm_eps
								_UpperCAmelCase      =			pruning_method
								_UpperCAmelCase      =			mask_init
								_UpperCAmelCase      =			mask_scale