





from torch import nn




class 	UpperCamelCase_							(  nn.Module     ):


							def __init__(				self     :List[Any]				,					__A     :List[str]				,					__A     :List[Any]		) -> List[Any]:



									"""simple docstring"""



									super().__init__()
									SCREAMING_SNAKE_CASE__								=   class_size
									SCREAMING_SNAKE_CASE__								=   embed_size
									# self.mlp1 = nn.Linear(embed_size, embed_size)
									# self.mlp2 = (nn.Linear(embed_size, class_size))
									SCREAMING_SNAKE_CASE__								=   nn.Linear(__A				,					__A		)


							def   _snake_case	(				self     :Tuple				,					__A     :Optional[Any]		) -> List[Any]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								=   self.mlp(__A		)
									return logits






from torch import nn






def 					SCREAMING_SNAKE_CASE__				(							UpperCamelCase__:				str     ):
		if act_fn in ["swish", "silu"]:
				return nn.SiLU()
		elif act_fn == "mish":
				return nn.Mish()
		elif act_fn == "gelu":
				return nn.GELU()
		else:
				raise ValueError(f'''Unsupported activation function: {act_fn}'''     )