





from torch import nn




def     A_      (	lowercase_	)      ->    List[str]:
       if act_fn in ["swish", "silu"]:
              return nn.SiLU()
       elif act_fn == "mish":
              return nn.Mish()
       elif act_fn == "gelu":
              return nn.GELU()
       else:
              raise ValueError(f'''Unsupported activation function: {act_fn}'''	)














import numpy as np






def      _snake_case     (__lowercase):
   return 1 / (1 + np.exp(-vector))






def      _snake_case     (__lowercase):
   return vector * sigmoid(__lowercase)


if __name__ == "__main__":
 import doctest

 doctest.testmod()





