How can I define the activation function as a hyperparameter in PyTorch through RAY.Tune?
This is the link to the main page of the topic I want.
https://docs.ray.io/en/latest/tune/examples/tune-pytorch-cifar.html#tune-pytorch-cifar-ref
But unfortunately, there is no good documentation to answer all the questions.
Also, if you know how I can define nested cross validation in this environment, please tell me.

I solved this problem as follows.
n_samples,n_features=X_train.shape
class NeuralNetwork (nn.Module):
def __init__(self,n_input_features,l1, l2,l3,config):
    super (NeuralNetwork, self).__init__()
    self.config = config
    self.linear1=nn.Linear(n_input_features,4*math.floor(n_input_features/2)+l1)
    self.linear2=nn.Linear(l1+4*math.floor(n_input_features/2),math.floor(n_input_features/2)+l2)
    self.linear3=nn.Linear(math.floor(n_input_features/2)+l2,math.floor(n_input_features/3)+l3)
    self.linear4=nn.Linear(math.floor(n_input_features/3)+l3,math.floor(n_input_features/6))
    self.linear5=nn.Linear(math.floor(n_input_features/6),1)

    self.a1 = self.config.get(&quot;a1&quot;)
    self.a2 = self.config.get(&quot;a2&quot;)
    self.a3 = self.config.get(&quot;a3&quot;)
    self.a4 = self.config.get(&quot;a4&quot;) 
@staticmethod
def activation_func(act_str):
    if act_str==&quot;tanh&quot; or act_str==&quot;sigmoid&quot;:
        return eval(&quot;torch.&quot;+act_str)
    elif act_str==&quot;silu&quot; or act_str==&quot;relu&quot; or act_str==&quot;leaky_relu&quot; or act_str==&quot;gelu&quot;:   
        return eval(&quot;torch.nn.functional.&quot;+act_str)
def forward(self,x):
    out=self.linear1(x)
    out=self.activation_func(self.a1)(out.float())
    out=self.linear2(out)
    out=self.activation_func(self.a2)(out.float())
    out=self.linear3(out)
    out=self.activation_func(self.a3)(out.float())
    out=self.linear4(out)
    out=self.activation_func(self.a3)(out.float())
    out=torch.sigmoid(self.linear5(out))
    y_predicted=out
    return y_predicted

