Create a skip connection in a neural network - non Resnet
I want to add a skip connection to my neural network; I'm not trying to
implement a ResNet, just a regular MLP. I can't find a resource that doesn't point to resnet or densenet. I tried naively adding layers, but it's throwing an error; I'd appreciate the help. thank you
    input_size = 615
    output_size = 40
    model = torch.nn.Sequential()
    layer_0 = model.add_module(&quot;linear_0&quot;, torch.nn.Linear(input_size, 2048))
    activ_0 = model.add_module(&quot;activation_0&quot;, ReLU())
    layer_1 = model.add_module(&quot;linear_1&quot;, torch.nn.Linear(2048, 2048))
    activ_1 = model.add_module(&quot;activation_1&quot;, ReLU())
    layer_2 = model.add_module(&quot;linear_2&quot;, torch.nn.Linear(2048, 2048))
    # skip connection
    skip_0 =  model.add_module(layer_1 + layer_2)
    activ_2 = model.add_module(&quot;activation_2&quot;, ReLU())
    layer_3 = model.add_module(&quot;linear_3&quot;, torch.nn.Linear(2048, output_size))


They way skip connections are usually implemented is in the forward function.
For example:
from torch import nn
import torch.nn.functional as nnf

class MLPWithSkip(nn.Module):
  def __init__(self, input_size, output_size):
    self.linear_modules = nn.ModuleList([nn.Linear(input_size, 2048),
                                         nn.linear(2048, 2048),
                                         nn.Linear(2048, 2048),
                                         nn.linear(2048, output_size)])

  def forward(self, x):
    h = []
    for layer in self.linear_modules[:-1]:
      x = layer(x)
      h.append(x)  # store the features
      x = nnf.relu(x)
    # implement the skip
    x = nnf.relu(h[-1] + h[-2])
    y = self.linear_modules[-1](x)
    return y

