How to pass an intermediate layer of one model to another model for skip connection in PyTorch
I want to define an encoder decoder architecture as two separate models and later connect them using nn.Sequential() as shown in the code below. Now, let's say that I want to connect/concatenate the output of Encoder conv4 block to the deconv1 block of Decoder as a skip connection. Is there a way to achieve that without combining the two models (encoder and decoder) into one. I want to keep them separate to be able to use output of the same encoder as input of multiple decoders.

class Encoder(nn.Module):

    def __init__(self, conv_dim=64, n_res_blocks=2):
        super(Encoder, self).__init__()

        # Define the encoder
        self.conv1 = conv(3, conv_dim, 4)
        self.conv2 = conv(conv_dim, conv_dim*2, 4)
        self.conv3 = conv(conv_dim*2, conv_dim*4, 4)
        self.conv4 = conv(conv_dim*4, conv_dim*4, 4)

        # Define the resnet part of the encoder
        # Residual blocks
        res_layers = []
        for layer in range(n_res_blocks):
            res_layers.append(ResidualBlock(conv_dim*4))
        # use sequential to create these layers
        self.res_blocks = nn.Sequential(*res_layers)

        # leaky relu function
        self.leaky_relu = nn.LeakyReLU(negative_slope=0.2)

    def forward(self, x):
        # define feedforward behavior, applying activations as necessary
        conv1 = self.leaky_relu(self.conv1(x))
        conv2 = self.leaky_relu(self.conv2(conv1))
        conv3 = self.leaky_relu(self.conv3(conv2))
        conv4 = self.leaky_relu(self.conv4(conv3))

        out = self.res_blocks(conv4)

        return out

# Define the Decoder Architecture
class Decoder(nn.Module):

    def __init__(self, conv_dim=64, n_res_blocks=2):
        super(Decoder, self).__init__()

        # Define the resnet part of the decoder
        # Residual blocks
        res_layers = []
        for layer in range(n_res_blocks):
            res_layers.append(ResidualBlock(conv_dim*4))
        # use sequential to create these layers
        self.res_blocks = nn.Sequential(*res_layers)

        # Define the decoder 
        self.deconv1 = deconv(conv_dim*4, conv_dim*4, 4)
        self.deconv2 = deconv(conv_dim*4, conv_dim*2, 4)
        self.deconv3 = deconv(conv_dim*2, conv_dim, 4)
        self.deconv4 = deconv(conv_dim, conv_dim, 4)

        # no batch norm on last layer
        self.out_layer = deconv(conv_dim, 3, 1, stride=1, padding=0, normalization=False)

        # leaky relu function
        self.leaky_relu = nn.LeakyReLU(negative_slope=0.2)

    def forward(self, x):
        # define feedforward behavior, applying activations as necessary
        res = self.res_blocks(x)

        deconv1 = self.leaky_relu(self.deconv1(res))
        deconv2 = self.leaky_relu(self.deconv2(deconv1))
        deconv3 = self.leaky_relu(self.deconv3(deconv2))
        deconv4 = self.leaky_relu(self.deconv4(deconv3))

        # tanh applied to last layer
        out = F.tanh(self.out_layer(deconv4))
        out = torch.clamp(out, min=-0.5, max=0.5)

        return out

def model():

    enc = Encoder(conv_dim=64, n_res_blocks=2)
    dec = Decoder(conv_dim=64, n_res_blocks=2)
    return nn.Sequential(enc, dec)


Instead of returning only the latent feature (output of last layer) from encoder you can return the output of intermediate layers along with the latent feature, may be as a list. Afterwards, in the decoder's forward function you can access the list of values returned from encoder (which is the parameters of decoder) and use that correspondingly in the decoder layer.

Hope this bit helps.
