TypeError: mul() argument 'other' (position 1) must be Tensor, not ReLU
I would like to add a torch.nn.ReLU() layer between fc1 and fc2 layer. 

Original code:

model:

# ...
self.fc1 = nn.Linear(4096, 256)
self.fc2 = nn.Linear(256, 4096)
# ...


def forward(...):
    # ...
    x = x.view(-1, 4096)
    x = self.fc1(x))
    if a7 is not None:
        x = x * a7.squeeze()
    # ...


I tried 

# ...
x = x.view(-1, 4096)
x = nn.ReLU(self.fc1(x)))
if a7 is not None:
    x = x * a7.squeeze()
# ...


and this error pops out.

My answer assumes __init__ was a typo and it should be forward. Let me know if that is not the case and I'll delete it.

import torch
from torch import nn

class SimpleModel(nn.Module):
  def __init__(self, with_relu=False):
    super(SimpleModel, self).__init__()
    self.fc1 = nn.Sequential(nn.Linear(3, 10), nn.ReLU(inplace=True)) if with_relu else nn.Linear(3, 10)
    self.fc2 = nn.Linear(10, 3)

  def forward(self, x):
    x = self.fc1(x)
    print(torch.min(x))  # just to show you ReLU is working...
    return self.fc2(x)

# Model without ReLU
net_without_relu = SimpleModel(with_relu=False)
print(net_without_relu)

# Model with ReLU
net_with_relu = SimpleModel(with_relu=True)
print(net_with_relu)

# random input data
x = torch.randn((5, 3))
print(x)

# we expect it to print something &lt; 0
output1 = net_without_relu(x)

# we expect it to print 0.
output2 = net_with_relu(x)


You can check the code below running on the Colab: https://colab.research.google.com/drive/1W3Dh4_KPd3iABx5FSzZm3tilm6tnJh0v



To use as you tried:

x = nn.ReLU(self.fc1(x)))


you can use the functional API:

from torch.nn import functional as F

# ...
x = F.relu(self.fc1(x)))

