PyTorch did not compute gradient and update parameters for 'masking' tensors?
I am coding a LeNet in PyTorch for MNIST dataset; I add a tensor self.mask_fc1\2\3 to mask some certain connections for the full connection layers. The code is like this: 

import torch
import torchvision
import torchvision.transforms as transforms
import matplotlib.pyplot as plt
import numpy as np
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim

def loadMNIST():
    transform = transforms.Compose([transforms.ToTensor()])

    trainset = torchvision.datasets.MNIST(root='./data', train=True,
                                          download=True, transform=transform)
    trainloader = torch.utils.data.DataLoader(trainset, batch_size=4,
                                              shuffle=True, num_workers=2)

    testset = torchvision.datasets.MNIST(root='./data', train=False,
                                         download=True, transform=transform)
    testloader = torch.utils.data.DataLoader(testset, batch_size=4,
                                             shuffle=False, num_workers=2)
    return trainloader, testloader

class Net(nn.Module):
    def __init__(self):
        super(Net, self).__init__()
        self.conv1 = nn.Conv2d(1, 6, 5, 1, 2)
        self.pool = nn.MaxPool2d(2, 2)
        self.conv2 = nn.Conv2d(6, 16, 5)
        self.fc1 = nn.Linear(16 * 5 * 5, 120)
        self.fc2 = nn.Linear(120, 84)
        self.fc3 = nn.Linear(84, 10)

        self.mask_fc1 = torch.ones(16 * 5 * 5, 120, requires_grad=True)
        self.mask_fc2 = torch.ones(120, 84, requires_grad=True)
        self.mask_fc3 = torch.ones(84, 10, requires_grad=True)

    def forward(self, x):
        x = self.pool(F.relu(self.conv1(x)))
        x = self.pool(F.relu(self.conv2(x)))
        x = x.view(-1, self.num_flat_features(x))

        # first layer
        x = x.matmul(self.fc1.weight.t() * self.mask_fc1)
        if self.fc1.bias is not None:
            x += torch.jit._unwrap_optional(self.fc1.bias)
        x = F.relu(x)

        # second layer
        x = x.matmul(self.fc2.weight.t() * self.mask_fc2)
        if self.fc2.bias is not None:
            x += torch.jit._unwrap_optional(self.fc2.bias)
        x = F.relu(x)

        # third layer
        x = x.matmul(self.fc3.weight.t() * self.mask_fc3)
        if self.fc3.bias is not None:
            x += torch.jit._unwrap_optional(self.fc3.bias)

        return x

    def num_flat_features(self, x):
        size = x.size()[1:]
        num_features = 1
        for s in size:
            num_features *= s
        return num_features


if __name__ == '__main__':
    trainloader, testloader = loadMNIST()

    net = Net()

    # train
    criterion = nn.CrossEntropyLoss()
    optimizer = optim.SGD(net.parameters(), lr=0.001, momentum=0.9)

    for epoch in range(2):
        running_loss = 0.0
        for i, data in enumerate(trainloader, 0):
            inputs, labels = data

            optimizer.zero_grad()

            outputs = net(inputs)
            loss = criterion(outputs, labels)
            loss.backward()
            optimizer.step()

            # print
            running_loss += loss.item()
            if i % 2000 == 1999:
                # mean loss
                print('[%d, %5d] loss: %.3f' % (epoch + 1, i + 1, running_loss / 2000))
                running_loss = 0.0

    print('Finished Training')

    # print the mask
    print(net.mask_fc1)


I achieve the masking in the forward function and implement linear layer by myself rather than call x = F.relu(self.fc1(x)), and the model performs normally (at last in terms of the loss and accuracy).

However, when I print self.mask_fc1/2/3, the tensor does not change during the training. Since the tensor was set requires_grad=True in function __init__, I cannot understand why it does not change. Maybe it is because the tensor multiplication?

For training, you need to register mask_fc1/2/3 as module parameters:

self.mask_fc1 = nn.Parameter(torch.ones(16 * 5 * 5, 120))

You can print net.parameters() after that to confirm.
