PyTorch does not make initial weights random
I created a Neural Network that takes two greyscale images 14x14 pixels portraying a digit (from MNIST database) and returns 1 if the first digit is less or equal to the second digit, returns 0 otherwise. The code runs, but every time the initial weights are the same. They should be random
Forcing the initial weights to be random, by using the following line of code in the Net class, does not help.
torch.nn.init.normal_(self.layer1.weight, mean=0.0, std=0.01)

Here is the code of the &quot;main.py&quot; file:
import os; os.environ[&quot;KMP_DUPLICATE_LIB_OK&quot;] = &quot;TRUE&quot;
import torch
import torch.nn as nn
from dlc_practical_prologue import *


class Net(nn.Module):
    def __init__(self):
        super().__init__()
        self.layer1 = nn.Linear(2*14*14, 32)
        #torch.nn.init.normal_(self.layer1.weight, mean=0.0, std=0.01)
        #self.layer2 = nn.Linear(100, 100)
        #self.layer3 = nn.Linear(100, 100)
        self.layer2 = nn.Linear(32, 1)

    def forward(self, x):
        x = torch.relu(self.layer1(x))
        #x = torch.relu(self.layer2(x))
        #x = torch.relu(self.layer3(x))
        x = torch.sigmoid(self.layer2(x))
        return x


if __name__ == '__main__':
    # Data initialization
    N = 1000
    train_input, train_target, train_classes, _, _, _, = generate_pair_sets(N)
    _, _, _, test_input, test_target, test_classes = generate_pair_sets(N)

    train_input = train_input.view(-1, 2*14*14)
    test_input = test_input.view(-1, 2*14*14)

    train_target = train_target.view(-1, 1)
    test_target = test_target.view(-1, 1)

    # I convert the type to torch.float32
    train_input, train_target, train_classes, test_input, test_target, test_classes = \
        train_input.type(torch.float32), train_target.type(torch.float32), train_classes.type(torch.long), \
        test_input.type(torch.float32), test_target.type(torch.float32), test_classes.type(torch.long)


    # Create the neural network
    net = Net()


    # Training
    learning_rate = 0.01
    # Use MSELoss
    loss = nn.MSELoss()
    # Use Adam optimizer
    optimizer = torch.optim.Adam(net.parameters(), lr=learning_rate)

    EPOCHS = 50

    for param in net.parameters():
        print(param)

    for epoch in range(EPOCHS):
        target_predicted = net(train_input)
        l = loss(train_target, target_predicted)  #loss = nn.MSELoss()
        #l = loss(target_predicted, train_target)
        l.backward()
        optimizer.step()
        optimizer.zero_grad()

        #print(l)


    # Testing
    total = 1000
    correct = 0
    with torch.no_grad():
        correct = ( test_target == net(test_input).round() ).sum()

    print(&quot;Accuracy %.2f%%&quot; % (correct / total * 100))

Here is the code for &quot;dlc_practical_monologue.py&quot;:
import os; os.environ[&quot;KMP_DUPLICATE_LIB_OK&quot;]=&quot;TRUE&quot;
import torch
from torchvision import datasets

import argparse
import os

import urllib

######################################################################

parser = argparse.ArgumentParser(description='DLC prologue file for practical sessions.')

parser.add_argument('--full',
                    action='store_true', default=False,
                    help = 'Use the full set, can take ages (default False)')

parser.add_argument('--tiny',
                    action='store_true', default=False,
                    help = 'Use a very small set for quick checks (default False)')

parser.add_argument('--seed',
                    type = int, default = 0,
                    help = 'Random seed (default 0, &lt; 0 is no seeding)')

parser.add_argument('--cifar',
                    action='store_true', default=False,
                    help = 'Use the CIFAR data-set and not MNIST (default False)')

parser.add_argument('--data_dir',
                    type = str, default = None,
                    help = 'Where are the PyTorch data located (default $PYTORCH_DATA_DIR or \'./data\')')

# Timur's fix
parser.add_argument('-f', '--file',
                    help = 'quick hack for jupyter')

args = parser.parse_args()

if args.seed &gt;= 0:
    torch.manual_seed(args.seed)

######################################################################
# The data

def convert_to_one_hot_labels(input, target):
    tmp = input.new_zeros(target.size(0), target.max() + 1)
    tmp.scatter_(1, target.view(-1, 1), 1.0)
    return tmp

def load_data(cifar = None, one_hot_labels = False, normalize = False, flatten = True):

    if args.data_dir is not None:
        data_dir = args.data_dir
    else:
        data_dir = os.environ.get('PYTORCH_DATA_DIR')
        if data_dir is None:
            data_dir = './data'

    if args.cifar or (cifar is not None and cifar):
        print('* Using CIFAR')
        cifar_train_set = datasets.CIFAR10(data_dir + '/cifar10/', train = True, download = True)
        cifar_test_set = datasets.CIFAR10(data_dir + '/cifar10/', train = False, download = True)

        train_input = torch.from_numpy(cifar_train_set.data)
        train_input = train_input.transpose(3, 1).transpose(2, 3).float()
        train_target = torch.tensor(cifar_train_set.targets, dtype = torch.int64)

        test_input = torch.from_numpy(cifar_test_set.data).float()
        test_input = test_input.transpose(3, 1).transpose(2, 3).float()
        test_target = torch.tensor(cifar_test_set.targets, dtype = torch.int64)

    else:
        print('* Using MNIST')

        ######################################################################
        # import torchvision

        # raw_folder = data_dir + '/mnist/raw/'
        # resources = [
            # (&quot;https://fleuret.org/dlc/data/train-images-idx3-ubyte.gz&quot;, &quot;f68b3c2dcbeaaa9fbdd348bbdeb94873&quot;),
            # (&quot;https://fleuret.org/dlc/data/train-labels-idx1-ubyte.gz&quot;, &quot;d53e105ee54ea40749a09fcbcd1e9432&quot;),
            # (&quot;https://fleuret.org/dlc/data/t10k-images-idx3-ubyte.gz&quot;, &quot;9fb629c4189551a2d022fa330f9573f3&quot;),
            # (&quot;https://fleuret.org/dlc/data/t10k-labels-idx1-ubyte.gz&quot;, &quot;ec29112dd5afa0611ce80d1b7f02629c&quot;)
        # ]

        # os.makedirs(raw_folder, exist_ok=True)

        # # download files
        # for url, md5 in resources:
            # filename = url.rpartition('/')[2]
            # torchvision.datasets.utils.download_and_extract_archive(url, download_root=raw_folder, filename=filename, md5=md5)
        ######################################################################

        mnist_train_set = datasets.MNIST(data_dir + '/mnist/', train = True, download = True)
        mnist_test_set = datasets.MNIST(data_dir + '/mnist/', train = False, download = True)

        train_input = mnist_train_set.data.view(-1, 1, 28, 28).float()
        train_target = mnist_train_set.targets
        test_input = mnist_test_set.data.view(-1, 1, 28, 28).float()
        test_target = mnist_test_set.targets

    if flatten:
        train_input = train_input.clone().reshape(train_input.size(0), -1)
        test_input = test_input.clone().reshape(test_input.size(0), -1)

    if args.full:
        if args.tiny:
            raise ValueError('Cannot have both --full and --tiny')
    else:
        if args.tiny:
            print('** Reduce the data-set to the tiny setup')
            train_input = train_input.narrow(0, 0, 500)
            train_target = train_target.narrow(0, 0, 500)
            test_input = test_input.narrow(0, 0, 100)
            test_target = test_target.narrow(0, 0, 100)
        else:
            print('** Reduce the data-set (use --full for the full thing)')
            train_input = train_input.narrow(0, 0, 1000)
            train_target = train_target.narrow(0, 0, 1000)
            test_input = test_input.narrow(0, 0, 1000)
            test_target = test_target.narrow(0, 0, 1000)

    print('** Use {:d} train and {:d} test samples'.format(train_input.size(0), test_input.size(0)))

    if one_hot_labels:
        train_target = convert_to_one_hot_labels(train_input, train_target)
        test_target = convert_to_one_hot_labels(test_input, test_target)

    if normalize:
        mu, std = train_input.mean(), train_input.std()
        train_input.sub_(mu).div_(std)
        test_input.sub_(mu).div_(std)

    return train_input, train_target, test_input, test_target

######################################################################

def mnist_to_pairs(nb, input, target):
    input = torch.functional.F.avg_pool2d(input, kernel_size = 2)
    a = torch.randperm(input.size(0))
    a = a[:2 * nb].view(nb, 2)
    input = torch.cat((input[a[:, 0]], input[a[:, 1]]), 1)
    classes = target[a]
    target = (classes[:, 0] &lt;= classes[:, 1]).long()
    return input, target, classes

######################################################################

def generate_pair_sets(nb):
    if args.data_dir is not None:
        data_dir = args.data_dir
    else:
        data_dir = os.environ.get('PYTORCH_DATA_DIR')
        if data_dir is None:
            data_dir = './data'

    train_set = datasets.MNIST(data_dir + '/mnist/', train = True, download = True)
    train_input = train_set.data.view(-1, 1, 28, 28).float()
    train_target = train_set.targets

    test_set = datasets.MNIST(data_dir + '/mnist/', train = False, download = True)
    test_input = test_set.data.view(-1, 1, 28, 28).float()
    test_target = test_set.targets

    return mnist_to_pairs(nb, train_input, train_target) + \
           mnist_to_pairs(nb, test_input, test_target)

######################################################################

Note that I have to add the following line of code to run the code on Windows 10, while it is not necessary to run it on Linux.
import os; os.environ[&quot;KMP_DUPLICATE_LIB_OK&quot;] = &quot;TRUE&quot;

Also on Linux I always get the same initial weights.
Please, can you help me?

Correct me if I'm wrong here but only the weights of the first layer should be the same each time you run this. The thing is when you import the dlc_practical_monologue.py there's this thing in it:
if args.seed &gt;= 0:
    torch.manual_seed(args.seed)

which fires up if the seed is &gt;=0 (default is 0).
This should only initialize the first layer with the same weights for each run. Check if this is the case.
