XOR-solving neural network is suffering from local minima import torch
import torch.nn as nn
import torch.optim as optim
from torch.autograd import Variable
import matplotlib.pyplot as plt
import numpy as np
x_data = torch.tensor([[0, 0], [0, 1], [1, 0], [1, 1]]).float()
y_data = torch.tensor([0, 1, 1, 0]).float()

class Model(nn.Module):
    def __init__(self, input_size, H1, output_size):
        super().__init__()
        self.linear_input = nn.Linear(input_size, 2)
        self.linear_output = nn.Linear(2, output_size)
        self.sigmoid = torch.nn.Sigmoid()

    def forward(self, x):
        x = self.sigmoid(self.linear_input(x))
        x = self.sigmoid(self.linear_output(x))
        return x

    def predict(self, x):
        return (self.forward(x) &gt;= 0.5).float()

model = Model(2,2,1)
lossfunc = nn.BCELoss()
optimizer = optim.Adam(model.parameters(), lr=0.01)
epochs = 2000
losses = []

for i in range(epochs):
    pred = model(x_data).view(-1)
    loss = lossfunc(pred, y_data)
    print("epochs:", i, "loss:", loss.item())
    losses.append(loss.item())
    optimizer.zero_grad()
    loss.backward() 
    optimizer.step()

def cal_score(X, y):
    y_pred = model.predict(X)
    score = float(torch.sum(y_pred.squeeze(-1) == y.byte().float())) / y.shape[0]
    return score

print('test score :', cal_score(x_data, y_data))
def plot_decision_boundray(X):
    x_span = np.linspace(min(X[:, 0]), max(X[:, 0]))
    y_span = np.linspace(min(X[:, 1]), max(X[:, 1]))
    xx, yy = np.meshgrid(x_span, y_span)
    grid = torch.tensor(np.c_[xx.ravel(), yy.ravel()]).float()
    pred_func = model.forward(grid)
    z = pred_func.view(xx.shape).detach().numpy()
    plt.contourf(xx, yy, z)
    plt.show()

plot_decision_boundray(x_data)


As you can see, it's a simple neural network which consists of one hidden layer using BCELoss and Adam.



Normally, it results in the correct one like above. 



However, it is sometimes stuck in a local minima and a decision boundary becomes awkward.

Because the input data is limited, I guess that preprocessing of those data might not be possible and only initial weights matter in this problem. I tried initializing them with normal distribution but it didn't work. How can I approach this problem?
From the log you can see that the loss value is still big compared to the first image, but at least it decrease on each iteration (even very very small). So I suggest you to:


try to use bigger learning rate, I think you model is underfiting.
try to use another weight initialization method


I think the first solution will solve this problem as I see no problem in the code.