Trouble with accuracy of multiclass perceptron
Firstly, in all the linear separator algorithms such as linear regression, logistic regression and the perceptron, adding the bias is as simple as adding a feature column consisting of all 1's. Then the third weight that will be trained will act as the bias $b$.



I have some working code for a multi-class perceptron

First let's generate some artificial data with 2 features. Each distribution of points is going to be Gaussian with a given mean and variance described as a list of lists for each dimension under the variable params.

def gen_data(params, n):
    dims = len(params[0])
    num_classes = len(params)

    x = np.zeros((n*num_classes, dims))
    y = np.zeros((n*num_classes,))
    for ix, i in enumerate(range(num_classes)):
        inst = np.random.randn(n, dims)

        for dim in range(dims): x[ix*n:(ix+1)*n,dim] = np.random.normal(params[ix][dim][0], 
                                                                        params[ix][dim][1], n)
        y[ix*n:(ix+1)*n] = ix
    return x, y

params = [[[ 5,1],  [ 5,1]], 
          [[ 0,1],  [ 0,1]],
          [[2, 1],  [ 2,1]],
          [[-2, 1],  [ 2,1]]]

n = 300
x, y = gen_data(params, 300)

plt.scatter(x[:,0], x[:,1])
plt.show()




Alright so now we have 4 distributions with different labels. Let's split the data for sanctity's sake.

x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.33)


Let's train the weights using the training data

def get_weights(x, y, n_epochs, verbose = 0):
    # Append a ones column to the feature for the bias
    data = np.ones((x.shape[0], x.shape[1]+1))
    data[:, 0:x.shape[1]] = x
    # Set the targets as integers for comparison
    targets = y.astype(int)

    # Initialize the weights as a matrix 
    # number of classes by number of features
    weights = np.ones((len(set(y)), x.shape[1]+1))
    for epoch in range(n_epochs):
        for i, target in zip(data, targets):
            temp = np.dot(i, weights.T)
            pred = np.argmax(temp)
            # If wrongly predicted update prediction
            if pred != target:
                weights[target, :] = weights[target, :] + i
                weights[pred, :] = weights[pred, :] - i

        if verbose == 1:
            print('Iteration: ', epoch)
            print(weights)
            print('---------------------------------------------')
    return weights

weights = get_weights(x_train, y_train, n_epochs = 30, verbose = 1)


This converges to approximately this


  [[  23.62752045   16.03867499 -111.        ]  
  [  -3.96545848  -8.66924406   47.        ] [  -0.94290763   -1.84413793   33.        ]  [ -14.71915434   -1.52529301   35.        ]]


We get an accuracy calculated using the score

def predict(x, weights):
    data = np.ones((  x.shape[0], x.shape[1]+1  ))
    data[:, 0:x.shape[1]] = x
    predictions = np.argmax(np.dot(data, weights.T), axis = 1)
    return predictions

def score(x, y, weights):
    pred = predict(x, weights)
    return sum(pred == y_test)/len(pred)

score(x_test, y_test, weights)



  0.8686868686868687


We can check our results using a confusion matrix. For the training set

from sklearn.metrics import confusion_matrix

predictions = predict(x_train, weights)
plt.imshow(confusion_matrix(y_train, predictions))
plt.show()




And the testing set

predictions = predict(x_test, weights)
plt.imshow(confusion_matrix(y_test, predictions))
plt.show()




So we see that in fact our algorithm is performing quite well.

We can then plot our points to see how it is classifying them. I will plot the training points as small circles, and the testing points as larger ones. The dark points are those which are misclassified

colors = ['y', 'r', 'b', 'g', 'k']

# Predict training set
predictions = predict(x_train, weights)
for i, t, p in zip(x_train, y_train, predictions):
    if t == p: plt.scatter(i[0], i[1], c=colors[int(t)], alpha = 0.2, s=20)
    else: plt.scatter(i[0], i[1], c=colors[int(t)], alpha = 1)

# Predict test set
predictions = predict(x_test, weights)
for i, t, p in zip(x_test, y_test, predictions):
    if t == p: plt.scatter(i[0], i[1], c=colors[int(t)], alpha = 0.2)
    else: plt.scatter(i[0], i[1], c=colors[int(t)], alpha = 1)

# Plot the linear separators
x1 = np.linspace(np.min(x[:,0]),np.max(x[:,1]),2)
x2 = np.zeros((weights.shape[0], 2))
for ix_w, weight in enumerate(weights):
    x2 = 1 * ( - weight[2] - weight[0]*x1) / weight[1]
    plt.plot(x1, x2, c = colors[ix_w])

plt.xlabel('Feature 1')
plt.ylabel('Feature 2')

plt.xlim([np.min(x[:,0]), np.max(x[:,0])])
plt.ylim([np.min(x[:,1]), np.max(x[:,1])])

plt.show()






This code generalizes to the binary classification task as well

params = [[[ 5,1],  [ 5,1]], 
          [[ 0,1],  [ 0,1]]]








Stop training on convergence

If you want to stop the algorithm based on convergence you can use a stop criteria. For example you can stop training once every weight in your matrix changes by less than a very small number. The very small number we usually choose is machine epsilon 2.220446049250313e-16, which is essentially zero. Sometimes this requirement is too stringent so it can be replaced by any number of significant values.

Change the get_weights code to include the break criteria as

from copy import deepcopy

def get_weights(x, y, n_epochs, verbose = 0):
    # Append a ones column to the feature for the bias
    data = np.ones((x.shape[0], x.shape[1]+1))
    data[:, 0:x.shape[1]] = x
    # Set the targets as integers for comparison
    targets = y.astype(int)

    # Initialize the weights as a matrix 
    # number of classes by number of features
    weights = np.zeros((len(set(y)), x.shape[1]+1))
    past_weights = np.zeros((len(set(y)), x.shape[1]+1))
    for epoch in range(n_epochs):
        for i, target in zip(data, targets):
            temp = np.dot(i, weights.T)
            pred = np.argmax(temp)
            # If wrongly predicted update prediction
            if pred != target:
                weights[target, :] = weights[target, :] + i
                weights[pred, :] = weights[pred, :] - i

        if np.abs(weights - past_weights).all() &lt; np.finfo(float).eps:
            break
        past_weights = deepcopy(weights)

        if verbose == 1:
            print('Iteration: ', epoch)
            print(weights)
            print('---------------------------------------------')
    return weights