pytorch conv2d vs numpy results are different
I'm working on implementing pytorch conv2d with numpy. But pytorch conv2d vs numpy results are different for the same input and conv weight. How to fix it? Thanks for any help.
Code sample below:
Note:
The code contains 4 parts:

Fixed random seed to generate fixed input and conv weight.
Implement conv2d with pytorch.
Implement conv2d with numpy.
Run conv2d and verify the results are different.

import random
import numpy as np
import torch
import torch.nn as nn

fixed_seed = 5179
np.random.seed(fixed_seed) 
random.seed(fixed_seed)
torch.manual_seed(fixed_seed)

np.set_printoptions(precision=8, floatmode=&quot;fixed&quot;)
torch.set_printoptions(precision=8)


def conv_forward_torch(input_image_tensor, weight, stride, pad):
    # input_image_tensor B  Hi Wi Ci
    # weight             Hk Wk Ci Co
    input_image_tensor = input_image_tensor.permute(0, 3, 1, 2) # B Ci Hi Wi
    weight = weight.permute(3, 2, 0, 1) # Co Ci Hk Wk
    output = torch.nn.functional.conv2d(input_image_tensor, weight, stride=stride, padding=pad) # B Co Ho Wo
    output = output.permute(0, 2, 3, 1).cpu().detach().numpy()  # B Ho Wo Co
    return output


def conv_forward_naive(x, w, stride, pad, bias = None):
    # x B  Hi Wi Ci
    # w Hk Wk Ci Co
    x = np.transpose(x, [0, 3, 1, 2]) # B  Ci Hi Wi
    w = np.transpose(w, [3, 2, 0, 1]) # Co Ci Kh Kw
    
    if pad != 0:
        x = np.pad(x, ((0, 0), (0 ,0), (pad, pad), (pad, pad)),'constant')
    
    b,  ci, hi, wi = x.shape
    co, ci, hk, wk = w.shape
    ho = np.floor(1 + (hi - hk) / stride).astype(int)
    wo = np.floor(1 + (wi - wk) / stride).astype(int)
    out = np.zeros((b, co, ho, wo), dtype=np.float32) # B Co Ho Wo
    
    x = np.expand_dims(x, axis=1) # B 1  Ci Hi Wi
    w = np.expand_dims(w, axis=0) # 1 Co Ci Hk Wk
    
    for i in range(ho):
        for j in range(wo):
            x_windows = x[:, :, :, i * stride:i * stride + hk, j * stride: j * stride + wk] # B 1 Ci Hk Wk
            out[:, :, i, j] = np.sum(x_windows * w, axis=(2, 3, 4)) # B Co
            
    out = np.transpose(out, [0, 2, 3, 1]) # B Ho Wo Co
    return out
    


B  = 1 # Batch size
Hi = 2 # Input height
Wi = 2 # Input width
Ci = 1 # Input channel
Co = 1 # Ouput channel

P = 0 # Padding size

Hk = 2 # Kernel height
Wk = 2 # Kernel width

S = 1 # Stride


input_image_tensor = torch.randn(B, Hi, Wi, Ci)
conv_weight_tensor = torch.randn(Hk, Wk, Ci, Co)

input_image = input_image_tensor.detach().numpy()
conv_weight = conv_weight_tensor.detach().numpy()

y_torch = conv_forward_torch(input_image_tensor, conv_weight_tensor, S, P)
y_np = conv_forward_naive(input_image, conv_weight, S, P)

is_same = y_torch == y_np
    
print(is_same, y_torch, y_np)

Expected output:
is_same should be True.
Actual output for y_torch, y_np:
[[[[False]]]] [[[[-3.62229419]]]] [[[[-3.62229395]]]]
torch.version = '1.11.0'
np.version = '1.20.1'

Your code works as expected. The result you observed is because of a difference in floating-point precisions between NumPy and PyTorch. To compare floating points you should not use a direct equal check, but instead something like np.allclose.
In this case your snippet indeed returns True:
&gt;&gt;&gt; np.allclose(y_torch, y_np)
True

