Why "conv1d" is different in C code, python and pytorch
I want to reproduce "Conv1D" results of pytorch in C code.

I tried to implement "Conv1D" using three methods (C code, Python, Pytorch), but the results are different. Only seven fraction digits are reasonable. Assuming there are multiple layers of conv1d in the structure, the fraction digits accuracy will gradually decrease.

According to everyone's recommend, I tried to change the C code type of input data to double but the result is still incorrect. 
Have I done something wrong?

For example:

　　The output of Pytorch: 0.2380688339471817017

　　The output of Python: 0.2380688637495040894

　　The output of C code (float): 0.2380688637

　　The output of C code (double): 0.238068885344539680

Here is my current implementation


Input:

　　input dim. = 80, output dim. = 128, kernel size = 5

　　Pytorch: Conv1D_input.npy, Conv1D_weight.npy 

　　Python: Conv1D_input.npy, Conv1D_weight.npy (same as Pytorch)

　　C code: Conv1D_input.txt, Conv1D_weight.txt (convert from Pytorch, IEEE 754 single precision)

　　Pytorch

import torch
import numpy as np
from torch import nn
from torch.autograd import Variable
import torch.nn.functional as F
import argparse
import sys
import io
import time
import os

class RNN(nn.Module):
    def __init__(self, input_size, hidden_size):
        super(RNN, self).__init__()
        self.input_size = input_size
        self.hidden_size = hidden_size

        self.c1 = nn.Conv1d(input_size, hidden_size, kernel_size = 5, bias=False)
        self.c1.weight = torch.nn.Parameter(torch.Tensor(np.load("CONV1D_WEIGHT.npy")))    

    def forward(self, inputs):
        c = self.c1(inputs)
        return c

input_size = 80
hidden_size = 128
kernel_size = 5

rnn = RNN(input_size, hidden_size)

inputs = torch.nn.Parameter(torch.Tensor(np.load("CONV1D_IN.npy")))
print("inputs", inputs)
outputs = rnn(inputs)
sub_np456 = outputs[0].cpu().detach().numpy()
np.savetxt("Pytorch_CONV1D_OUTPUT.txt", sub_np456)
print('outputs', outputs)


　　Python

import struct
import numpy as np

if __name__ == "__main__":
    row = 80
    col = 327
    count = 0
    res_out_dim = 128
    in_dim = 80
    kernel_size = 5
    filter = np.zeros((80, 5), dtype = np.float32)
    featureMaps = np.zeros((128, 323), dtype = np.float32)

    spectrum = np.load("CONV1D_INPUT.npy")
    weight = np.load("CONV1D_WEIGHT.npy")

    spectrum_2d = spectrum.reshape(80, 327)

    for i in range(res_out_dim):
        for j in range(in_dim): 
            for k in range(kernel_size):
                filter[j][k] = weight[i][j][k]

        while count &lt; (col-kernel_size+1):
            for j in range(in_dim):
                for k in range(count, kernel_size+count):
                    featureMaps[i][count] = featureMaps[i][count] + spectrum_2d[j][k]*filter[j][k-count]
            count = count + 1
        count = 0

    np.savetxt("Python_CONV1D_OUTPUT.txt", featureMaps)


　　C code (float)

#include&lt;stdio.h&gt;
#include&lt;stdlib.h&gt;
#include&lt;math.h&gt;
#include&lt;time.h&gt;

const char CONV1D_WEIGHT[] = "CONV1D_WEIGHT.txt";
const char CONV1D_INPUT[] = "CONV1D_INPUT.txt";

void parameterFree(float **matrix, int row)
{
    int i = 0;
    for(i=0; i&lt;row; i++)
        free(matrix[i]);
    free(matrix);
}

float** createMatrix_2D(int row, int col)
{
    int i = 0;
    float **matrix = NULL;
    matrix = (float**)malloc(sizeof(float*) * row);
    if(matrix == NULL)
        printf("Matrix2D malloc failed\n");
    for(i=0; i&lt;row; i++)
    {
        matrix[i] = (float*)malloc(sizeof(float) * col);
        if(matrix[i] == NULL)  
            printf("Matrix2D malloc failed\n");
    }

    return matrix;
}

float** conv_1D(const char weightFile[], float **source, int *row, int *col, int in_dim, int res_out_dim, int kernel_size)
{   
    float **filter = createMatrix_2D(in_dim, kernel_size);   
    float **featureMaps = createMatrix_2D(res_out_dim, *col-kernel_size+1);
    int i = 0, j = 0, k = 0, count = 0;
    char str[10];
    float data = 0.0;
    FILE *fp = fopen(weightFile, "r");   
    if(fp == NULL)
        printf("Resnet file open failed\n");
    else
    {                  
        /*initial featureMaps*/
        for(i=0; i&lt;res_out_dim; i++)
        {
            for(j=0; j&lt;*col-kernel_size+1; j++)
            {
                featureMaps[i][j] = 0.0;
            }
        }

        /*next filter*/        
        for(i=0; i&lt;res_out_dim; i++) 
        {
            /*read filter*/
            for(j=0; j&lt;in_dim; j++) 
            {
                for(k=0; k&lt;kernel_size; k++) 
                {
                    fscanf(fp, "%s", str);
                    sscanf(str, "%x", &amp;data);
                    filter[j][k] = data;                                        
                }               
            }

            /* (part of source * filter) */            
            while(count &lt; *col-kernel_size+1)
            {                
                for(j=0; j&lt;in_dim; j++)
                {                                       
                    for(k=count; k&lt;kernel_size+count; k++)
                    {
                        featureMaps[i][count] += source[j][k]*filter[j][k-count];
                    }
                }               
                count++;               
            }
            count = 0;          
        }
        fclose(fp);
    }
    parameterFree(source, *row);
    parameterFree(filter, in_dim);
    *row = res_out_dim;
    *col = *col-kernel_size+1;       

    return featureMaps;  
}

int main()
{
    int row = 80;
    int col = 327;
    int in_dim = 80;
    int res_out_dim = 128;
    int kernel_size = 5;
    int i, j;
    float data;
    char str[10];

    float **input = createMatrix_2D(row, col);
    FILE *fp = fopen(CONV1D_INPUT, "r");
    FILE *fp2 = fopen("C code_CONV1D_OUTPUT.txt", "w");
    if(fp == NULL)
        printf("File open failed\n");
    else
    {
        for(i=0; i&lt;row; i++) 
        {
            for(j=0; j&lt;col; j++) 
            {
                fscanf(fp, "%s", str);
                sscanf(str, "%x", &amp;data);
                input[i][j] = data;                                        
            }               
        }
    }

    float **CONV1D_ANS = conv_1D(CONV1D_WEIGHT, input, &amp;row, &amp;col, in_dim, res_out_dim, kernel_size);

    for(i=0; i&lt;row; i++) 
    {
        for(j=0; j&lt;col; j++) 
        {
            fprintf(fp2, "[%.12f] ", CONV1D_ANS[i][j]);
        }
        fprintf(fp2, "\n");
    }

    return 0;
}


　　C code (double)

#include&lt;stdio.h&gt;
#include&lt;stdlib.h&gt;
#include&lt;math.h&gt;
#include&lt;time.h&gt;

const char CONV1D_WEIGHT[] = "CONV1D_WEIGHT.txt";
const char CONV1D_INPUT[] = "CONV1D_INPUT.txt";

void parameterFree(double **matrix, int row)
{
    int i = 0;
    for(i=0; i&lt;row; i++)
        free(matrix[i]);
    free(matrix);
}

double** createMatrix_2D(int row, int col)
{
    int i = 0;
    double **matrix = NULL;
    matrix = (double**)malloc(sizeof(double*) * row);
    if(matrix == NULL)
        printf("Matrix2D malloc failed\n");
    for(i=0; i&lt;row; i++)
    {
        matrix[i] = (double*)malloc(sizeof(double) * col);
        if(matrix[i] == NULL)  
            printf("Matrix2D malloc failed\n");
    }

    return matrix;
}

double** conv_1D(const char weightFile[], double **source, int *row, int *col, int in_dim, int res_out_dim, int kernel_size)
{   
    double **filter = createMatrix_2D(in_dim, kernel_size);   
    double **featureMaps = createMatrix_2D(res_out_dim, *col-kernel_size+1);
    int i = 0, j = 0, k = 0, count = 0;
    char str[10];
    float data = 0.0;
    FILE *fp = fopen(weightFile, "r");   
    if(fp == NULL)
        printf("Resnet file open failed\n");
    else
    {                  
        /*initial featureMaps*/
        for(i=0; i&lt;res_out_dim; i++)
        {
            for(j=0; j&lt;*col-kernel_size+1; j++)
            {
                featureMaps[i][j] = 0.0;
            }
        }

        /*next filter*/        
        for(i=0; i&lt;res_out_dim; i++) 
        {
            /*read filter*/
            for(j=0; j&lt;in_dim; j++) 
            {
                for(k=0; k&lt;kernel_size; k++) 
                {
                    fscanf(fp, "%s", str);
                    sscanf(str, "%x", &amp;data);
                    filter[j][k] = (double)data;                                        
                }               
            }

            /* (part of source * filter) */            
            while(count &lt; *col-kernel_size+1)
            {                
                for(j=0; j&lt;in_dim; j++)
                {                                       
                    for(k=count; k&lt;kernel_size+count; k++)
                    {
                        featureMaps[i][count] += source[j][k]*filter[j][k-count];
                    }
                }               
                count++;               
            }
            count = 0;          
        }
        fclose(fp);
    }
    parameterFree(source, *row);
    parameterFree(filter, in_dim);
    *row = res_out_dim;
    *col = *col-kernel_size+1;       

    return featureMaps;  
}

int main()
{
    int row = 80;
    int col = 327;
    int in_dim = 80;
    int res_out_dim = 128;
    int kernel_size = 5;
    int i, j;
    float data;
    char str[10];

    double **input = createMatrix_2D(row, col);
    FILE *fp = fopen(CONV1D_INPUT, "r");
    FILE *fp2 = fopen("C code_CONV1D_OUTPUT.txt", "w");
    if(fp == NULL)
        printf("File open failed\n");
    else
    {
        for(i=0; i&lt;row; i++) 
        {
            for(j=0; j&lt;col; j++) 
            {
                fscanf(fp, "%s", str);
                sscanf(str, "%x", &amp;data);
                input[i][j] = (double)data;                                        
            }               
        }
    }

    double **CONV1D_ANS = conv_1D(CONV1D_WEIGHT, input, &amp;row, &amp;col, in_dim, res_out_dim, kernel_size);

    for(i=0; i&lt;row; i++) 
    {
        for(j=0; j&lt;col; j++) 
        {
            fprintf(fp2, "[%.18f] ", CONV1D_ANS[i][j]);
        }
        fprintf(fp2, "\n");
    }

    return 0;
}



Floating point numbers are not precise (by design). Depending on in which order operations are performed, the results might vary. Even worse, some formulas are straight numerical unstable, whereas another one for the same analytical expression can be stable.

Compilers often rearange statements as an optimization measure. Convolution is an operation which contains notoriously many operations and loops. So unless you directly compare executed bytecode, this speculation is kind of pointless.
