support/parse_data.py
def MNIST():
    '''
    Data preparation function, specific to the MNIST handwritten
    digits data set. 
    URL: http://yann.lecun.com/exdb/mnist/
    '''
    dataset = "MNIST"
    dinfo = classes.DataInfo()
    dinfo.mname = "LgstReg" # hard-coded model name.

    print("Preparation (", dataset, ")...")
    print("Inputs (training)...")
    toread = os.path.join(DATA_PATH,
                          dataset,
                          "train-images-idx3-ubyte")
    towrite = os.path.join("data", dataset, ("X_tr" + ".dat"))
    with open(toread, mode="rb") as f_bin:
    
        f_bin.seek(4)
        b = f_bin.read(4)
        n = int.from_bytes(b, byteorder="big")
        b = f_bin.read(4)
        d_rows = int.from_bytes(b, byteorder="big")
        b = f_bin.read(4)
        d_cols = int.from_bytes(b, byteorder="big")
        d = d_rows * d_cols

        with open(towrite, mode="bw") as g_bin:
    
            bytes_left = n * d
            idx = 0
            data_arr = np.empty( (n*d), dtype=np.uint8 )
            while bytes_left > 0:
                b = f_bin.read(1)
                data_arr[idx] = np.uint8(int.from_bytes(b, byteorder="big"))
                bytes_left -= 1
                idx += 1
            
            data_arr.tofile(g_bin)
            
    dinfo.X_tr["shape"] = (n,d)
    dinfo.X_tr["path"] = towrite
    dinfo.X_tr["dtype"] = np.uint8

    # --------------------------- #

    print("Inputs (testing)...")
    toread = os.path.join(DATA_PATH,
                          dataset,
                          "t10k-images-idx3-ubyte")
    towrite = os.path.join("data", dataset, ("X_te" + ".dat"))
    with open(toread, mode="rb") as f_bin:
    
        f_bin.seek(4)
        b = f_bin.read(4)
        n = int.from_bytes(b, byteorder="big")
        b = f_bin.read(4)
        d_rows = int.from_bytes(b, byteorder="big")
        b = f_bin.read(4)
        d_cols = int.from_bytes(b, byteorder="big")
        d = d_rows * d_cols

        with open(towrite, mode="bw") as g_bin:
        
            bytes_left = n * d
            idx = 0
            data_arr = np.empty( (n*d), dtype=np.uint8 )
            while bytes_left > 0:
                b = f_bin.read(1)
                data_arr[idx] = np.uint8(int.from_bytes(b, byteorder="big"))
                bytes_left -= 1
                idx += 1

            data_arr.tofile(g_bin)
            
    dinfo.X_te["shape"] = (n,d)
    dinfo.X_te["path"] = towrite
    dinfo.X_te["dtype"] = np.uint8

    # --------------------------- #

    print("Outputs (training)...")
    toread = os.path.join(DATA_PATH,
                          dataset,
                          "train-labels-idx1-ubyte")
    towrite = os.path.join("data", dataset, ("y_tr" + ".dat"))
    with open(toread, mode="rb") as f_bin:
    
        f_bin.seek(4)
        b = f_bin.read(4)
        n = int.from_bytes(b, byteorder="big")
        d = 1
        
        with open(towrite, mode="bw") as g_bin:
        
            bytes_left = n * d
            idx = 0
            data_arr = np.empty( (n*d), dtype=np.uint8 )
            while bytes_left > 0:
                b = f_bin.read(1)
                data_arr[idx] = np.uint8(int.from_bytes(b, byteorder="big"))
                bytes_left -= 1
                idx += 1

            data_arr.tofile(g_bin)

    dinfo.y_tr["shape"] = (n,d)
    dinfo.y_tr["path"] = towrite
    dinfo.y_tr["dtype"] = np.uint8


    # --------------------------- #

    print("Outputs (testing)...")
    toread = os.path.join(DATA_PATH,
                          dataset,
                          "t10k-labels-idx1-ubyte")
    towrite = os.path.join("data", dataset, ("y_te" + ".dat"))
    with open(toread, mode="rb") as f_bin:
    
        f_bin.seek(4)
        b = f_bin.read(4)
        n = int.from_bytes(b, byteorder="big")
        d = 1
    
        with open(towrite, mode="bw") as g_bin:
        
            bytes_left = n * d
            idx = 0
            data_arr = np.empty( (n*d), dtype=np.uint8 )
            while bytes_left > 0:
                b = f_bin.read(1)
                data_arr[idx] = np.uint8(int.from_bytes(b, byteorder="big"))
                bytes_left -= 1
                idx += 1
            
            data_arr.tofile(g_bin)

    dinfo.y_te["shape"] = (n,d)
    dinfo.y_te["path"] = towrite
    dinfo.y_te["dtype"] = np.uint8


    # Save the dinfo dictionary for future use (so we don't have to read
    # the original data every time).
    towrite = os.path.join("data", dataset, "info.dat")
    with open(towrite, mode="wb") as f:
        pickle.dump(dinfo, f)

    # Finally, return the dinfo dict.
    return dinfo

    Data preparation function, specific to the MNIST handwritten
    digits data set. 
    URL: http://yann.lecun.com/exdb/mnist/
    
Data preparation function, specific to the MNIST handwritten
digits data set.
Data preparation function specific to the MNIST handwritten digits data set .
def MNIST():
    dataset = "MNIST"
    dinfo = classes.DataInfo()
    dinfo.mname = "LgstReg" 
    print("Preparation (", dataset, ")...")
    print("Inputs (training)...")
    toread = os.path.join(DATA_PATH,
                          dataset,
                          "train-images-idx3-ubyte")
    towrite = os.path.join("data", dataset, ("X_tr" + ".dat"))
    with open(toread, mode="rb") as f_bin:
        f_bin.seek(4)
        b = f_bin.read(4)
        n = int.from_bytes(b, byteorder="big")
        b = f_bin.read(4)
        d_rows = int.from_bytes(b, byteorder="big")
        b = f_bin.read(4)
        d_cols = int.from_bytes(b, byteorder="big")
        d = d_rows * d_cols
        with open(towrite, mode="bw") as g_bin:
            bytes_left = n * d
            idx = 0
            data_arr = np.empty( (n*d), dtype=np.uint8 )
            while bytes_left > 0:
                b = f_bin.read(1)
                data_arr[idx] = np.uint8(int.from_bytes(b, byteorder="big"))
                bytes_left -= 1
                idx += 1
            data_arr.tofile(g_bin)
    dinfo.X_tr["shape"] = (n,d)
    dinfo.X_tr["path"] = towrite
    dinfo.X_tr["dtype"] = np.uint8
    print("Inputs (testing)...")
    toread = os.path.join(DATA_PATH,
                          dataset,
                          "t10k-images-idx3-ubyte")
    towrite = os.path.join("data", dataset, ("X_te" + ".dat"))
    with open(toread, mode="rb") as f_bin:
        f_bin.seek(4)
        b = f_bin.read(4)
        n = int.from_bytes(b, byteorder="big")
        b = f_bin.read(4)
        d_rows = int.from_bytes(b, byteorder="big")
        b = f_bin.read(4)
        d_cols = int.from_bytes(b, byteorder="big")
        d = d_rows * d_cols
        with open(towrite, mode="bw") as g_bin:
            bytes_left = n * d
            idx = 0
            data_arr = np.empty( (n*d), dtype=np.uint8 )
            while bytes_left > 0:
                b = f_bin.read(1)
                data_arr[idx] = np.uint8(int.from_bytes(b, byteorder="big"))
                bytes_left -= 1
                idx += 1
            data_arr.tofile(g_bin)
    dinfo.X_te["shape"] = (n,d)
    dinfo.X_te["path"] = towrite
    dinfo.X_te["dtype"] = np.uint8
    print("Outputs (training)...")
    toread = os.path.join(DATA_PATH,
                          dataset,
                          "train-labels-idx1-ubyte")
    towrite = os.path.join("data", dataset, ("y_tr" + ".dat"))
    with open(toread, mode="rb") as f_bin:
        f_bin.seek(4)
        b = f_bin.read(4)
        n = int.from_bytes(b, byteorder="big")
        d = 1
        with open(towrite, mode="bw") as g_bin:
            bytes_left = n * d
            idx = 0
            data_arr = np.empty( (n*d), dtype=np.uint8 )
            while bytes_left > 0:
                b = f_bin.read(1)
                data_arr[idx] = np.uint8(int.from_bytes(b, byteorder="big"))
                bytes_left -= 1
                idx += 1
            data_arr.tofile(g_bin)
    dinfo.y_tr["shape"] = (n,d)
    dinfo.y_tr["path"] = towrite
    dinfo.y_tr["dtype"] = np.uint8
    print("Outputs (testing)...")
    toread = os.path.join(DATA_PATH,
                          dataset,
                          "t10k-labels-idx1-ubyte")
    towrite = os.path.join("data", dataset, ("y_te" + ".dat"))
    with open(toread, mode="rb") as f_bin:
        f_bin.seek(4)
        b = f_bin.read(4)
        n = int.from_bytes(b, byteorder="big")
        d = 1
        with open(towrite, mode="bw") as g_bin:
            bytes_left = n * d
            idx = 0
            data_arr = np.empty( (n*d), dtype=np.uint8 )
            while bytes_left > 0:
                b = f_bin.read(1)
                data_arr[idx] = np.uint8(int.from_bytes(b, byteorder="big"))
                bytes_left -= 1
                idx += 1
            data_arr.tofile(g_bin)
    dinfo.y_te["shape"] = (n,d)
    dinfo.y_te["path"] = towrite
    dinfo.y_te["dtype"] = np.uint8
    towrite = os.path.join("data", dataset, "info.dat")
    with open(towrite, mode="wb") as f:
        pickle.dump(dinfo, f)
    return dinfo
def MNIST ( ) : dataset = "MNIST" dinfo = classes . DataInfo ( ) dinfo . mname = "LgstReg" print ( "Preparation (" , dataset , ")..." ) print ( "Inputs (training)..." ) toread = os . path . join ( DATA_PATH , dataset , "train-images-idx3-ubyte" ) towrite = os . path . join ( "data" , dataset , ( "X_tr" + ".dat" ) ) with open ( toread , mode = "rb" ) as f_bin : f_bin . seek ( 4 ) b = f_bin . read ( 4 ) n = int . from_bytes ( b , byteorder = "big" ) b = f_bin . read ( 4 ) d_rows = int . from_bytes ( b , byteorder = "big" ) b = f_bin . read ( 4 ) d_cols = int . from_bytes ( b , byteorder = "big" ) d = d_rows * d_cols with open ( towrite , mode = "bw" ) as g_bin : bytes_left = n * d idx = 0 data_arr = np . empty ( ( n * d ) , dtype = np . uint8 ) while bytes_left > 0 : b = f_bin . read ( 1 ) data_arr [ idx ] = np . uint8 ( int . from_bytes ( b , byteorder = "big" ) ) bytes_left -= 1 idx += 1 data_arr . tofile ( g_bin ) dinfo . X_tr [ "shape" ] = ( n , d ) dinfo . X_tr [ "path" ] = towrite dinfo . X_tr [ "dtype" ] = np . uint8 print ( "Inputs (testing)..." ) toread = os . path . join ( DATA_PATH , dataset , "t10k-images-idx3-ubyte" ) towrite = os . path . join ( "data" , dataset , ( "X_te" + ".dat" ) ) with open ( toread , mode = "rb" ) as f_bin : f_bin . seek ( 4 ) b = f_bin . read ( 4 ) n = int . from_bytes ( b , byteorder = "big" ) b = f_bin . read ( 4 ) d_rows = int . from_bytes ( b , byteorder = "big" ) b = f_bin . read ( 4 ) d_cols = int . from_bytes ( b , byteorder = "big" ) d = d_rows * d_cols with open ( towrite , mode = "bw" ) as g_bin : bytes_left = n * d idx = 0 data_arr = np . empty ( ( n * d ) , dtype = np . uint8 ) while bytes_left > 0 : b = f_bin . read ( 1 ) data_arr [ idx ] = np . uint8 ( int . from_bytes ( b , byteorder = "big" ) ) bytes_left -= 1 idx += 1 data_arr . tofile ( g_bin ) dinfo . X_te [ "shape" ] = ( n , d ) dinfo . X_te [ "path" ] = towrite dinfo . X_te [ "dtype" ] = np . uint8 print ( "Outputs (training)..." ) toread = os . path . join ( DATA_PATH , dataset , "train-labels-idx1-ubyte" ) towrite = os . path . join ( "data" , dataset , ( "y_tr" + ".dat" ) ) with open ( toread , mode = "rb" ) as f_bin : f_bin . seek ( 4 ) b = f_bin . read ( 4 ) n = int . from_bytes ( b , byteorder = "big" ) d = 1 with open ( towrite , mode = "bw" ) as g_bin : bytes_left = n * d idx = 0 data_arr = np . empty ( ( n * d ) , dtype = np . uint8 ) while bytes_left > 0 : b = f_bin . read ( 1 ) data_arr [ idx ] = np . uint8 ( int . from_bytes ( b , byteorder = "big" ) ) bytes_left -= 1 idx += 1 data_arr . tofile ( g_bin ) dinfo . y_tr [ "shape" ] = ( n , d ) dinfo . y_tr [ "path" ] = towrite dinfo . y_tr [ "dtype" ] = np . uint8 print ( "Outputs (testing)..." ) toread = os . path . join ( DATA_PATH , dataset , "t10k-labels-idx1-ubyte" ) towrite = os . path . join ( "data" , dataset , ( "y_te" + ".dat" ) ) with open ( toread , mode = "rb" ) as f_bin : f_bin . seek ( 4 ) b = f_bin . read ( 4 ) n = int . from_bytes ( b , byteorder = "big" ) d = 1 with open ( towrite , mode = "bw" ) as g_bin : bytes_left = n * d idx = 0 data_arr = np . empty ( ( n * d ) , dtype = np . uint8 ) while bytes_left > 0 : b = f_bin . read ( 1 ) data_arr [ idx ] = np . uint8 ( int . from_bytes ( b , byteorder = "big" ) ) bytes_left -= 1 idx += 1 data_arr . tofile ( g_bin ) dinfo . y_te [ "shape" ] = ( n , d ) dinfo . y_te [ "path" ] = towrite dinfo . y_te [ "dtype" ] = np . uint8 towrite = os . path . join ( "data" , dataset , "info.dat" ) with open ( towrite , mode = "wb" ) as f : pickle . dump ( dinfo , f ) return dinfo
Data preparation function, specific to the MNIST handwritten
digits data set.
Data preparation function specific to the MNIST handwritten digits data set .
'''
    Data preparation function, specific to the MNIST handwritten
    digits data set. 
    URL: http://yann.lecun.com/exdb/mnist/
    ''' # hard-coded model name. # --------------------------- # # --------------------------- # # --------------------------- # # Save the dinfo dictionary for future use (so we don't have to read # the original data every time). # Finally, return the dinfo dict.