<reponame>hpcc-systems/LinearRegression
##############################################################################
# HPCC SYSTEMS software Copyright (C) 2017 HPCC Systems.  All rights reserved.
###############################################################################
# This python program parses the ECL datafile and inserts (replaces) the analytic
# values produced by statsmodels.OLS.  These are used by OLSValidate.ecl as
# reference values for the various OLS analytics.
import numpy as np
import sklearn
import sklearn.linear_model as lm
import statsmodels.api as sm

# doit
#
# Converts one file by adding an autogenerated section
# containing the reference results for the Linear Regression
# and all of its analytic functions.
#
def doit(filepath):
    f = open(filepath)
    instr = f.read()
    f.close()

    # Find the data contents section and parse it
    start = instr.find("EXPORT content := DATASET([")
    start2 = instr.find('{', start)
    datastr = instr[start2:]
    end = datastr.find(']')
    datastr = datastr[:end]
    #print(instr[:100])
    lines = datastr.split('\n')
    #print(lines[:5])
    newLines = []
    for line in lines:
        line = line.strip()
        if line:
            # Strip the '{'
            line = line[1:]
        if line and line[-1] == ',':
            # Strip the ',' if present
            line = line[:-1]
        if line:
            line = line[:-1]
        if line:
          newLines.append(line)
    lines = newLines
    #print(lines[:5])
    # Form records of tokens
    tokrecs = [lines[j].split(',') for j in range(len(lines))]

    recs = []
    for line in tokrecs:
      #print('line = ', line)
      rec = [float(token.strip()) for token in line if len(token) > 0]
      recs.append(rec)

    indeps = [rec[1:-1] for rec in recs if len(rec) > 0]
    x = np.array(indeps)
    deps = [rec[-1] for rec in recs if len(rec) > 0]
    y = np.array(deps)

    # Add a constant 1 column for Y intercept
    xc = sm.tools.add_constant(x)

    # Create the linear regression including intercept
    ols = sm.OLS(y, xc, None, True)

    # Get the model (OLS_result object)
    olsmod = ols.fit()

    # Summarize the results for reference
    print('info(ols) = ', olsmod.summary())

    # Get the betas
    betas = olsmod.params

    # Format the autogeneration section
    outSection = """
    //*StartGen*
    // The following lines are auto-generated.
    // Do not make any manual changes below this point.
    // Note: Beta 1 is for the intercept
"""
    outSection += """    EXPORT betas := DATASET([
"""
    betastrlist = []
    for i in range(len(betas)):
      betastr = "      {1, %d, 1, %f}"%(i+1, betas[i])
      betastrlist.append(betastr)
    outbetas = ',\n'.join(betastrlist)
    outSection += outbetas
    outSection += """
    ], Types.NumericField);
"""
    outSection += "\n    EXPORT Rsquared := %f;\n"%(olsmod.rsquared,)
    outSection += "\n    EXPORT adjRsquared := %f;\n"%(olsmod.rsquared_adj,)
    outSection += "\n    EXPORT Total_SS := %f;\n"%(olsmod.centered_tss,)
    outSection += "\n    EXPORT Model_SS := %f;\n"%(olsmod.ess,)
    outSection += "\n    EXPORT Error_SS := %f;\n"%(olsmod.ssr,)
    outSection += "\n    EXPORT Total_DF := %f;\n"%(olsmod.df_model + olsmod.df_resid,)
    outSection += "\n    EXPORT Model_DF := %f;\n"%(olsmod.df_model,)
    outSection += "\n    EXPORT Error_DF := %f;\n"%(olsmod.df_resid,)
    outSection += "\n    EXPORT Model_MS := %f;\n"%(olsmod.mse_model,)
    outSection += "\n    EXPORT Error_MS := %f;\n"%(olsmod.mse_resid,)
    outSection += "\n    EXPORT Model_F := %f;\n"%(olsmod.fvalue,)
    outSection += "\n    EXPORT AIC := %f;\n"%(olsmod.aic,)
    outSection += "\n    EXPORT Ftest := %f;\n"%(olsmod.f_pvalue,)

    outSection += "\n    EXPORT SET OF REAL8 SE := %s;\n"%(repr(list(olsmod.bse)),)
    outSection += "\n    EXPORT SET OF REAL8 TStat := %s;\n"%(repr(list(olsmod.tvalues)),)
    outSection += "\n    EXPORT SET OF REAL8 Pval := %s;\n"%(repr(list(olsmod.pvalues)),)
    confintStarts = []
    confintEnds   = []
    confints95    = olsmod.conf_int(.05)
    for interval in confints95:
      confintStarts.append(interval[0])
      confintEnds.append(interval[1])
    outSection += "\n    EXPORT SET OF REAL8 ConfintStarts := %s;\n"%(repr(confintStarts),)
    outSection += "\n    EXPORT SET OF REAL8 ConfintEnds := %s;\n"%(repr(confintEnds),)
    outSection += """    //*EndGen*
"""
    #print(outSection)

    # Insert it into the file.  If there's already an
    # autogenerated section, replace it.  Otherwise, append
    # it to the bottom of the module code.
    agStart = instr.find('\n    //*StartGen*')
    if agStart >= 0:
      agEnd = instr.find('//*EndGen*\n', agStart)
      outstr = instr[:agStart] + outSection + instr[agEnd+11:]
    else:
      cut = instr.rfind('END;')
      outstr = instr[:cut] + outSection + instr[cut:]
    f = open(filepath + '.out', 'w')
    f.write(outstr)
    f.close()

import sys
filepath = sys.argv[1]

print('Test Dataset = ', filepath)
doit(filepath)
