# -*- coding: utf-8 -*-
"""
Created on Wed May  6 11:08:41 2020

Polynomial Regression with Lasso Regularization

@author: Dr. Dr. Danny E. P. Vanpoucke
@web   : https://dannyvanpoucke.be
"""
import pandas as pd
from TModelClass import TModelClass
import numpy as np
from TModelResults import TModelResults
from Bootstrap import TBootstrap, Bootstrap_1Col
from TModelQualityData import TModelQualityData
from sklearn.pipeline import Pipeline

#from sklearn.utils.testing import ignore_warnings
from sklearn.exceptions import ConvergenceWarning


class TPolynomialLassoModel(TModelClass):
    """
    Child class representing the polynomial regression model, using Lasso Regularisation
    """
    def __init__(self,name,Target, Feature: pd.DataFrame, 
                 Target_test, Feature_test: pd.DataFrame,
                 Pipeline: Pipeline,
                 EnsemblePipeline: Pipeline,
                 Degree: int=2, Interaction: bool=False, 
                 Bias: bool=True, CrossVal: int=5):
        """
        Constructor to set up a polynomial model with Lasso Regularisation. The Alpha 
        values are obtained via cross-validation. 100 alpha values are checked along the path.
    
        parameters:
         - name : the name of the object instance
         - Feature : The features to use & transform
         - Target     : the training target data
         - Target_test: the test target data
         - Feature_test: the untransformed features for testing.
         - Pipeline : a pipeline generated by the PipelineFactory
         - EnsemblePipeline : the pipeline generated by the PipelineFactory for the entire ensemble dataset
         - Degree  : The polynomial degree (integer), DEFAULT=2
         - Interaction : Boolean variable indicating if ONLY interaction terms are included in the 
                     polynomial features (True) or that all terms are included (False), DEFAULT=False
         - Bias    : Boolean indicating if a bias column is added (i.e. where all powers 
                      are zero, acts as intercept in linear model). [DEFAULT = True]
         - CrossVal: How-many-fold crossvalidation is required? [DEFAULT = 5]
     
        It sets the following properties
            - pipeline : a pipeline object containing the preprocessing transformations (excluding the fitter function)
            - CVmodel  : the fitter function to be used (should be an sklearn function with "fit" method)
            - model : The model is only set once the CV_model has run a fit-operation
            - feature_tf: the transformed features as obtained by the pipeline  
            
            - parJobs : number of cores to use. Go full parallel on serial case, and single core on parallel code
        parameters sets for accounting/tracking purposes
            - crossVal : the Crossvalidation model/size (=CrossVal parameter) 
            - bestAlpha: best alpha-hyperparameter as obtained by ElasticNetCV
            - bestL1_ratio: best l1_ratio-hyperparameter as obtained by ElasticNetCV
            - cv_iter : number of iterations needed by ElasticNetCV
            - cv_warns: %-number of convergence-warnings thrown during ElasticNetCV = #warnings/(#alphas * #l1_ratio)
            
        """
        #from sklearn.preprocessing import StandardScaler
        #from sklearn.preprocessing import PolynomialFeatures
        from sklearn.linear_model import LassoCV
        from multiprocessing import current_process
    
        super().__init__(name,Target, Feature,Target_test, Feature_test)
        if current_process().name == 'MainProcess':
            print('Hello from the main process')
            self.parJobs=-1
        else:
            print('Hello from child process')
            self.parJobs=1
        
        
        self.nameModel='Polynomial Model with Lasso Regularisation'
        self.name=name
        print("Initialising the child class:",self.nameModel)
        #create a pipeline (can be extended to contain more functions, p67)
        self.pipeline = Pipeline
        self.EnsPipe = EnsemblePipeline      
        self.feature_tf = self.pipeline.fit_transform(Feature) #this is a numpy array...
                            
        #track the r1 and alpha lists for the CV model
        self.n_alphas=13 # 100 is the sklearn default
        self.alphas=[1.0e-6,1.0e-5,1.0e-4,1.0e-3,1.0e-2,0.1,1.0,10.0,100.0,1.0e3,1.0e4,1.0e5,1.0e6]
        
        self.CVmodel = LassoCV(
                         #eps=1e-8, #length of the path: Alpha_min/Alpha_max = 1.0E-3
                         #n_alphas=self.n_alphas, #test 100(default) alphas
                         alphas=self.alphas, #set the alphas automatically
                         fit_intercept=True, #the data is already centered by standard scaler
                         normalize=False,#standardization happens elsewhere, note that this is a tricky thing for small data
                         precompute='auto',   #let sklearn decide if the Gram matrix needs precomputing
                         max_iter = 1000,#maximum number of iterations
                         tol = 1.0E-3,
                         cv=CrossVal,         #X-fold crossvalidation--> for our small data: leave-one-out?
                         n_jobs=self.parJobs  #if you use -1 (= go parallel over all cores), you can not suppress the annoying convergence warning
                        )
        #to keep track of some options:
        self.crossVal=CrossVal
        self.bestAlpha=0
        self.cv_iter=0
        self.cv_warns=0 # %-number of warnings thrown during ElasticNetCV = #warnings/(#alphas * #l1_ratio)
        self.model=None
        self.coefUsage=None # contains the % of non-zero values of each coefficient over the model instances of the ensemble, is set upon creation of the average
        self.sklCVfail=False # are there reasons to believe the sk-learn CV failed miserably?
    
    def __PrintAlphMSESeries(self,avg,vmin,vmax,nrow):
        """
            Printing function to be used to trace LASSO alphas
        """
        f=open("ConvLASSO.dat","a+")
        for nr in range(nrow):
            line=" %0.8f  %0.8f  %0.8f  %0.8f  \n" %(self.CVmodel.alphas_[nr],avg[nr],vmax[nr]-avg[nr],avg[nr]-vmin[nr])
            f.write(line)
        line=" \n"
        f.write(line)
        f.close()
    
    def __GenerateLassoPath(self, gridType: str=None):
        """
        Some extra internal heuristics catching failures on small data.
        It seems like we are in trouble with the error getting worse for smaller regularisation.
        sk-learn CV optimisation fails. Try having a look at the lasso_path, where do we have 
        non-zero coefs?
        
        Parameters:
            - gridType : "piecelin", "lin" [default piecelin]
        
        returns:
            - set the CVmodel.alpha_ to the "beter" choice alpha value
            - set the alphas grid to a new grid
        
        """
        from sklearn.linear_model import lasso_path
        
        alphas, coef_path, _ = lasso_path(
                 X=self.feature_tf, #training data
                 y=self.target,     #targets
                 alphas=self.alphas, #set the alphas automatically
                 precompute='auto'   #let sklearn decide if the Gram matrix needs precomputing
                 )
        cntNoZero=np.count_nonzero(coef_path,axis=0)
        self.sklCVfail=True #so standard CV failed...we need to take over
        
        update=False
        #f=open("PathLASSO.dat","a+")
        for nr in range(len(alphas)):#alphas go from big to small
            if ((cntNoZero[nr]>0)and(not update)):
                update=True
                ca=alphas[nr]
                #as the sk-learn at this point will have failed miserably, 
                #replace the sklearn "best" alpha with a less epic fail
                self.CVmodel.alpha_=ca #other attributes of the CV model are not used (phew)
                if (gridType=="lin"):#linear
                    self.alphas=list(i*0.1*ca for i in range(1,21))
                    self.n_alphas=20
                else: #default: peisewise linear
                    self.alphas=list(i*0.1*ca for i in range(1,10))
                    self.alphas.extend(list(i*ca for i in range(1,10)))
                    self.n_alphas=18
                #break#get out of this loop and do not add more alphas
            #line=str(alphas[nr])+" "+str(coef_path[:,nr]).strip('[]')+"==> "+str(cntNoZero[nr])+" \n"
            #f.write(line)
        #line=" \n"
        #f.write(line)
        #f.close()    
    
    def __GenerateLinAlphaSeries(self, AtBest:bool=False ,Print: bool=False)-> bool:
        """
        Small private function to create a piecewise linear grid at the step of the average MSE plot
        
        !! WARNING: We assume sk-learn has sorted the alphas largest to smallest (this is the case for version 0.21.2...but this is python)
        
        Parameters:
            - Print : print the curve, Boolean, Default=False
            - AtBest : Generate a linear grid around the "Best" Alpha (True) or around the Log-dropoff (False, Default)
        
        Returns: Boolean indicating if the grid was updated, and shoul dbe run again
        """
        #find the drop of point
        update=False
        if (AtBest): # close top the best value: from 0.1 to 2x best
            ca=self.CVmodel.alpha_
            self.alphas=list(i*0.1*ca for i in range(1,21))
            self.n_alphas=20
            update=True
        else:
            avg=np.average(self.CVmodel.mse_path_,axis=1) #go over all columns(axis=1), to get the minimum of a row, axis=0
            nrow=len(avg)
            if (Print):
                vmin=np.amin(self.CVmodel.mse_path_,axis=1)
                vmax=np.amax(self.CVmodel.mse_path_,axis=1)
                self.__PrintAlphMSESeries(avg,vmin,vmax,nrow)
    
            for nr in range(nrow-1): #the alphas_ are sorted from large to small
                diff=avg[nr+1]-avg[nr]
                if ((diff<0.0)and(abs(diff)>(0.05*avg[nr]))):#drop needs to be larger than 5%
                #if ((abs(diff)>(0.05*avg[nr]))):#drop needs to be larger than 5%--> sometimes there is an increase...illogical as bias(avg error) should increase...but also case in Menon paper
                    ca=self.CVmodel.alphas_[nr+1]
                    self.alphas=list(i*0.1*ca for i in range(1,10))
                    self.alphas.extend(list(i*ca for i in range(1,10)))
                    self.n_alphas=18
                    update=True
                    break#get out of this loop and do not add more alphas
            
            if (not update): #it seems like we are in trouble with the error getting 
                             #worse for smaller regularisation
                #try having a look at the lasso_path, where do we have non-zero coefs?
                self.__GenerateLassoPath(gridType="piecelin")
                update=True
                
        return update
    
    
    #@ignore_warnings(category=ConvergenceWarning)    
    def fit(self):
        """
        Class-method wrapping the fit-method of the sklearn model
           - Target : a pandas dataframe with the Target data belonging to the 
                   Features provided upon initialisation.
        """
        import warnings
        import numpy as np
        from sklearn.linear_model import Lasso
        PrintMSEcurve=False
        
        with warnings.catch_warnings(record=True) as caught_warnings:
            warnings.simplefilter("always",category=ConvergenceWarning)
            self.CVmodel.fit(self.feature_tf,self.target)
        warnCount=len(caught_warnings)
        
        update=self.__GenerateLinAlphaSeries(AtBest=False, Print=False)
                
        if update:
            params=dict()
            params['alphas']=self.alphas
            self.CVmodel.set_params(**params)#clean way to update the model as we only wish to change the alphas
            #run the piece-wise linear grid            
            with warnings.catch_warnings(record=True) as caught_warnings:
                warnings.simplefilter("always",category=ConvergenceWarning)
                self.CVmodel.fit(self.feature_tf,self.target)
            warnCount=len(caught_warnings)
            if (np.count_nonzero(self.CVmodel.coef_)==0): #no remaining coefficients--> check the path for better option
                self.__GenerateLassoPath(gridType="lin")
            if (PrintMSEcurve):
                avg=np.average(self.CVmodel.mse_path_,axis=1) #go over all columns(axis=1), to get the minimum of a row, axis=0
                vmin=np.amin(self.CVmodel.mse_path_,axis=1)
                vmax=np.amax(self.CVmodel.mse_path_,axis=1)
                nrow=len(avg)
                self.__PrintAlphMSESeries(avg,vmin,vmax,nrow)
        
        
        totcnt=self.CVmodel.alphas_.size
        self.cv_warns=(warnCount/totcnt)*100.0
        #keep track of the best alpha and mixing, and use these to set up the actual model
        if (not self.sklCVfail):
            self.bestAlpha=self.CVmodel.alpha_
        else:
            self.bestAlpha=self.CVmodel.alpha_*0.50 #reduce regularization somewhat to avoid purely intercept models
        self.cv_iter=self.CVmodel.n_iter_
        
        self.model=Lasso(alpha=self.bestAlpha,
                         fit_intercept=True, #the data is already centered by standard scaler...but that seems not to be relevant/ or what is meant by this
                         normalize=False, #standardization is performed elsewhere
                         precompute=True     #precompute the Gram matrix (default)
                         )
        print("=== MODEL WAS OPT HERE : best alpha=",self.bestAlpha,
              " in [",self.CVmodel.alphas_[-1],", ",self.CVmodel.alphas_[0],"] ",
              " n-alpha=",self.n_alphas, " #WARN-->",warnCount,'/',totcnt," : ",self.cv_warns," % ================")
        #print(na,": #WARN-->",warnCount,'/',totcnt," : ",self.cv_warns," % | a=",self.bestAlpha," r1=",self.bestL1_ratio)
        #print(" %5i : #WARN--> %5i / %5i : %9.4f %%  | a=  %9.4f  r1=  %9.4f "%(na,warnCount,totcnt,self.cv_warns,self.bestAlpha,self.bestL1_ratio) )
        with warnings.catch_warnings(record=True) as caught_warnings:
            warnings.simplefilter("always",category=ConvergenceWarning)
            self.model.fit(self.feature_tf,self.target)
        self.setCoefficients()
        NZC=np.count_nonzero(self.model.coef_)
        print("did some fitting, Parent-style:",type(self.model).__name__,"   --> #non-zero coef=",f'{NZC: 5d}' )
#        f=open("ZeroCoef.dat","a+")
#        line="   --> #non-zero coef="+f'{NZC: 5d}'+" \n"
#        f.write(line)
#        f.close()
              
              
    #@ignore_warnings(category=ConvergenceWarning)   
    def fitSanityCheck(self)->int:
        """
        Class method which should cover/deal with failures of sklearn.
        
        For some reason, sklearn LinearRegression randomly fails on small datasets.
        This failure gives rise to huge coefficents. Hoever, just shuffling the 
        data seems to resolve the issue.
        
        This function returns the number of shuffles needed to regain sanity.
        """
        #import sys
        import warnings
        import numpy as np
        from sklearn.linear_model import Lasso

        #If we had too many warnings then something probably went wrong
        #There are two ways to fix the warnings:
        # 1) an alpha range closer to the best one
        # 2) more iterations
        # So let us try both 2x
        #start while loop untill it works...or n_alphas becomes too big?
        cnt=0
        insane=False
        PrintMSEcurve=False
        
        if (not self.sklCVfail): # normal sanitycheck
            thress=1.0
             #larger than 1% warnings & not on the edge
            insane=(self.cv_warns>thress)or(self.CVmodel.alpha_==self.CVmodel.alphas_[-1])or(self.CVmodel.alpha_==self.CVmodel.alphas_[0])
            warnCount=0
            totcnt=self.CVmodel.alphas_.size
            
            while (insane and (cnt<5)): #more than 5% warnings, and less than 10K alphas
                cnt+=1
                if (cnt % 2 == 0):#even: increase n_iter to 10K
                    params=dict()
                    params['max_iter']=10000#10x default
                    self.CVmodel.set_params(**params)#clean way to update the model as we only wish to change the alphas
                    #run the piece-wise linear grid            
                    with warnings.catch_warnings(record=True) as caught_warnings:
                        warnings.simplefilter("always",category=ConvergenceWarning)
                        self.CVmodel.fit(self.feature_tf,self.target)
                    warnCount=len(caught_warnings)
                else:#odd = new alpha range
                    update=self.__GenerateLinAlphaSeries(AtBest=True, Print=False)
                    if update:
                        params=dict()
                        params['alphas']=self.alphas
                        params['max_iter']=1000#back to our default
                        self.CVmodel.set_params(**params)#clean way to update the model as we only wish to change the alphas
                        #run the piece-wise linear grid            
                        with warnings.catch_warnings(record=True) as caught_warnings:
                            warnings.simplefilter("always",category=ConvergenceWarning)
                            self.CVmodel.fit(self.feature_tf,self.target)
                        warnCount=len(caught_warnings)
                        if (PrintMSEcurve):
                            avg=np.average(self.CVmodel.mse_path_,axis=1) #go over all columns(axis=1), to get the minimum of a row, axis=0
                            vmin=np.amin(self.CVmodel.mse_path_,axis=1)
                            vmax=np.amax(self.CVmodel.mse_path_,axis=1)
                            nrow=len(avg)
                            self.__PrintAlphMSESeries(avg,vmin,vmax,nrow)
                    
                totcnt=self.CVmodel.alphas_.size
                self.cv_warns=(warnCount/totcnt)*100.0
                insane=(self.cv_warns>thress)or(self.CVmodel.alpha_==self.CVmodel.alphas_[-1])or(self.CVmodel.alpha_==self.CVmodel.alphas_[0])
            
                #insane=(self.cv_warns>thress) #larger than 1% warnings    
                #keep track of the best alpha and mixing, and use these to set up the actual model
                self.bestAlpha=self.CVmodel.alpha_
                self.cv_iter=self.CVmodel.n_iter_
                print("=*= MODEL WAS OPT HERE : best alpha=",self.bestAlpha,
                  " in [",self.CVmodel.alphas_[-1],", ",self.CVmodel.alphas_[0],"] ",
                  " n-alpha=",self.n_alphas, " #WARN-->",warnCount,'/',totcnt," : ",self.cv_warns," % =====SANITY======")
            
            if (cnt>0):# only generate new non-CV model once
                #after sanity, setup our "Lasso" model
                self.model=Lasso(alpha=self.bestAlpha,
                             fit_intercept=True, #the data is already centered by standard scaler...but that seems not to be relevant/ or what is meant by this
                             normalize=False, #standardization is performed elsewhere
                             precompute=True     #precompute the Gram matrix (default)
                             )
                with warnings.catch_warnings(record=True) as caught_warnings:
                    warnings.simplefilter("always",category=ConvergenceWarning)
                    self.model.fit(self.feature_tf,self.target)
                self.setCoefficients()
        else: # the sanitycheck if the average error of the CV becomes larger due to terms
#            NZC=np.count_nonzero(self.model.coef_)
            insane=(np.count_nonzero(self.model.coef_)==0)
            while (insane and (cnt<5)): #more than 5% warnings, and less than 10K alphas
                cnt+=1
                self.bestAlpha=self.bestAlpha*0.5 #halve the alpha (up to 4x --> = x0.0625)
                self.model=Lasso(alpha=self.bestAlpha,
                             fit_intercept=True, #the data is already centered by standard scaler...but that seems not to be relevant/ or what is meant by this
                             normalize=False, #standardization is performed elsewhere
                             precompute=True     #precompute the Gram matrix (default)
                             )
                with warnings.catch_warnings(record=True) as caught_warnings:
                    warnings.simplefilter("always",category=ConvergenceWarning)
                    self.model.fit(self.feature_tf,self.target)
                insane=(np.count_nonzero(self.model.coef_)==0)
#            if (NZC==0):
#                f=open("ZeroCoef.dat","a+")
#                line="   ++> #non-zero coef="+f'{NZC: 5d}'+" Fixed: "+str(not insane)+" \n"
#                f.write(line)
#                f.close()
            
            self.setCoefficients()#only update the coeficients for the last case
                
        if insane:
            print("EPIC FAIL? The sanity check did not appear to fix our problem ",self.name,
                  ". NOT Terminating this sick job! ==> sk-learn CV-fail= ",self.sklCVfail)
            #sys.exit()
        
        return cnt

##serial version
#    def setAverageCoefficients(self, EnsembleData: TModelResults, setCI: bool):
#        """
#        Use the ensemble data to create an "average" model, and set the "coefficients"
#        in the current model. This should be performed in each model separately
#        
#        --> needs to include hyper paramaters...how do we deal with multi-preference?
#        
#        """
#        from sklearn.linear_model import Lasso
#        #import time
#        
#        # 1. Calculate the average coefficients
#        # 1.1. transform them to arrays
#        #start = time.perf_counter_ns()
#        #print("3.1) Average Coefficients : AVG")
#        intercept=np.zeros(EnsembleData.NData)
#        coef=np.zeros((EnsembleData.NData,EnsembleData.modelCoef[0]['coef_'][1].shape[1]))
#        for i in range(EnsembleData.NData):
#            mcf=EnsembleData.modelCoef[i]
#            intercept[i]=np.asarray(mcf['intercept_'][1]).ravel()
#            coef[i,:]=np.asarray(mcf['coef_'][1]).ravel()
#            print(i,")",coef[i,:])
#            
#        mean_intercept=np.mean(intercept,axis=0)#axis is the varying direction, so 0 means we calculate the average of a column by varying the row
#        mean_coef=np.mean(coef,axis=0) 
#        print("MEANS=> Int=",mean_intercept,"  COEF=",mean_coef)
#        # 2. Set the model coefficients to these averaged values
#        # ENR and sklearn black-boxing complicate things a bit here:
#        self.model=Lasso(alpha=self.bestAlpha,     # temp values -> will be mean?
#                         fit_intercept=True, #the data is already centered by standard scaler...but that seems not to be relevant/ or what is meant by this
#                         normalize=False, #standardization is performed elsewhere
#                         precompute=True     #precompute the Gram matrix (default)
#                         )
#       #self.model.fit(self.feature_tf,self.target) #just to make sure the intercept and coef attributes are defined--> fuck python 
#       
#        self.model.intercept_=mean_intercept
#        self.model.coef_=mean_coef
#        self.isAverage = True
#        self.hasCI=False
#        if setCI:
#            #end = time.perf_counter_ns()
#            #print("3.2.a) Average Coefficients : CI Intercept ",(end-start)/10E9)
#            # 3. Calculate Confidence Interval using Bootstrapper tech?
#            # & 4. Store the CI data
#            ## For the intercept
#            boot=TBootstrap(data=intercept,Func=np.mean)
#            #end = time.perf_counter_ns()
#            #print("3.2.b) NPboot",(end-start)/1E9)
#            boot.NPbootstrap(n_iter=2000, Jackknife=True)
#            #end = time.perf_counter_ns()
#            #print("3.2.c) Con Int",(end-start)/1E9)
#            avgm, avgp = boot.ConfidenceInterval(CItype="BCa",alpha=0.05,n_samples=2000)#95%confidence interval
#            self.CI["intercept_lo"]=avgm
#            self.CI["intercept_hi"]=avgp
#            ## For the coefficients
#            avgml=list()
#            avgpl=list()
#            for col in range(EnsembleData.modelCoef[0]['coef_'][1].shape[1]):
#                #end = time.perf_counter_ns()
#                #print("3.2) Average Coefficients : CI Coef ",col," ",(end-start)/1E9)
#                boot=TBootstrap(data=coef[:,col],Func=np.mean)
#                boot.NPbootstrap(n_iter=2000, Jackknife=True)
#                avgm, avgp = boot.ConfidenceInterval(CItype="BCa",alpha=0.05)#95%confidence interval
#                avgml.append(avgm)
#                avgpl.append(avgp)
#                
#            self.CI["coef_lo"]=avgml
#            self.CI["coef_hi"]=avgpl
#            self.hasCI = True
#            
#        #store the resulting coefficients in our wrapper tracker...and we are done
#        self.setCoefficients()
#        self.Quality=TModelQualityData(EData=EnsembleData)
            
#paralel 
    def setAverageCoefficients(self, EnsembleData: TModelResults, setCI: bool):
        """
        Use the ensemble data to create an "average" model, and set the "coefficients"
        in the current model. This should be performed in each model separately
        
        --> needs to include hyper paramaters...how do we deal with multi-preference?
        
        """
        from sklearn.linear_model import Lasso
        import multiprocessing as mp
        from HPCTools import get_num_procs
        #import time
        
        # 1. Calculate the average coefficients
        # 1.1. transform them to arrays
        #start = time.perf_counter_ns()
        #print("3.1) Average Coefficients : AVG")
        self.coefUsage=np.zeros(EnsembleData.modelCoef[0]['coef_'][1].shape[1])
        
        
        intercept=np.zeros(EnsembleData.NData)
        coef=np.zeros((EnsembleData.NData,EnsembleData.modelCoef[0]['coef_'][1].shape[1]))
        alphas=np.zeros(EnsembleData.NData)
        for i in range(EnsembleData.NData):
            mcf=EnsembleData.modelCoef[i]
            intercept[i]=np.asarray(mcf['intercept_'][1]).ravel()
            coef[i,:]=np.asarray(mcf['coef_'][1]).ravel()
            stra=mcf['alpha'][1]
            alphas[i]=stra.split()[3]
        
        for j in range(EnsembleData.modelCoef[0]['coef_'][1].shape[1]):
            self.coefUsage[j]=100.0*(np.count_nonzero(coef[:,j])/EnsembleData.NData) # The %-fraction of non-zero versions of each coefficient
        
        self.bestAlpha=np.mean(alphas,axis=0)
        mean_intercept=np.mean(intercept,axis=0)#axis is the varying direction, so 0 means we calculate the average of a column by varying the row
        mean_coef=np.mean(coef,axis=0) 
        print("MEANS=> Int=",mean_intercept,"  COEF=",mean_coef)
        # 2. Set the model coefficients to these averaged values
        # ENR and sklearn black-boxing complicate things a bit here:
        self.model=Lasso(alpha=self.bestAlpha,     # temp values -> will be mean?
                         fit_intercept=True, #the data is already centered by standard scaler...but that seems not to be relevant/ or what is meant by this
                         normalize=False, #standardization is performed elsewhere
                         precompute=True     #precompute the Gram matrix (default)
                         )
        #The black box nature of sklearn/python rears its ugly head here.
        #apparently we are not allowed to create an estimator ourselves by setting
        #the coefficients...ok maybe in this case we should replace the lasso with 
        #a normal poly since we make no use of the power-features of lasso
        #however, this is rather annoyng.
        # So just to kill the possibility of sklearn trowing an exception when 
        # predicting, just run it once and overwritte the intercept and coef attributes 
        # --> fuck you python 
        
        #print("FEATURE_TF=",self.feature_tf)
        
        #print("TARGET_TF=",self.target)
#        print("DICT=",self.model)
#        print("fit_intercept=", self.model.fit_intercept)
#        #print("__doc__=", self.model.__doc__)
#        print("_estimator_type=",self.model._estimator_type)
#        print("_preprocess_data=",self.model._preprocess_data)
#        print("selection=",self.model.selection)
#        print("__getattribute__=",self.model.__getattribute__)
#        #print("bool=",self.model.__bool__)
#        print("get_state pre fit =",self.model.__getstate__())
        #print("=",self.model.__setstate__())
        ftt2=[0,1,2,3,4,5,6]
        ftt=[[1,2,3],[0,0.1,0.0],[0.1,0.1,0.1],[3,2,2],[5.2,3,4],[0.0,0.2,0.3],[-0.1,-0.2,-0.3]]
        self.model.fit(ftt,ftt2)
#        print("get_state post fit=",self.model.__getstate__())
        #self.model.fit(self.feature_tf,self.target) #just to make sure the intercept and coef attributes are defined--> fuck python 
       
        self.model.intercept_=mean_intercept
        self.model.coef_=mean_coef
        self.isAverage = True
        self.hasCI=False
        #print("get_state post setting=",self.model.__getstate__())
        
        if setCI:
            #end = time.perf_counter_ns()
            #print("3.2.a) Average Coefficients : CI Intercept ",(end-start)/10E9)
            # 3. Calculate Confidence Interval using Bootstrapper tech?
            # & 4. Store the CI data
            ## For the intercept
            boot=TBootstrap(data=intercept,Func=np.mean)
            #end = time.perf_counter_ns()
            #print("3.2.b) NPboot",(end-start)/1E9)
            boot.NPbootstrap(n_iter=2000, Jackknife=True)
            #end = time.perf_counter_ns()
            #print("3.2.c) Con Int",(end-start)/1E9)
            avgm, avgp = boot.ConfidenceInterval(CItype="BCa",alpha=0.05,n_samples=2000)#95%confidence interval
            self.CI["intercept_lo"]=avgm
            self.CI["intercept_hi"]=avgp
            print("===BOOT INTERCEPT:",avgm,avgp)
            
            ## For the coefficients
            # Parallelisation for sections performing bootstraps. 
            # Parallelization at the highest level of a column, 
            # ??Is the overhead sufficiently low to have benefits?
            # 1. create our process pool with as many processes as physical cores
            pool=mp.Pool(processes=get_num_procs(-1))
            # 2. set drones to work
            alpha=0.05 #95%confidence interval
            
            print("col-range=",EnsembleData.modelCoef[0]['coef_'][1].shape[1])
            for col in range(EnsembleData.modelCoef[0]['coef_'][1].shape[1]):
                print("col",col," coef[:,col]=",coef[:,col]," type=",type(coef[:,col]))
            
            
            drones=[pool.apply_async(Bootstrap_1Col, args=(col,coef[:,col],alpha)) for col in range(EnsembleData.modelCoef[0]['coef_'][1].shape[1])]
            # 3. as we can not assume the cols to be produced in the correct order
            #    --> make it a dict
            ciDict=dict()
            for drone in drones:
                col,avgm,avgp=drone.get()
                ciDict[col]=list([avgm,avgp])
            # 4. wait untill all processes are finished
            pool.close()
            pool.join()
            # 5. and put then in the corrcet order in the list        
            avgml=list()
            avgpl=list()
            for col in range(EnsembleData.modelCoef[0]['coef_'][1].shape[1]):
                avgml.append(ciDict[col][0])
                avgpl.append(ciDict[col][1])
                
            self.CI["coef_lo"]=avgml
            self.CI["coef_hi"]=avgpl
            self.hasCI = True
            
        #store the resulting coefficients in our wrapper tracker...and we are done
        #print("Store resulting coefficients.")
        self.setCoefficients()
        self.Quality=TModelQualityData(EData=EnsembleData)
    
    
    def printAverageCoefficients(self, File: str=None):
        """
        Print a block of information to a file, containing the averaged coefficients for the 
        polynomial model.
        
        --> needs to include hyper paramaters...
        
        
        parameters:
            - self:
            - File: string containing a filename, if None standard output is used. Default=None
        """
        coefstr=list()
        maxl=0
        
        pipeline=None
        for name, step in self.EnsPipe.steps:
            if (name == 'poly_features'):
                pipeline=self.EnsPipe
        if pipeline==None:
            for name, step in self.pipeline.steps:
                if (name == 'poly_features'):
                    pipeline=self.pipeline
        
        
        for i in range(pipeline['poly_features'].powers_.shape[0]):
            line="("
            for j in range(pipeline['poly_features'].powers_.shape[1]):
                pw=pipeline['poly_features'].powers_[i,j]
                if (pw != 0):
                    line=line+"x_"+str(j)+"^"+str(pw)
            line=line+")"
            if len(line)==2: #so this term is missing
                line="(1)"
            curl=len(line) #keep track of the length of the longest coefficient string
            if (curl>maxl):
                maxl=curl
                
            coefstr.append(line)
        
        if File is None:
            print("======= THE AVERAGED MODEL ==============")
            print(" Model : ",self.name)
            print(self.Quality.QualitiesText())
            
            
            
            if self.hasCI:
                print("Intercept  : ",f'{self.model.intercept_: 12.7f}'," and CI=[",
                       f'{self.CI["intercept_lo"]: 12.7f}'," ; ",f'{self.CI["intercept_hi"]: 12.7f}',"]")
                for col in range(len(self.model.coef_)):
                    #print("coef ",coefstr[col]," : ",self.model.coef_[col]," and CI=[",self.CI["coef_lo"][col]," ; ",self.CI["coef_hi"][col],"]")
                    print("coef ",f'{coefstr[col]: <{maxl}}'," : ",f'{self.model.coef_[col]: 12.7f}',
                            " and CI=[",f'{self.CI["coef_lo"][col]: 12.7f}'," ; ",
                            f'{self.CI["coef_hi"][col]: 12.7f}',"]   usage= ",f'{self.coefUsage[col]: 8.3f}'," %")
            else:
                print("Intercept  : ",f'{self.model.intercept_: 12.7f}')
                for col in range(len(self.model.coef_)):
                    print("coef ",f'{coefstr[col]: <{maxl}}'," : ",f'{self.model.coef_[col]: 12.7f}',"   usage= ",f'{self.coefUsage[col]: 8.3f}'," %")
            print("====================================\n\n")
        else:
            foo=open(File,"a+",)
            foo.write("======= THE AVERAGED MODEL ==============\n")
            line=" Model : "+self.name+"\n"
            foo.write(line)
            foo.write(self.Quality.QualitiesText())
            if self.hasCI:
                line="Intercept  : "+f'{self.model.intercept_: 12.7f}'+" and CI=["+f'{self.CI["intercept_lo"]: 12.7f}'+" ; "+f'{self.CI["intercept_hi"]: 12.7f}'+"] \n"
                foo.write(line)
                for col in range(len(self.model.coef_)):
                    line="coef "+f'{coefstr[col]: <{maxl}}'+" : "+f'{self.model.coef_[col]: 12.7f}'+" and CI=["+f'{self.CI["coef_lo"][col]: 12.7f}'+" ; "+f'{self.CI["coef_hi"][col]: 12.7f}'+"]    usage= "+f'{self.coefUsage[col]: 8.3f}'+" % \n"
                    foo.write(line)
            else:
                line="Intercept  : "+f'{self.model.intercept_: 12.7f}'+"\n"
                foo.write(line)
                for col in range(len(self.model.coef_)):
                    line="coef "+f'{coefstr[col]: <{maxl}}'+" : "+f'{self.model.coef_[col]: 12.7f}'+"    usage= "+f'{self.coefUsage[col]: 8.3f}'+" % \n"
                    foo.write(line)
            foo.write("====================================\n\n")
            foo.close() 
    
    
    
               
    def setCoefficients(self):
        """
        Class-method printing the fitting coefficients for a polynomial regression 
        with elastic net regularization
        """
        import numpy as np
        super().setCoefficients()
        #--------- hyper parameters -------------------
        self.modelcoef['header_hyperparameter']=[self.coefindex,"The selected best alpha and mixing hyper-parameters for Elastic Net Regularization are:"]
        line="  - alpha     = %0.3f  " % (self.bestAlpha )
        self.modelcoef['alpha']=[-(self.coefindex+1),line]
        line="  - n_iter_CV = %i  " % (self.cv_iter )
        self.modelcoef['n_iter']=[-(self.coefindex+2),line]
        line="  - CV-warnings = %0.2f  %%" % (self.cv_warns )
        self.modelcoef['cv_warns']=[-(self.coefindex+3),line]
        
        #------------usual coefficients-----------------
        self.modelcoef['header_coef']=[self.coefindex+4,"The coefficients for each target (one per row) are given by:"]
        self.modelcoef['coef_']=[self.coefindex+5,np.array([self.model.coef_])]
        self.modelcoef['header_intercept']=[self.coefindex+6,"The intercepts for each target (one per row) are given by:"]
        self.modelcoef['intercept_']=[self.coefindex+7,np.array([self.model.intercept_])]
        self.coefindex+=8
        
        
