DataPipeline/Base.py
def run(self, callback=None):
        """Do initial import/fitting run with the current config"""

        self.stop=False
        self.loadModels()
        self.prepareData()
        print 'processing files in queue..'

        self.parseLabels()
        imported = {}   #raw data
        results = {}    #fitted data
        #print self.queue

        for key in self.queue:
            filename = self.queue[key]
            lines = self.openRaw(filename)
            if lines == None:
                continue
            data = self.doImport(lines)
            imported[key] = data

        #rebuild dict into a nested structure if it's flat (i.e. from single files)
        '''from Data import NestedData
        D = NestedData(imported)
        D.buildNestedStructure([0,2])
        D.show()
        imported = D.data
        self.namelabels = None'''

        #try to average replicates here before we process
        if self.replicates == 1:
            if self.namelabels != None:
                imported = Utilities.addReplicates(imported, self.namelabels)
            else:
                print 'no replicates detected from labels'

        #re-arrange the imported dict if we want to group our output per field
        if self.groupbyfields == 1:
            imported = Utilities.arrangeDictbySecondaryKey(imported, self.namelabels)

        total = len(imported)
        #print imported
        #print self.namelabels

        c=0.0
        for key in imported:
            if self.stop == True:
                print 'cancelled'
                return
            #set filename
            fname = os.path.basename(key)
            fname = os.path.join(self.workingdir, fname)
            data = imported[key]

            if self.function1 != '':
                data = self.doProcessingStep(data, fname)

            if self.namelabels == None or not self.namelabels.has_key(key):
                namelabel = key
            else:
                namelabel = self.namelabels[key]
            #print namelabel, key
            #print data

            #if we have models to fit this means we might need to propagate fit data
            if self.model1 != '':
                Em = EkinProject()
                #grouping by file labels handled here
                if self.groupbyname == 1:
                    #we don't pass the last model if it has to be
                    #reserved for a final round of fitting from the files dict
                    models = self.models[:-1]
                    variables = self.variables[:-1]
                    E,fits = self.processFits(rawdata=data, Em=Em,
                                               models=models,variables=variables)
                else:
                    E,fits = self.processFits(rawdata=data, Em=Em)
                results[namelabel] = fits
                #print E.datasets, namelabel
            else:
                #if no fitting we just put the data in ekin
                Em = Utilities.getEkinProject(data)
                results[namelabel] = data

            Em.saveProject(fname)
            Em.exportDatasets(fname, append=True)
            if self.model1 != '':
                self.saveFitstoCSV(Em, fname)
            if self.saveplots == 1:
                self.saveEkinPlotstoImages(Em, fname)
            c+=1.0
            if callback != None:
                callback(c/total*100)

        #if grouped by file names then we process that here from results
        if self.groupbyname == 1:
            results = Utilities.extractSecondaryKeysFromDict(results)
            Em = EkinProject()
            #print results
            E,fits = self.processFits(rawdata=results, Em=Em)
            fname = os.path.join(self.workingdir, 'final')
            Em.saveProject(os.path.join(self.workingdir, fname))
            Em.exportDatasets(os.path.join(self.workingdir, fname))
            if self.model1 != '':
                self.saveFitstoCSV(Em, fname)
            self.saveEkinPlotstoImages(Em, fname)
        print 'processing done'
        print 'results saved to %s' %self.workingdir
        self.results = results
        return results
Do initial import/fitting run with the current config
Do initial import/fitting run with the current config
Do initial import / fitting run with the current config
def run(self, callback=None):
        self.stop=False
        self.loadModels()
        self.prepareData()
        print 'processing files in queue..'
        self.parseLabels()
        imported = {}   
        results = {}    
        for key in self.queue:
            filename = self.queue[key]
            lines = self.openRaw(filename)
            if lines == None:
                continue
            data = self.doImport(lines)
            imported[key] = data
        if self.replicates == 1:
            if self.namelabels != None:
                imported = Utilities.addReplicates(imported, self.namelabels)
            else:
                print 'no replicates detected from labels'
        if self.groupbyfields == 1:
            imported = Utilities.arrangeDictbySecondaryKey(imported, self.namelabels)
        total = len(imported)
        c=0.0
        for key in imported:
            if self.stop == True:
                print 'cancelled'
                return
            fname = os.path.basename(key)
            fname = os.path.join(self.workingdir, fname)
            data = imported[key]
            if self.function1 != '':
                data = self.doProcessingStep(data, fname)
            if self.namelabels == None or not self.namelabels.has_key(key):
                namelabel = key
            else:
                namelabel = self.namelabels[key]
            if self.model1 != '':
                Em = EkinProject()
                if self.groupbyname == 1:
                    models = self.models[:-1]
                    variables = self.variables[:-1]
                    E,fits = self.processFits(rawdata=data, Em=Em,
                                               models=models,variables=variables)
                else:
                    E,fits = self.processFits(rawdata=data, Em=Em)
                results[namelabel] = fits
            else:
                Em = Utilities.getEkinProject(data)
                results[namelabel] = data
            Em.saveProject(fname)
            Em.exportDatasets(fname, append=True)
            if self.model1 != '':
                self.saveFitstoCSV(Em, fname)
            if self.saveplots == 1:
                self.saveEkinPlotstoImages(Em, fname)
            c+=1.0
            if callback != None:
                callback(c/total*100)
        if self.groupbyname == 1:
            results = Utilities.extractSecondaryKeysFromDict(results)
            Em = EkinProject()
            E,fits = self.processFits(rawdata=results, Em=Em)
            fname = os.path.join(self.workingdir, 'final')
            Em.saveProject(os.path.join(self.workingdir, fname))
            Em.exportDatasets(os.path.join(self.workingdir, fname))
            if self.model1 != '':
                self.saveFitstoCSV(Em, fname)
            self.saveEkinPlotstoImages(Em, fname)
        print 'processing done'
        print 'results saved to %s' %self.workingdir
        self.results = results
        return results
def run ( self , callback = None ) : self . stop = False self . loadModels ( ) self . prepareData ( ) print 'processing files in queue..' self . parseLabels ( ) imported = { } results = { } for key in self . queue : filename = self . queue [ key ] lines = self . openRaw ( filename ) if lines == None : continue data = self . doImport ( lines ) imported [ key ] = data '''from Data import NestedData
        D = NestedData(imported)
        D.buildNestedStructure([0,2])
        D.show()
        imported = D.data
        self.namelabels = None''' if self . replicates == 1 : if self . namelabels != None : imported = Utilities . addReplicates ( imported , self . namelabels ) else : print 'no replicates detected from labels' if self . groupbyfields == 1 : imported = Utilities . arrangeDictbySecondaryKey ( imported , self . namelabels ) total = len ( imported ) c = 0.0 for key in imported : if self . stop == True : print 'cancelled' return fname = os . path . basename ( key ) fname = os . path . join ( self . workingdir , fname ) data = imported [ key ] if self . function1 != '' : data = self . doProcessingStep ( data , fname ) if self . namelabels == None or not self . namelabels . has_key ( key ) : namelabel = key else : namelabel = self . namelabels [ key ] if self . model1 != '' : Em = EkinProject ( ) if self . groupbyname == 1 : models = self . models [ : - 1 ] variables = self . variables [ : - 1 ] E , fits = self . processFits ( rawdata = data , Em = Em , models = models , variables = variables ) else : E , fits = self . processFits ( rawdata = data , Em = Em ) results [ namelabel ] = fits else : Em = Utilities . getEkinProject ( data ) results [ namelabel ] = data Em . saveProject ( fname ) Em . exportDatasets ( fname , append = True ) if self . model1 != '' : self . saveFitstoCSV ( Em , fname ) if self . saveplots == 1 : self . saveEkinPlotstoImages ( Em , fname ) c += 1.0 if callback != None : callback ( c / total * 100 ) if self . groupbyname == 1 : results = Utilities . extractSecondaryKeysFromDict ( results ) Em = EkinProject ( ) E , fits = self . processFits ( rawdata = results , Em = Em ) fname = os . path . join ( self . workingdir , 'final' ) Em . saveProject ( os . path . join ( self . workingdir , fname ) ) Em . exportDatasets ( os . path . join ( self . workingdir , fname ) ) if self . model1 != '' : self . saveFitstoCSV ( Em , fname ) self . saveEkinPlotstoImages ( Em , fname ) print 'processing done' print 'results saved to %s' % self . workingdir self . results = results return results
Do initial import/fitting run with the current config
Do initial import / fitting run with the current config
"""Do initial import/fitting run with the current config""" #raw data #fitted data #print self.queue #rebuild dict into a nested structure if it's flat (i.e. from single files) '''from Data import NestedData
        D = NestedData(imported)
        D.buildNestedStructure([0,2])
        D.show()
        imported = D.data
        self.namelabels = None''' #try to average replicates here before we process #re-arrange the imported dict if we want to group our output per field #print imported #print self.namelabels #set filename #print namelabel, key #print data #if we have models to fit this means we might need to propagate fit data #grouping by file labels handled here #we don't pass the last model if it has to be #reserved for a final round of fitting from the files dict #print E.datasets, namelabel #if no fitting we just put the data in ekin #if grouped by file names then we process that here from results #print results
self
callback