merge_machine/normalizer.py
def run_all_transforms(self):
        '''Runs all modules on data in memory. And config from module names
        # TODO: move to abstract_data_project ?
        # TODO: check skipped
        '''
        self._check_mem_data()
        
        print(self.metadata['log'][self.mem_data_info['file_name']]['recode_types']['skipped'])
        
        print('mem_data_info:\n', self.mem_data_info)
        
        all_run_infos = {}
        # Only run all if there is a MINI version of the file
        if self.metadata['has_mini']:
            for module_name in self.MODULE_ORDER:
                if self.MODULES['transform'][module_name].get('use_in_full_run', False):
                    
                    # Module will be skipped if it has no __run_info.json
                    # OR IF it has "skipped" set to true in the log
                    if self.metadata['log'][self.mem_data_info['file_name']][module_name]['skipped']:
                        run_info = {'skipped': True}
                        logging.warning('WARNING: MODULE {0} WAS NOT RUN'.format(module_name))
                    else:
#                        try:                        
                        run_info_name = MINI_PREFIX + self.mem_data_info['file_name'] + '__run_info.json'
                        
                        # TODO: deal with this
                        if module_name != 'concat_with_init':
                            params = self.read_config_data(module_name, run_info_name)['params']
                        else:
                            params = None
                        
                        print('params:\n', params)
                        
                        # Load parameters from config files
                        print('run_all at', module_name)
                        _, run_info = self.transform(module_name, params)
                        print('run_all finished', module_name)
#                        except: 
#                            run_info = {'skipped': True}
#                            logging.warning('WARNING: MODULE {0} WAS NOT RUN'.format(module_name))
                    all_run_infos[module_name] = run_info
        else:
            logging.warning('run_all_transforms was called on a project without' \
                            ' a mini version. Nothing was done...')
Runs all modules on data in memory. And config from module names
        # TODO: move to abstract_data_project ?
        # TODO: check skipped
        
Runs all modules on data in memory. And config from module names
TODO: move to abstract_data_project .
TODO: check skipped
Runs all modules on data in memory . And config from module names TODO : move to abstract_data_project . TODO : check skipped
def run_all_transforms(self):
        self._check_mem_data()
        print(self.metadata['log'][self.mem_data_info['file_name']]['recode_types']['skipped'])
        print('mem_data_info:\n', self.mem_data_info)
        all_run_infos = {}
        if self.metadata['has_mini']:
            for module_name in self.MODULE_ORDER:
                if self.MODULES['transform'][module_name].get('use_in_full_run', False):
                    if self.metadata['log'][self.mem_data_info['file_name']][module_name]['skipped']:
                        run_info = {'skipped': True}
                        logging.warning('WARNING: MODULE {0} WAS NOT RUN'.format(module_name))
                    else:
                        run_info_name = MINI_PREFIX + self.mem_data_info['file_name'] + '__run_info.json'
                        if module_name != 'concat_with_init':
                            params = self.read_config_data(module_name, run_info_name)['params']
                        else:
                            params = None
                        print('params:\n', params)
                        print('run_all at', module_name)
                        _, run_info = self.transform(module_name, params)
                        print('run_all finished', module_name)
                    all_run_infos[module_name] = run_info
        else:
            logging.warning('run_all_transforms was called on a project without' \
                            ' a mini version. Nothing was done...')
def run_all_transforms ( self ) : self . _check_mem_data ( ) print ( self . metadata [ 'log' ] [ self . mem_data_info [ 'file_name' ] ] [ 'recode_types' ] [ 'skipped' ] ) print ( 'mem_data_info:\n' , self . mem_data_info ) all_run_infos = { } if self . metadata [ 'has_mini' ] : for module_name in self . MODULE_ORDER : if self . MODULES [ 'transform' ] [ module_name ] . get ( 'use_in_full_run' , False ) : if self . metadata [ 'log' ] [ self . mem_data_info [ 'file_name' ] ] [ module_name ] [ 'skipped' ] : run_info = { 'skipped' : True } logging . warning ( 'WARNING: MODULE {0} WAS NOT RUN' . format ( module_name ) ) else : run_info_name = MINI_PREFIX + self . mem_data_info [ 'file_name' ] + '__run_info.json' if module_name != 'concat_with_init' : params = self . read_config_data ( module_name , run_info_name ) [ 'params' ] else : params = None print ( 'params:\n' , params ) print ( 'run_all at' , module_name ) _ , run_info = self . transform ( module_name , params ) print ( 'run_all finished' , module_name ) all_run_infos [ module_name ] = run_info else : logging . warning ( 'run_all_transforms was called on a project without' ' a mini version. Nothing was done...' )
Runs all modules on data in memory.
Runs all modules on data in memory .
'''Runs all modules on data in memory. And config from module names
        # TODO: move to abstract_data_project ?
        # TODO: check skipped
        ''' # Only run all if there is a MINI version of the file # Module will be skipped if it has no __run_info.json # OR IF it has "skipped" set to true in the log #                        try:                         # TODO: deal with this # Load parameters from config files #                        except:  #                            run_info = {'skipped': True} #                            logging.warning('WARNING: MODULE {0} WAS NOT RUN'.format(module_name))
self