merge_machine/abstract_data_project.py
def run_transform_module(self, module_name, partial_data, params):
        '''
        Runs the selected module on the dataframe in partial_data and stores
        modifications in the run_info buffer.
        '''
        logging.info('At module: {0}'.format(module_name))
        # Apply module transformation
        valid_columns = [col for col in partial_data if '__MODIFIED' not in col]     
        modified_columns = [col for col in partial_data if '__MODIFIED' in col]
        old_modified = partial_data[modified_columns]
        
        new_partial_data, modified = self.MODULES['transform'][module_name] \
                                    ['func'](partial_data[valid_columns], params)

        # Store modificiations in run_info_buffer
        self.run_info_buffer[(module_name, self.mem_data_info['file_name'])]['mod_count'] = \
            self._add_mod(self.run_info_buffer[(module_name, self.mem_data_info['file_name'])]['mod_count'], \
                         self._count_modifications(modified))

        modified.columns = [col + '__MODIFIED' for col in modified.columns]
        for col in modified.columns:
            if col in old_modified.columns:
                new_partial_data.loc[:, col] = old_modified.loc[:, col] | modified.loc[:, col]
            else:
                new_partial_data[col] = modified[col]
        return new_partial_data

        Runs the selected module on the dataframe in partial_data and stores
        modifications in the run_info buffer.
        
Runs the selected module on the dataframe in partial_data and stores
modifications in the run_info buffer.
Runs the selected module on the dataframe in partial_data and stores modifications in the run_info buffer .
def run_transform_module(self, module_name, partial_data, params):
        logging.info('At module: {0}'.format(module_name))
        valid_columns = [col for col in partial_data if '__MODIFIED' not in col]     
        modified_columns = [col for col in partial_data if '__MODIFIED' in col]
        old_modified = partial_data[modified_columns]
        new_partial_data, modified = self.MODULES['transform'][module_name] \
                                    ['func'](partial_data[valid_columns], params)
        self.run_info_buffer[(module_name, self.mem_data_info['file_name'])]['mod_count'] = \
            self._add_mod(self.run_info_buffer[(module_name, self.mem_data_info['file_name'])]['mod_count'], \
                         self._count_modifications(modified))
        modified.columns = [col + '__MODIFIED' for col in modified.columns]
        for col in modified.columns:
            if col in old_modified.columns:
                new_partial_data.loc[:, col] = old_modified.loc[:, col] | modified.loc[:, col]
            else:
                new_partial_data[col] = modified[col]
        return new_partial_data
def run_transform_module ( self , module_name , partial_data , params ) : logging . info ( 'At module: {0}' . format ( module_name ) ) valid_columns = [ col for col in partial_data if '__MODIFIED' not in col ] modified_columns = [ col for col in partial_data if '__MODIFIED' in col ] old_modified = partial_data [ modified_columns ] new_partial_data , modified = self . MODULES [ 'transform' ] [ module_name ] [ 'func' ] ( partial_data [ valid_columns ] , params ) self . run_info_buffer [ ( module_name , self . mem_data_info [ 'file_name' ] ) ] [ 'mod_count' ] = self . _add_mod ( self . run_info_buffer [ ( module_name , self . mem_data_info [ 'file_name' ] ) ] [ 'mod_count' ] , self . _count_modifications ( modified ) ) modified . columns = [ col + '__MODIFIED' for col in modified . columns ] for col in modified . columns : if col in old_modified . columns : new_partial_data . loc [ : , col ] = old_modified . loc [ : , col ] | modified . loc [ : , col ] else : new_partial_data [ col ] = modified [ col ] return new_partial_data
Runs the selected module on the dataframe in partial_data and stores
modifications in the run_info buffer.
Runs the selected module on the dataframe in partial_data and stores modifications in the run_info buffer .
'''
        Runs the selected module on the dataframe in partial_data and stores
        modifications in the run_info buffer.
        ''' # Apply module transformation # Store modificiations in run_info_buffer
self
module_name
partial_data
params