merge_machine/normalizer.py
def concat_with_init(self):
        '''
        Concatenates original table to data in memory (changes column names as well)
        
        TODO: merge with transform
        '''
        
        self._check_mem_data()
        
        # If data is in INIT, skip this step overall
        if self.mem_data_info['module_name'] == 'INIT':
            self.set_skip('concat_with_init', self.mem_data_info['file_name'], True)
            return {'TRUE DAT': 'real talk'}, {'yolo': 'yes indeed'}
        
        # Initiate log
        log = self._init_active_log('concat_with_init', 'transform')
    
        og_file_name = self.mem_data_info['file_name']
        og_file_path = self.path_to('INIT', og_file_name)

        og_tab = self._static_load_data(og_file_path, None, None)
        
        def _rename_column(col):
            if '__' in col:
                return col
            else:
                return col + '__NORMALIZED'
    
        def _my_concat(og_data, data):
            data.columns = [_rename_column(col) for col in data.columns]
            data = pd.concat([og_data, data], 1)
        
            # This should be same as selected columns: TODO: replace here
            base_modified_columns = set(x.split('__', 1)[0] for x in data.columns)
        
            # Re-order columns
            columns = []
            for col in og_data.columns:
                columns.append(col)
                if col in base_modified_columns:
                    modified_cols = filter(lambda x: col + '__' in x, data.columns)
                    columns.extend(modified_cols)
                    
            data = data[columns]
            return data
        
        self.mem_data = (_my_concat(og_data, data) for og_data, data in zip(og_tab, self.mem_data))
        
        self.mem_data_info['module_name'] = 'concat_with_init'
        
        run_info = {} # TODO: check specifications for run_info

        # Complete log
        log = self._end_active_log(log, error=False)
                          
        # Add time to run_info (# TODO: is this the best way?)
        run_info['start_timestamp'] = log['start_timestamp']
        run_info['end_timestamp'] = log['end_timestamp']
        run_info['params'] = {}
        
        # Update buffers
        self.log_buffer.append(log)
        self.run_info_buffer[('concat_with_init', og_file_name)] = run_info        
        return log, run_info

        Concatenates original table to data in memory (changes column names as well)
        
        TODO: merge with transform
        
Concatenates original table to data in memory (changes column names as well)
TODO: merge with transform
Concatenates original table to data in memory ( changes column names as well ) TODO : merge with transform
def concat_with_init(self):
        self._check_mem_data()
        if self.mem_data_info['module_name'] == 'INIT':
            self.set_skip('concat_with_init', self.mem_data_info['file_name'], True)
            return {'TRUE DAT': 'real talk'}, {'yolo': 'yes indeed'}
        log = self._init_active_log('concat_with_init', 'transform')
        og_file_name = self.mem_data_info['file_name']
        og_file_path = self.path_to('INIT', og_file_name)
        og_tab = self._static_load_data(og_file_path, None, None)
        def _rename_column(col):
            if '__' in col:
                return col
            else:
                return col + '__NORMALIZED'
        def _my_concat(og_data, data):
            data.columns = [_rename_column(col) for col in data.columns]
            data = pd.concat([og_data, data], 1)
            base_modified_columns = set(x.split('__', 1)[0] for x in data.columns)
            columns = []
            for col in og_data.columns:
                columns.append(col)
                if col in base_modified_columns:
                    modified_cols = filter(lambda x: col + '__' in x, data.columns)
                    columns.extend(modified_cols)
            data = data[columns]
            return data
        self.mem_data = (_my_concat(og_data, data) for og_data, data in zip(og_tab, self.mem_data))
        self.mem_data_info['module_name'] = 'concat_with_init'
        run_info = {} 
        log = self._end_active_log(log, error=False)
        run_info['start_timestamp'] = log['start_timestamp']
        run_info['end_timestamp'] = log['end_timestamp']
        run_info['params'] = {}
        self.log_buffer.append(log)
        self.run_info_buffer[('concat_with_init', og_file_name)] = run_info        
        return log, run_info
def concat_with_init ( self ) : self . _check_mem_data ( ) if self . mem_data_info [ 'module_name' ] == 'INIT' : self . set_skip ( 'concat_with_init' , self . mem_data_info [ 'file_name' ] , True ) return { 'TRUE DAT' : 'real talk' } , { 'yolo' : 'yes indeed' } log = self . _init_active_log ( 'concat_with_init' , 'transform' ) og_file_name = self . mem_data_info [ 'file_name' ] og_file_path = self . path_to ( 'INIT' , og_file_name ) og_tab = self . _static_load_data ( og_file_path , None , None ) def _rename_column ( col ) : if '__' in col : return col else : return col + '__NORMALIZED' def _my_concat ( og_data , data ) : data . columns = [ _rename_column ( col ) for col in data . columns ] data = pd . concat ( [ og_data , data ] , 1 ) base_modified_columns = set ( x . split ( '__' , 1 ) [ 0 ] for x in data . columns ) columns = [ ] for col in og_data . columns : columns . append ( col ) if col in base_modified_columns : modified_cols = filter ( lambda x : col + '__' in x , data . columns ) columns . extend ( modified_cols ) data = data [ columns ] return data self . mem_data = ( _my_concat ( og_data , data ) for og_data , data in zip ( og_tab , self . mem_data ) ) self . mem_data_info [ 'module_name' ] = 'concat_with_init' run_info = { } log = self . _end_active_log ( log , error = False ) run_info [ 'start_timestamp' ] = log [ 'start_timestamp' ] run_info [ 'end_timestamp' ] = log [ 'end_timestamp' ] run_info [ 'params' ] = { } self . log_buffer . append ( log ) self . run_info_buffer [ ( 'concat_with_init' , og_file_name ) ] = run_info return log , run_info
Concatenates original table to data in memory (changes column names as well)
TODO: merge with transform
Concatenates original table to data in memory ( changes column names as well ) TODO : merge with transform
'''
        Concatenates original table to data in memory (changes column names as well)
        
        TODO: merge with transform
        ''' # If data is in INIT, skip this step overall # Initiate log # This should be same as selected columns: TODO: replace here # Re-order columns # TODO: check specifications for run_info # Complete log # Add time to run_info (# TODO: is this the best way?) # Update buffers
self