merge_machine/abstract_data_project.py
def make_mini(self, params):
        '''
        Creates a smaller version of the table in memory. 
        Set mem_data_info and current file to mini
        
        INPUT:
            - params:
                sample_size: Number of rows to include in the "MINI" file
                randomize: chose sample_size elements randomly from the file in
                            memory
        '''
        # TODO: Change current 
        # TODO: Current for normalize ?        
        self._check_mem_data()
    
        # Set defaults
        sample_size = params.get('sample_size', self.CHUNKSIZE - 1)
        randomize = params.get('randomize', True)       
        new_file_name = self._mini_from_og(self.mem_data_info['file_name'])

        
        if self.mem_data_info['module_name'] != 'INIT':
            raise Exception('make_mini can only be called on data in memory from the INIT module')
       
        part_tab = next(self.mem_data)
        
        # Only create file if it is larger than sample size
        if part_tab.shape[0] > sample_size:            
            logging.info('Running make_mini')
            self.clean_after('INIT', new_file_name) # TODO: check module_name for clean_after
            
            # Initiate log
            log = self._init_active_log('INIT', 'transform')  # TODO: hack here: module_name should be 'make_mini'
            
            if randomize:
                sample_index = np.random.permutation(part_tab.index)[:sample_size]
            else:
                sample_index = part_tab.index[:sample_size]
            
            # Replace data in memory
            self.mem_data = (x for x in [part_tab.loc[sample_index, :]])
            
            # Update metadata and log
            self.metadata['has_mini'] = True
            self.mem_data_info['file_name'] = new_file_name
            
            # Create new empty log in metadata
            self.metadata['log'][new_file_name] = self._default_log()
        
            log['og_file_name'] = log['file_name']
            log['file_name'] = new_file_name
            
            # TODO: think if transformation should / should not be complete
    
            # Complete log
            log = self._end_active_log(log, error=False) 
        
            # TODO: Make sure that run_info_buffer should not be extended
            return log
        
        else:
            self.metadata['has_mini'] = False
            return {}

        Creates a smaller version of the table in memory. 
        Set mem_data_info and current file to mini
        
        INPUT:
            - params:
                sample_size: Number of rows to include in the "MINI" file
                randomize: chose sample_size elements randomly from the file in
                            memory
        
Creates a smaller version of the table in memory.
Set mem_data_info and current file to mini

Number of rows to include in the "MINI" file
randomize: chose sample_size elements randomly from the file in
memory
Creates a smaller version of the table in memory . Set mem_data_info and current file to mini Number of rows to include in the " MINI " file randomize : chose sample_size elements randomly from the file in memory
def make_mini(self, params):
        self._check_mem_data()
        sample_size = params.get('sample_size', self.CHUNKSIZE - 1)
        randomize = params.get('randomize', True)       
        new_file_name = self._mini_from_og(self.mem_data_info['file_name'])
        if self.mem_data_info['module_name'] != 'INIT':
            raise Exception('make_mini can only be called on data in memory from the INIT module')
        part_tab = next(self.mem_data)
        if part_tab.shape[0] > sample_size:            
            logging.info('Running make_mini')
            self.clean_after('INIT', new_file_name) 
            log = self._init_active_log('INIT', 'transform')  
            if randomize:
                sample_index = np.random.permutation(part_tab.index)[:sample_size]
            else:
                sample_index = part_tab.index[:sample_size]
            self.mem_data = (x for x in [part_tab.loc[sample_index, :]])
            self.metadata['has_mini'] = True
            self.mem_data_info['file_name'] = new_file_name
            self.metadata['log'][new_file_name] = self._default_log()
            log['og_file_name'] = log['file_name']
            log['file_name'] = new_file_name
            log = self._end_active_log(log, error=False) 
            return log
        else:
            self.metadata['has_mini'] = False
            return {}
def make_mini ( self , params ) : self . _check_mem_data ( ) sample_size = params . get ( 'sample_size' , self . CHUNKSIZE - 1 ) randomize = params . get ( 'randomize' , True ) new_file_name = self . _mini_from_og ( self . mem_data_info [ 'file_name' ] ) if self . mem_data_info [ 'module_name' ] != 'INIT' : raise Exception ( 'make_mini can only be called on data in memory from the INIT module' ) part_tab = next ( self . mem_data ) if part_tab . shape [ 0 ] > sample_size : logging . info ( 'Running make_mini' ) self . clean_after ( 'INIT' , new_file_name ) log = self . _init_active_log ( 'INIT' , 'transform' ) if randomize : sample_index = np . random . permutation ( part_tab . index ) [ : sample_size ] else : sample_index = part_tab . index [ : sample_size ] self . mem_data = ( x for x in [ part_tab . loc [ sample_index , : ] ] ) self . metadata [ 'has_mini' ] = True self . mem_data_info [ 'file_name' ] = new_file_name self . metadata [ 'log' ] [ new_file_name ] = self . _default_log ( ) log [ 'og_file_name' ] = log [ 'file_name' ] log [ 'file_name' ] = new_file_name log = self . _end_active_log ( log , error = False ) return log else : self . metadata [ 'has_mini' ] = False return { }
Creates a smaller version of the table in memory.
Creates a smaller version of the table in memory .
'''
        Creates a smaller version of the table in memory. 
        Set mem_data_info and current file to mini
        
        INPUT:
            - params:
                sample_size: Number of rows to include in the "MINI" file
                randomize: chose sample_size elements randomly from the file in
                            memory
        ''' # TODO: Change current  # TODO: Current for normalize ?         # Set defaults # Only create file if it is larger than sample size # TODO: check module_name for clean_after # Initiate log # TODO: hack here: module_name should be 'make_mini' # Replace data in memory # Update metadata and log # Create new empty log in metadata # TODO: think if transformation should / should not be complete # Complete log # TODO: Make sure that run_info_buffer should not be extended
self
params