merge_machine/linker.py
def update_results(self, labels):
        '''Updates the merged table in Elasticsearch to take into account the
        new labels.
        '''
        # TODO: source indices
        
        new_rows = []
        columns = set()
        for label in labels:
            current_row = es.get(self.index_name, 'structure', label['source_id'])['_source']
            if label['is_match']:                
                if current_row['__ID_REF'] != label['ref_id']:
                    new_ref = es.get(self.ref.project_id, 'structure', label['ref_id'])['_source']
                    new_ref = {key + '__REF': val for key, val in new_ref.items()}
                    new_row = {key: val for key, val in current_row.items()}
                    new_row.update(new_ref)
                    new_row['__IS_MATCH'] = True
                    new_row['__CONFIDENCE'] = 999
                    new_row['__ID_REF'] = label['ref_id']
                    
                    # TODO: what to do with __ES_SCORE, __ID_QUERY, __THRESH
                else:
                    new_row = {key: val for key, val in current_row.items()}
                    new_row['__IS_MATCH'] = True
                    new_row['__CONFIDENCE'] = 999
            else:
                new_row = {col: val for col, val in current_row.items()}
                
                nan_cols = list(filter(lambda x: x[-5:]=='__REF', new_row.keys())) \
                            + ['__CONFIDENCE', '__ES_SCORE', '__ID_QUERY', \
                               '__ID_REF', '__IS_MATCH', '__THRESH']
                
                for col in nan_cols:
                    new_row[col] = np.nan
                
            columns.update(new_row.keys())
            new_rows.append((label['source_id'], new_row))
            
        if new_rows:
            dtype = {col: self._choose_dtype(col) for col in columns}
            tab = pd.DataFrame([x[1] for x in new_rows], index=[x[0] for x in new_rows])
                
            # Fix for dtype that is not working in DataFrame call
            for k, v in dtype.items():
                if v == str:
                    tab[k].fillna('', inplace=True)
                tab[k] = tab[k].astype(v)
            
            ref_gen = (x for x in [tab])
            self.update_index(ref_gen)
        
        # Dirty method to keep track of modifications
        file_name = self.metadata['log'].keys()
        assert len(file_name) == 1
        file_name = list(file_name)[0]
        self.metadata['log'][file_name]['upload_es_train']['was_modified'] = True
        self._write_metadata()
Updates the merged table in Elasticsearch to take into account the
        new labels.
        
Updates the merged table in Elasticsearch to take into account the
new labels.
Updates the merged table in Elasticsearch to take into account the new labels .
def update_results(self, labels):
        new_rows = []
        columns = set()
        for label in labels:
            current_row = es.get(self.index_name, 'structure', label['source_id'])['_source']
            if label['is_match']:                
                if current_row['__ID_REF'] != label['ref_id']:
                    new_ref = es.get(self.ref.project_id, 'structure', label['ref_id'])['_source']
                    new_ref = {key + '__REF': val for key, val in new_ref.items()}
                    new_row = {key: val for key, val in current_row.items()}
                    new_row.update(new_ref)
                    new_row['__IS_MATCH'] = True
                    new_row['__CONFIDENCE'] = 999
                    new_row['__ID_REF'] = label['ref_id']
                else:
                    new_row = {key: val for key, val in current_row.items()}
                    new_row['__IS_MATCH'] = True
                    new_row['__CONFIDENCE'] = 999
            else:
                new_row = {col: val for col, val in current_row.items()}
                nan_cols = list(filter(lambda x: x[-5:]=='__REF', new_row.keys())) \
                            + ['__CONFIDENCE', '__ES_SCORE', '__ID_QUERY', \
                               '__ID_REF', '__IS_MATCH', '__THRESH']
                for col in nan_cols:
                    new_row[col] = np.nan
            columns.update(new_row.keys())
            new_rows.append((label['source_id'], new_row))
        if new_rows:
            dtype = {col: self._choose_dtype(col) for col in columns}
            tab = pd.DataFrame([x[1] for x in new_rows], index=[x[0] for x in new_rows])
            for k, v in dtype.items():
                if v == str:
                    tab[k].fillna('', inplace=True)
                tab[k] = tab[k].astype(v)
            ref_gen = (x for x in [tab])
            self.update_index(ref_gen)
        file_name = self.metadata['log'].keys()
        assert len(file_name) == 1
        file_name = list(file_name)[0]
        self.metadata['log'][file_name]['upload_es_train']['was_modified'] = True
        self._write_metadata()
def update_results ( self , labels ) : new_rows = [ ] columns = set ( ) for label in labels : current_row = es . get ( self . index_name , 'structure' , label [ 'source_id' ] ) [ '_source' ] if label [ 'is_match' ] : if current_row [ '__ID_REF' ] != label [ 'ref_id' ] : new_ref = es . get ( self . ref . project_id , 'structure' , label [ 'ref_id' ] ) [ '_source' ] new_ref = { key + '__REF' : val for key , val in new_ref . items ( ) } new_row = { key : val for key , val in current_row . items ( ) } new_row . update ( new_ref ) new_row [ '__IS_MATCH' ] = True new_row [ '__CONFIDENCE' ] = 999 new_row [ '__ID_REF' ] = label [ 'ref_id' ] else : new_row = { key : val for key , val in current_row . items ( ) } new_row [ '__IS_MATCH' ] = True new_row [ '__CONFIDENCE' ] = 999 else : new_row = { col : val for col , val in current_row . items ( ) } nan_cols = list ( filter ( lambda x : x [ - 5 : ] == '__REF' , new_row . keys ( ) ) ) + [ '__CONFIDENCE' , '__ES_SCORE' , '__ID_QUERY' , '__ID_REF' , '__IS_MATCH' , '__THRESH' ] for col in nan_cols : new_row [ col ] = np . nan columns . update ( new_row . keys ( ) ) new_rows . append ( ( label [ 'source_id' ] , new_row ) ) if new_rows : dtype = { col : self . _choose_dtype ( col ) for col in columns } tab = pd . DataFrame ( [ x [ 1 ] for x in new_rows ] , index = [ x [ 0 ] for x in new_rows ] ) for k , v in dtype . items ( ) : if v == str : tab [ k ] . fillna ( '' , inplace = True ) tab [ k ] = tab [ k ] . astype ( v ) ref_gen = ( x for x in [ tab ] ) self . update_index ( ref_gen ) file_name = self . metadata [ 'log' ] . keys ( ) assert len ( file_name ) == 1 file_name = list ( file_name ) [ 0 ] self . metadata [ 'log' ] [ file_name ] [ 'upload_es_train' ] [ 'was_modified' ] = True self . _write_metadata ( )
Updates the merged table in Elasticsearch to take into account the
new labels.
Updates the merged table in Elasticsearch to take into account the new labels .
'''Updates the merged table in Elasticsearch to take into account the
        new labels.
        ''' # TODO: source indices # TODO: what to do with __ES_SCORE, __ID_QUERY, __THRESH # Fix for dtype that is not working in DataFrame call # Dirty method to keep track of modifications
self
labels