merge_machine/missing_values.py
def mv_from_common_values(all_top_values, score=0.5):
    '''Looks for values common in at least two columns'''
    # Create dict with: {value: set_of_columns_where_common} with values present in at least two columns
    popular_values = dict()
    for col_1, top_values_1 in all_top_values.items():
        for col_2, top_values_2 in all_top_values.items():
            if col_1 != col_2:
                common_values = [x for x in top_values_1.index if x in top_values_2.index]
                for val in common_values:
                    if val not in popular_values:
                        popular_values[val] = set([col_1, col_2])
                    else:
                        popular_values[val].add(col_1)
                        popular_values[val].add(col_2)
    
    if popular_values:
        # Questionable heuristic: return value most frequent
        temp = [(val, len(cols)) for val, cols in popular_values.items()]
        temp.sort(key=lambda x: x[1], reverse=True)
        mv_value = temp[0][0]
        return [(mv_value, score, 'common_values')]
    return  []
Looks for values common in at least two columns
Looks for values common in at least two columns
Looks for values common in at least two columns
def mv_from_common_values(all_top_values, score=0.5):
    popular_values = dict()
    for col_1, top_values_1 in all_top_values.items():
        for col_2, top_values_2 in all_top_values.items():
            if col_1 != col_2:
                common_values = [x for x in top_values_1.index if x in top_values_2.index]
                for val in common_values:
                    if val not in popular_values:
                        popular_values[val] = set([col_1, col_2])
                    else:
                        popular_values[val].add(col_1)
                        popular_values[val].add(col_2)
    if popular_values:
        temp = [(val, len(cols)) for val, cols in popular_values.items()]
        temp.sort(key=lambda x: x[1], reverse=True)
        mv_value = temp[0][0]
        return [(mv_value, score, 'common_values')]
    return  []
def mv_from_common_values ( all_top_values , score = 0.5 ) : popular_values = dict ( ) for col_1 , top_values_1 in all_top_values . items ( ) : for col_2 , top_values_2 in all_top_values . items ( ) : if col_1 != col_2 : common_values = [ x for x in top_values_1 . index if x in top_values_2 . index ] for val in common_values : if val not in popular_values : popular_values [ val ] = set ( [ col_1 , col_2 ] ) else : popular_values [ val ] . add ( col_1 ) popular_values [ val ] . add ( col_2 ) if popular_values : temp = [ ( val , len ( cols ) ) for val , cols in popular_values . items ( ) ] temp . sort ( key = lambda x : x [ 1 ] , reverse = True ) mv_value = temp [ 0 ] [ 0 ] return [ ( mv_value , score , 'common_values' ) ] return [ ]
Looks for values common in at least two columns
Looks for values common in at least two columns
'''Looks for values common in at least two columns''' # Create dict with: {value: set_of_columns_where_common} with values present in at least two columns # Questionable heuristic: return value most frequent
all_top_values
score