merge_machine/missing_values.py
def infer_mvs(tab, params=None):
    """
    API MODULE
    
    Run mv inference processes for each column and for the entire table
    """
    PROBABLE_MVS = ['nan', 'none', 'na', 'n/a', '\\n', ' ', 'non renseigne', \
                    'nr', 'no value', 'null', 'missing value']
    ALWAYS_MVS = ['']
    
    if params is None:
        params = {}
    
    # Set variables and replace by default values
    PROBABLE_MVS.extend(params.get('probable_mvs', []))
    ALWAYS_MVS.extend(params.get('always_mvs', []))
    num_top_values = params.get('num_top_values', 10)
    
    # Compute most frequent values per column
    all_top_values = compute_all_top_values(tab, num_top_values)
    
    col_mvs = dict()
    # Look at each column and infer mv
    for col, top_values in all_top_values.items():
        col_mvs[col] = []
        if (not top_values.any()) or (top_values.iloc[0] == 1):
            continue
        col_mvs[col].extend(mv_from_len_diff(top_values))
        col_mvs[col].extend(mv_from_len_ratio(top_values))
        col_mvs[col].extend(mv_from_not_digit(top_values))
        col_mvs[col].extend(mv_from_punctuation(top_values))
        col_mvs[col].extend(mv_from_usual_forms(top_values, PROBABLE_MVS))
        col_mvs[col].extend(mv_from_usual_forms(top_values, ALWAYS_MVS, 10**3))
        col_mvs[col].extend(mv_from_letter_repetition(top_values))
        col_mvs[col] = correct_score(col_mvs[col], PROBABLE_MVS)
        col_mvs[col].sort(key=lambda x: x[1], reverse=True)


    # Transfer output to satisfy API standards
    def triplet_to_dict(val):
        return {'val': val[0], 'score': val[1], 'origin': val[2]}

    common_mvs = [triplet_to_dict(val) for val in mv_from_common_values_2(col_mvs)]
    columns_mvs = {key:[triplet_to_dict(val) for val in vals] for key, vals in col_mvs.items()}
    infered_mvs = {'columns': columns_mvs, 'all': common_mvs}
    return {'mvs_dict': infered_mvs, 'thresh': 0.6}

    API MODULE
    
    Run mv inference processes for each column and for the entire table
    
API MODULE
Run mv inference processes for each column and for the entire table
API MODULE Run mv inference processes for each column and for the entire table
def infer_mvs(tab, params=None):
    PROBABLE_MVS = ['nan', 'none', 'na', 'n/a', '\\n', ' ', 'non renseigne', \
                    'nr', 'no value', 'null', 'missing value']
    ALWAYS_MVS = ['']
    if params is None:
        params = {}
    PROBABLE_MVS.extend(params.get('probable_mvs', []))
    ALWAYS_MVS.extend(params.get('always_mvs', []))
    num_top_values = params.get('num_top_values', 10)
    all_top_values = compute_all_top_values(tab, num_top_values)
    col_mvs = dict()
    for col, top_values in all_top_values.items():
        col_mvs[col] = []
        if (not top_values.any()) or (top_values.iloc[0] == 1):
            continue
        col_mvs[col].extend(mv_from_len_diff(top_values))
        col_mvs[col].extend(mv_from_len_ratio(top_values))
        col_mvs[col].extend(mv_from_not_digit(top_values))
        col_mvs[col].extend(mv_from_punctuation(top_values))
        col_mvs[col].extend(mv_from_usual_forms(top_values, PROBABLE_MVS))
        col_mvs[col].extend(mv_from_usual_forms(top_values, ALWAYS_MVS, 10**3))
        col_mvs[col].extend(mv_from_letter_repetition(top_values))
        col_mvs[col] = correct_score(col_mvs[col], PROBABLE_MVS)
        col_mvs[col].sort(key=lambda x: x[1], reverse=True)
    def triplet_to_dict(val):
        return {'val': val[0], 'score': val[1], 'origin': val[2]}
    common_mvs = [triplet_to_dict(val) for val in mv_from_common_values_2(col_mvs)]
    columns_mvs = {key:[triplet_to_dict(val) for val in vals] for key, vals in col_mvs.items()}
    infered_mvs = {'columns': columns_mvs, 'all': common_mvs}
    return {'mvs_dict': infered_mvs, 'thresh': 0.6}
def infer_mvs ( tab , params = None ) : PROBABLE_MVS = [ 'nan' , 'none' , 'na' , 'n/a' , '\\n' , ' ' , 'non renseigne' , 'nr' , 'no value' , 'null' , 'missing value' ] ALWAYS_MVS = [ '' ] if params is None : params = { } PROBABLE_MVS . extend ( params . get ( 'probable_mvs' , [ ] ) ) ALWAYS_MVS . extend ( params . get ( 'always_mvs' , [ ] ) ) num_top_values = params . get ( 'num_top_values' , 10 ) all_top_values = compute_all_top_values ( tab , num_top_values ) col_mvs = dict ( ) for col , top_values in all_top_values . items ( ) : col_mvs [ col ] = [ ] if ( not top_values . any ( ) ) or ( top_values . iloc [ 0 ] == 1 ) : continue col_mvs [ col ] . extend ( mv_from_len_diff ( top_values ) ) col_mvs [ col ] . extend ( mv_from_len_ratio ( top_values ) ) col_mvs [ col ] . extend ( mv_from_not_digit ( top_values ) ) col_mvs [ col ] . extend ( mv_from_punctuation ( top_values ) ) col_mvs [ col ] . extend ( mv_from_usual_forms ( top_values , PROBABLE_MVS ) ) col_mvs [ col ] . extend ( mv_from_usual_forms ( top_values , ALWAYS_MVS , 10 ** 3 ) ) col_mvs [ col ] . extend ( mv_from_letter_repetition ( top_values ) ) col_mvs [ col ] = correct_score ( col_mvs [ col ] , PROBABLE_MVS ) col_mvs [ col ] . sort ( key = lambda x : x [ 1 ] , reverse = True ) def triplet_to_dict ( val ) : return { 'val' : val [ 0 ] , 'score' : val [ 1 ] , 'origin' : val [ 2 ] } common_mvs = [ triplet_to_dict ( val ) for val in mv_from_common_values_2 ( col_mvs ) ] columns_mvs = { key : [ triplet_to_dict ( val ) for val in vals ] for key , vals in col_mvs . items ( ) } infered_mvs = { 'columns' : columns_mvs , 'all' : common_mvs } return { 'mvs_dict' : infered_mvs , 'thresh' : 0.6 }
API MODULE
Run mv inference processes for each column and for the entire table
API MODULE Run mv inference processes for each column and for the entire table
"""
    API MODULE
    
    Run mv inference processes for each column and for the entire table
    """ # Set variables and replace by default values # Compute most frequent values per column # Look at each column and infer mv # Transfer output to satisfy API standards
tab
params