merge_machine/missing_values.py
def replace_mvs(tab, params):
    """
    API MODULE
    
    Replace the values that should be mvs by actual np.nan. Values in 'all' 
    will be replaced in the entire table whereas values in 'columns' will only
    be replaced in the specified columns.
    
    INPUT:
        tab: pandas DataFrame to modify
        params:
            mvs_dict: dict indicating mv values with scores. For example:
                    {
                        'all': [],
                        'columns': {'dech': [('-', 2.0, 'unknown')],
                                    'distance': [('-', 1, 'unknown')]}
                    }
            thresh: minimum score to remove mvs
    
    OUTPUT:
        tab: same table with values replaced by np.nan
        modified: Indicate if value was modified    
    """
    
    # Set variables and replace by default values
    mvs_dict = params['mvs_dict']
    thresh = params.get('thresh', DEFAULT_THRESH)
    
    # Replace
    assert sorted(list(mvs_dict.keys())) == ['all', 'columns']

    # Run information
    modified = pd.DataFrame(False, index=tab.index, columns=tab.columns)

    for mv in mvs_dict['all']:
        val, score = mv['val'], mv['score']
        #        run_info['replace_num']['all'][val] = 0
        if score >= thresh:
            # Metrics
            modified = modified | (tab == val)
            
            # Do transformation
            tab.replace(val, np.nan, inplace=True)
    
    for col, mv_values in mvs_dict['columns'].items():
        for mv in mv_values:
            val, score = mv['val'], mv['score']
            if score >= thresh:
                # Metrics
                if tab[col].notnull().any():
                    modified[col] = modified[col] | (tab[col] == val)
                # Do transformation
                tab[col].replace(val, np.nan, inplace=True)
    
    return tab, modified

    API MODULE
    
    Replace the values that should be mvs by actual np.nan. Values in 'all' 
    will be replaced in the entire table whereas values in 'columns' will only
    be replaced in the specified columns.
    
    INPUT:
        tab: pandas DataFrame to modify
        params:
            mvs_dict: dict indicating mv values with scores. For example:
                    {
                        'all': [],
                        'columns': {'dech': [('-', 2.0, 'unknown')],
                                    'distance': [('-', 1, 'unknown')]}
                    }
            thresh: minimum score to remove mvs
    
    OUTPUT:
        tab: same table with values replaced by np.nan
        modified: Indicate if value was modified    
    
API MODULE
Replace the values that should be mvs by actual np.nan. Values in 'all'
will be replaced in the entire table whereas values in 'columns' will only
be replaced in the specified columns.

pandas DataFrame to modify
params:
mvs_dict: dict indicating mv values with scores.

same table with values replaced by np.nan
modified: Indicate if value was modified
API MODULE Replace the values that should be mvs by actual np . nan . Values in ' all ' will be replaced in the entire table whereas values in ' columns ' will only be replaced in the specified columns . pandas DataFrame to modify params : mvs_dict : dict indicating mv values with scores . same table with values replaced by np . nan modified : Indicate if value was modified
def replace_mvs(tab, params):
    mvs_dict = params['mvs_dict']
    thresh = params.get('thresh', DEFAULT_THRESH)
    assert sorted(list(mvs_dict.keys())) == ['all', 'columns']
    modified = pd.DataFrame(False, index=tab.index, columns=tab.columns)
    for mv in mvs_dict['all']:
        val, score = mv['val'], mv['score']
        if score >= thresh:
            modified = modified | (tab == val)
            tab.replace(val, np.nan, inplace=True)
    for col, mv_values in mvs_dict['columns'].items():
        for mv in mv_values:
            val, score = mv['val'], mv['score']
            if score >= thresh:
                if tab[col].notnull().any():
                    modified[col] = modified[col] | (tab[col] == val)
                tab[col].replace(val, np.nan, inplace=True)
    return tab, modified
def replace_mvs ( tab , params ) : mvs_dict = params [ 'mvs_dict' ] thresh = params . get ( 'thresh' , DEFAULT_THRESH ) assert sorted ( list ( mvs_dict . keys ( ) ) ) == [ 'all' , 'columns' ] modified = pd . DataFrame ( False , index = tab . index , columns = tab . columns ) for mv in mvs_dict [ 'all' ] : val , score = mv [ 'val' ] , mv [ 'score' ] if score >= thresh : modified = modified | ( tab == val ) tab . replace ( val , np . nan , inplace = True ) for col , mv_values in mvs_dict [ 'columns' ] . items ( ) : for mv in mv_values : val , score = mv [ 'val' ] , mv [ 'score' ] if score >= thresh : if tab [ col ] . notnull ( ) . any ( ) : modified [ col ] = modified [ col ] | ( tab [ col ] == val ) tab [ col ] . replace ( val , np . nan , inplace = True ) return tab , modified
API MODULE
Replace the values that should be mvs by actual np.nan.
API MODULE Replace the values that should be mvs by actual np . nan .
"""
    API MODULE
    
    Replace the values that should be mvs by actual np.nan. Values in 'all' 
    will be replaced in the entire table whereas values in 'columns' will only
    be replaced in the specified columns.
    
    INPUT:
        tab: pandas DataFrame to modify
        params:
            mvs_dict: dict indicating mv values with scores. For example:
                    {
                        'all': [],
                        'columns': {'dech': [('-', 2.0, 'unknown')],
                                    'distance': [('-', 1, 'unknown')]}
                    }
            thresh: minimum score to remove mvs
    
    OUTPUT:
        tab: same table with values replaced by np.nan
        modified: Indicate if value was modified    
    """ # Set variables and replace by default values # Replace # Run information #        run_info['replace_num']['all'][val] = 0 # Metrics # Do transformation # Metrics # Do transformation
tab
params