feature-engineering-and-data-transformations/using-toy-exmaples/feature_selection/filter_method.py
def corr_feature_detect(data,threshold=0.8):
    """ detect highly-correlated features of a Dataframe
    Parameters
    ----------
    data : pd.Dataframe
    threshold : threshold to identify the variable correlated
        
    Returns
    -------
    pairs of correlated variables
    """
    
    corrmat = data.corr()
    corrmat = corrmat.abs().unstack() # absolute value of corr coef
    corrmat = corrmat.sort_values(ascending=False)
    corrmat = corrmat[corrmat >= threshold]
    corrmat = corrmat[corrmat < 1] # remove the digonal
    corrmat = pd.DataFrame(corrmat).reset_index()
    corrmat.columns = ['feature1', 'feature2', 'corr']
   
    grouped_feature_ls = []
    correlated_groups = []
    
    for feature in corrmat.feature1.unique():
        if feature not in grouped_feature_ls:
    
            # find all features correlated to a single feature
            correlated_block = corrmat[corrmat.feature1 == feature]
            grouped_feature_ls = grouped_feature_ls + list(
                correlated_block.feature2.unique()) + [feature]
    
            # append the block of features to the list
            correlated_groups.append(correlated_block)
    return correlated_groups
 detect highly-correlated features of a Dataframe
    Parameters
    ----------
    data : pd.Dataframe
    threshold : threshold to identify the variable correlated
        
    Returns
    -------
    pairs of correlated variables
    
detect highly-correlated features of a Dataframe
Parameters

data : pd.Dataframe
threshold : threshold to identify the variable correlated

Returns

pairs of correlated variables
detect highly - correlated features of a Dataframe Parameters data : pd . Dataframe threshold : threshold to identify the variable correlated Returns pairs of correlated variables
def corr_feature_detect(data,threshold=0.8):
    corrmat = data.corr()
    corrmat = corrmat.abs().unstack() 
    corrmat = corrmat.sort_values(ascending=False)
    corrmat = corrmat[corrmat >= threshold]
    corrmat = corrmat[corrmat < 1] 
    corrmat = pd.DataFrame(corrmat).reset_index()
    corrmat.columns = ['feature1', 'feature2', 'corr']
    grouped_feature_ls = []
    correlated_groups = []
    for feature in corrmat.feature1.unique():
        if feature not in grouped_feature_ls:
            correlated_block = corrmat[corrmat.feature1 == feature]
            grouped_feature_ls = grouped_feature_ls + list(
                correlated_block.feature2.unique()) + [feature]
            correlated_groups.append(correlated_block)
    return correlated_groups
def corr_feature_detect ( data , threshold = 0.8 ) : corrmat = data . corr ( ) corrmat = corrmat . abs ( ) . unstack ( ) corrmat = corrmat . sort_values ( ascending = False ) corrmat = corrmat [ corrmat >= threshold ] corrmat = corrmat [ corrmat < 1 ] corrmat = pd . DataFrame ( corrmat ) . reset_index ( ) corrmat . columns = [ 'feature1' , 'feature2' , 'corr' ] grouped_feature_ls = [ ] correlated_groups = [ ] for feature in corrmat . feature1 . unique ( ) : if feature not in grouped_feature_ls : correlated_block = corrmat [ corrmat . feature1 == feature ] grouped_feature_ls = grouped_feature_ls + list ( correlated_block . feature2 . unique ( ) ) + [ feature ] correlated_groups . append ( correlated_block ) return correlated_groups
detect highly-correlated features of a Dataframe
Parameters
detect highly - correlated features of a Dataframe Parameters
""" detect highly-correlated features of a Dataframe
    Parameters
    ----------
    data : pd.Dataframe
    threshold : threshold to identify the variable correlated
        
    Returns
    -------
    pairs of correlated variables
    """ # absolute value of corr coef # remove the digonal # find all features correlated to a single feature # append the block of features to the list
data
threshold