build/lib/AutoMxL/Explore/Explore.py
def explore(df, verbose=False):
    """Identify variables types and gives global information about the dataset

    - Variables type :
        - date
        - identifier
        - verbatim
        - boolean
        - categorical
        - numerical
    - variables containing NA values
    - low variance and unique values variables

    See get_features_type function doc for type identification heuristics

    Parameters
    ----------
    df : DataFrame
        input dataset
    verbose : boolean (Default False)
        Get logging information

    Returns
    -------
    dict
        {x : variables names list }

        - date : date features
        - identifier : identifier features
        - verbatim : verbatim features
        - boolean : boolean features
        - categorical : categorical features
        - numerical : numerical features
        - categorical : categorical features
        - date : date features
        - NA : features which contains NA values
        - low_variance : list of the features with low variance
    """
    # dataset dimensions
    if verbose:
        color_print("Dimensions :")
        print("  > row number :", df.shape[0], "\n  > col number :", df.shape[1])

    #########################
    # Low variance features
    #########################
    if verbose:
        color_print('Low variance features')

    l_low_var = \
        low_variance_features(df, var_list=df._get_numeric_data().columns.tolist(), threshold=0, rescale=True,
                              verbose=verbose).index.tolist()

    # categorical features with unique values
    l_unique = [col for col in df.columns.tolist() if df[col].dtype == 'object' and df[col].nunique(dropna=True) == 1]

    l_low_var = l_low_var + l_unique

    df_valid = df.drop(l_low_var, axis=1).copy()

    #################
    # features type #
    #################
    d_features = get_features_type(df_valid, l_var=None, th=0.95)

    if verbose:
        color_print("Features type identification : ")
        list(map(lambda typ:
                 print("  > " + typ + " : " + str(len(d_features[typ])) + ' (' + str(
                     round(len(d_features[typ]) / df_valid.shape[1] * 100)) + '%)'),
                 d_features.keys()))

    ######################
    # NA values analysis
    ######################
    df_col = pd.DataFrame(df_valid.columns.values, columns=['variables'])
    df_col['Nbr NA'] = df_valid.isna().sum().tolist()
    df_col['Taux NA'] = df_col['Nbr NA'] / df_valid.shape[0]
    # features containing NA values
    NA_columns = df_col.loc[df_col['Nbr NA'] > 0].sort_values('Nbr NA', ascending=False).variables.tolist()
    col_des = df_col['Taux NA'].describe()

    if verbose:
        color_print(str(len(NA_columns)) + " features containing NA")
        print('  > Taux NA moyen : ' + str(round(col_des['mean'] * 100, 2)) + '%',
              '\n  >           min : ' + str(round(col_des['min'] * 100, 2)) + '%',
              '\n  >           max : ' + str(round(col_des['max'] * 100, 2)) + '%')

    # store into DataFrame
    d_features['NA'] = NA_columns
    d_features['low_variance'] = l_low_var

    return d_features
Identify variables types and gives global information about the dataset

    - Variables type :
        - date
        - identifier
        - verbatim
        - boolean
        - categorical
        - numerical
    - variables containing NA values
    - low variance and unique values variables

    See get_features_type function doc for type identification heuristics

    Parameters
    ----------
    df : DataFrame
        input dataset
    verbose : boolean (Default False)
        Get logging information

    Returns
    -------
    dict
        {x : variables names list }

        - date : date features
        - identifier : identifier features
        - verbatim : verbatim features
        - boolean : boolean features
        - categorical : categorical features
        - numerical : numerical features
        - categorical : categorical features
        - date : date features
        - NA : features which contains NA values
        - low_variance : list of the features with low variance
    
Identify variables types and gives global information about the dataset
Variables type :
date
identifier
verbatim
boolean
categorical
numerical
variables containing NA values
low variance and unique values variables

See get_features_type function doc for type identification heuristics

Parameters

df : DataFrame
input dataset
verbose : boolean (Default False)
Get logging information

Returns

dict
{x : variables names list }


Identify variables types and gives global information about the dataset Variables type : date identifier verbatim boolean categorical numerical variables containing NA values low variance and unique values variables See get_features_type function doc for type identification heuristics Parameters df : DataFrame input dataset verbose : boolean ( Default False ) Get logging information Returns dict { x : variables names list }
def explore(df, verbose=False):
    if verbose:
        color_print("Dimensions :")
        print("  > row number :", df.shape[0], "\n  > col number :", df.shape[1])
    if verbose:
        color_print('Low variance features')
    l_low_var = \
        low_variance_features(df, var_list=df._get_numeric_data().columns.tolist(), threshold=0, rescale=True,
                              verbose=verbose).index.tolist()
    l_unique = [col for col in df.columns.tolist() if df[col].dtype == 'object' and df[col].nunique(dropna=True) == 1]
    l_low_var = l_low_var + l_unique
    df_valid = df.drop(l_low_var, axis=1).copy()
    d_features = get_features_type(df_valid, l_var=None, th=0.95)
    if verbose:
        color_print("Features type identification : ")
        list(map(lambda typ:
                 print("  > " + typ + " : " + str(len(d_features[typ])) + ' (' + str(
                     round(len(d_features[typ]) / df_valid.shape[1] * 100)) + '%)'),
                 d_features.keys()))
    #####
    #####
    df_col = pd.DataFrame(df_valid.columns.values, columns=['variables'])
    df_col['Nbr NA'] = df_valid.isna().sum().tolist()
    df_col['Taux NA'] = df_col['Nbr NA'] / df_valid.shape[0]
    NA_columns = df_col.loc[df_col['Nbr NA'] > 0].sort_values('Nbr NA', ascending=False).variables.tolist()
    col_des = df_col['Taux NA'].describe()
    if verbose:
        color_print(str(len(NA_columns)) + " features containing NA")
        print('  > Taux NA moyen : ' + str(round(col_des['mean'] * 100, 2)) + '%',
              '\n  >           min : ' + str(round(col_des['min'] * 100, 2)) + '%',
              '\n  >           max : ' + str(round(col_des['max'] * 100, 2)) + '%')
    d_features['NA'] = NA_columns
    d_features['low_variance'] = l_low_var
    return d_features
def explore ( df , verbose = False ) : if verbose : color_print ( "Dimensions :" ) print ( "  > row number :" , df . shape [ 0 ] , "\n  > col number :" , df . shape [ 1 ] ) if verbose : color_print ( 'Low variance features' ) l_low_var = low_variance_features ( df , var_list = df . _get_numeric_data ( ) . columns . tolist ( ) , threshold = 0 , rescale = True , verbose = verbose ) . index . tolist ( ) l_unique = [ col for col in df . columns . tolist ( ) if df [ col ] . dtype == 'object' and df [ col ] . nunique ( dropna = True ) == 1 ] l_low_var = l_low_var + l_unique df_valid = df . drop ( l_low_var , axis = 1 ) . copy ( ) d_features = get_features_type ( df_valid , l_var = None , th = 0.95 ) if verbose : color_print ( "Features type identification : " ) list ( map ( lambda typ : print ( "  > " + typ + " : " + str ( len ( d_features [ typ ] ) ) + ' (' + str ( round ( len ( d_features [ typ ] ) / df_valid . shape [ 1 ] * 100 ) ) + '%)' ) , d_features . keys ( ) ) ) df_col = pd . DataFrame ( df_valid . columns . values , columns = [ 'variables' ] ) df_col [ 'Nbr NA' ] = df_valid . isna ( ) . sum ( ) . tolist ( ) df_col [ 'Taux NA' ] = df_col [ 'Nbr NA' ] / df_valid . shape [ 0 ] NA_columns = df_col . loc [ df_col [ 'Nbr NA' ] > 0 ] . sort_values ( 'Nbr NA' , ascending = False ) . variables . tolist ( ) col_des = df_col [ 'Taux NA' ] . describe ( ) if verbose : color_print ( str ( len ( NA_columns ) ) + " features containing NA" ) print ( '  > Taux NA moyen : ' + str ( round ( col_des [ 'mean' ] * 100 , 2 ) ) + '%' , '\n  >           min : ' + str ( round ( col_des [ 'min' ] * 100 , 2 ) ) + '%' , '\n  >           max : ' + str ( round ( col_des [ 'max' ] * 100 , 2 ) ) + '%' ) d_features [ 'NA' ] = NA_columns d_features [ 'low_variance' ] = l_low_var return d_features
Identify variables types and gives global information about the dataset
 Variables type :
 date
 identifier
 verbatim
 boolean
 categorical
 numerical
 variables containing NA values
 low variance and unique values variables
Identify variables types and gives global information about the dataset Variables type : date identifier verbatim boolean categorical numerical variables containing NA values low variance and unique values variables
"""Identify variables types and gives global information about the dataset

    - Variables type :
        - date
        - identifier
        - verbatim
        - boolean
        - categorical
        - numerical
    - variables containing NA values
    - low variance and unique values variables

    See get_features_type function doc for type identification heuristics

    Parameters
    ----------
    df : DataFrame
        input dataset
    verbose : boolean (Default False)
        Get logging information

    Returns
    -------
    dict
        {x : variables names list }

        - date : date features
        - identifier : identifier features
        - verbatim : verbatim features
        - boolean : boolean features
        - categorical : categorical features
        - numerical : numerical features
        - categorical : categorical features
        - date : date features
        - NA : features which contains NA values
        - low_variance : list of the features with low variance
    """ # dataset dimensions ######################### # Low variance features ######################### # categorical features with unique values ################# # features type # ################# ###################### # NA values analysis ###################### # features containing NA values # store into DataFrame
df
verbose