Tilburg Thesis_Malawi School Dropout/src/thesis_utils_general.py
def load_stata_file(filepath, 
                    index_cols,
                    drop_minornans=False, 
                    drop_unlabeled=False):
    """ Load data and metadata from Stata file"""
    data = pd.read_stata(filepath, convert_categoricals=False).set_index(index_cols)

    with StataReader(filepath) as reader:
        reader.value_labels()
            
        mapping = {col: reader.value_label_dict[t] for col, t in 
                   zip(reader.varlist, reader.lbllist)
                   if t in reader.value_label_dict}

        data.replace(mapping, inplace=True)
        
        # convert the categorical variables into
        # the category type
        for c in data.columns:
            if c in mapping:
                data[c] = data[c].astype('category')
                        
        # drop records with only a few nans
        if drop_minornans: 
            nan_counts = (data.applymap(pd.isnull)
                          .sum(axis=0)
                          .sort_values(ascending=False))
            nan_cols = nan_counts[(nan_counts > 0) & (nan_counts < 100)].index.values
            data = data.dropna(subset=nan_cols)
        # drop unlabeled categorical values
        def find_unlabeled(x):
            if x.name in mapping.keys():
                return [val if (val in mapping[x.name].values() or pd.isnull(val)) 
                        else 'UNLABELED' for val in x]
            else:
                return x
            
        data = data.apply(find_unlabeled)
        data = data[~data.applymap(lambda x: x == "UNLABELED").any(axis=1)]
        
    return data
 Load data and metadata from Stata file
Load data and metadata from Stata file
Load data and metadata from Stata file
def load_stata_file(filepath, 
                    index_cols,
                    drop_minornans=False, 
                    drop_unlabeled=False):
    data = pd.read_stata(filepath, convert_categoricals=False).set_index(index_cols)
    with StataReader(filepath) as reader:
        reader.value_labels()
        mapping = {col: reader.value_label_dict[t] for col, t in 
                   zip(reader.varlist, reader.lbllist)
                   if t in reader.value_label_dict}
        data.replace(mapping, inplace=True)
        for c in data.columns:
            if c in mapping:
                data[c] = data[c].astype('category')
        if drop_minornans: 
            nan_counts = (data.applymap(pd.isnull)
                          .sum(axis=0)
                          .sort_values(ascending=False))
            nan_cols = nan_counts[(nan_counts > 0) & (nan_counts < 100)].index.values
            data = data.dropna(subset=nan_cols)
        def find_unlabeled(x):
            if x.name in mapping.keys():
                return [val if (val in mapping[x.name].values() or pd.isnull(val)) 
                        else 'UNLABELED' for val in x]
            else:
                return x
        data = data.apply(find_unlabeled)
        data = data[~data.applymap(lambda x: x == "UNLABELED").any(axis=1)]
    return data
def load_stata_file ( filepath , index_cols , drop_minornans = False , drop_unlabeled = False ) : data = pd . read_stata ( filepath , convert_categoricals = False ) . set_index ( index_cols ) with StataReader ( filepath ) as reader : reader . value_labels ( ) mapping = { col : reader . value_label_dict [ t ] for col , t in zip ( reader . varlist , reader . lbllist ) if t in reader . value_label_dict } data . replace ( mapping , inplace = True ) for c in data . columns : if c in mapping : data [ c ] = data [ c ] . astype ( 'category' ) if drop_minornans : nan_counts = ( data . applymap ( pd . isnull ) . sum ( axis = 0 ) . sort_values ( ascending = False ) ) nan_cols = nan_counts [ ( nan_counts > 0 ) & ( nan_counts < 100 ) ] . index . values data = data . dropna ( subset = nan_cols ) def find_unlabeled ( x ) : if x . name in mapping . keys ( ) : return [ val if ( val in mapping [ x . name ] . values ( ) or pd . isnull ( val ) ) else 'UNLABELED' for val in x ] else : return x data = data . apply ( find_unlabeled ) data = data [ ~ data . applymap ( lambda x : x == "UNLABELED" ) . any ( axis = 1 ) ] return data
Load data and metadata from Stata file
Load data and metadata from Stata file
""" Load data and metadata from Stata file""" # convert the categorical variables into # the category type # drop records with only a few nans # drop unlabeled categorical values
filepath
index_cols
drop_minornans
drop_unlabeled