# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 2
#     language: python
#     name: python2
# ---

# # Examples of datasets cleaning using the autoc package 

# ## Import Packages 

import pandas as pd 
import seaborn as sns 
from autoc.explorer import cserie,DataExploration
import matplotlib.pyplot as plt
import matplotlib
matplotlib.style.use('ggplot')
# %matplotlib inline
plt.rcParams['figure.figsize'] = (12.0, 8)

# ## Springleaf dataset analysis 

# you can find the kaggle competition [here](https://www.kaggle.com/c/springleaf-marketing-response)

# +

path_to_data = '/Users/ericfourrier/Documents/Data/SpringLeaf/train.csv'
# -

df = pd.read_csv(path_to_data)

exploration = DataExploration(df)

# ### Using different methods of the class 

# +
# Missing value count

# columns 
exploration.nacolcount()
exploration.narowcount()
# -

# detecting full na columns 
na_cols = cserie(exploration._nacolcount.Napercentage == 1)
df.loc[:,na_cols]

# detecting full na rows
na_rows = cserie(exploration._narowcount.Napercentage == 1)
na_rows # no full missing rows 

# ### Using Structure

# Looking at full structure of the dataset 
df_infos = exploration.structure()

na_cols2 = cserie(df_infos.na_columns) # looking at complete na columns 
na_cols2

constant_cols = cserie(df_infos.constant_columns) # looking at constant columns
constant_cols

# ### Small study on missing values 

# missing values per rows distribution
print(float(sum(exploration._narowcount.Napercentage > 0.0003)))/df.shape[0]
exploration._narowcount.Napercentage.plot(kind = 'hist',bins=200,xlim = (0,0.0003))

# missing values per cols distribution
df_infos.perc_missing[df_infos.perc_missing !=0].hist(bins = 100)

# many missing columns 
exploration.manymissing(a=0.7) # more than 70 % missing variables 

# missing values per cols distribution for higher missing percentage 
df_infos.perc_missing[df_infos.perc_missing > 0.01].hist(bins = 100)

df_infos

# ### Explore character variables 

# ### Correlation matrix 

reduced_data=exploration.data.loc[:,exploration.data.columns[range(0,100)]]
plt.figure(figsize=(30,15))
corr_matrix = reduced_data.corr()
from autoc.utils.coorplot import plot_corrmatrix
plot_corrmatrix(corr_matrix,size=0.4)



StarcoderdataJupyterScriptsDedupFiltered