"""
**In this notebook I have done data preprocessing and EDA that is aimed at feature selection and data analysis. This notebook is work in progress as I will be adding various model prediction outcomes as well.**
"""
# This Python 3 environment comes with many helpful analytics libraries installed
# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python
# For example, here's several helpful packages to load

import numpy as np # linear algebra
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)

# Input data files are available in the read-only "../input/" directory
# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory

import os
for dirname, _, filenames in os.walk('/kaggle/input'):
    for filename in filenames:
        print(os.path.join(dirname, filename))

# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using "Save & Run All" 
# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session
#import necessary libraries
from pandas_profiling import ProfileReport
import seaborn as sns
import matplotlib.pyplot as plt
train=pd.read_csv("/kaggle/input/widsdatathon2021/TrainingWiDS2021.csv")
train.head()
pd.set_option('max_rows', None)
train.isna().sum()
train.shape
traincopy=train.copy()
traincopy.head()
#eliminate features with more than 80% missing values
train=train[train.columns[train.isnull().mean() < 0.8]]
train.shape
train.drop(["Unnamed: 0"],axis=1,inplace=True)
train.head()
#missingno is library to visulaize missing data
import missingno as msno
msno.bar(train)
#PandasProfiling is a very handy tool for EDA
trainprofile = ProfileReport(train,'EDA')
trainprofile
import plotly.express as px
fig = px.histogram(train[['age','gender','diabetes_mellitus','bmi']].dropna(), x="age", y="diabetes_mellitus", color="gender",
                   marginal="box", hover_data=train[['age','gender','diabetes_mellitus','bmi']].columns)
fig.show()
#one hot encoding is a representation of categorical variables as binary vectors
encoded_training = pd.get_dummies(train)
missing_values=encoded_training.copy()
#imputing missing values using SimpleImputer
from sklearn.impute import SimpleImputer
imputer = SimpleImputer(missing_values=np.nan, strategy='mean')
imputer = imputer.fit(missing_values)
trainingfinal = pd.DataFrame(imputer.transform(missing_values))
trainingfinal.columns=missing_values.columns
trainingfinal.head()
trainingfinal.columns.get_loc("diabetes_mellitus")
X=trainingfinal.drop(["diabetes_mellitus"],axis=1)
y = trainingfinal.iloc[:,141]
#selecting top 10 features for analysis
from sklearn.ensemble import ExtraTreesClassifier
model = ExtraTreesClassifier()
model.fit(X,y)
print(model.feature_importances_) 
feat_importances = pd.Series(model.feature_importances_, index=X.columns)
feat_importances.nlargest(10).plot(kind='barh')
plt.show()
traintop=trainingfinal[["d1_bun_min","d1_bun_max","age","bmi","weight","d1_glucose_min","h1_glucose_min","h1_glucose_max","glucose_apache","d1_glucose_max","diabetes_mellitus"]].copy()
traintop.head()
sns.pairplot(traintop)
#Randomly select 50% data for further analysis
trainsample=traintop.sample(frac=0.5, random_state=1)
trainsample.shape
cor=trainsample.corr()
sns.set(rc={'figure.figsize':(15,15)})
sns.heatmap(cor,annot=True)