import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import missingno as msno
from pandas.api.types import CategoricalDtype
df = pd.read_csv('/kaggle/input/heart-stroke/train_strokes.csv')
df
df.info()
# observe missing values in 'bmi' and 'smoking_status'
"""
### dealing with missing values
"""
print(df.isna().sum())
print(df.duplicated().sum())
print(df['age'].min(), df['age'].max())
msno.matrix(df.sort_values(by=['age']))
# create a new column, age_group
# 0-4, 5-12, 13-18, 19-25, 26-40, 41-60, >60
def group_age(age):
    if age <= 4:
        return '0-4'
    if 5<=age<=12:
        return '5-12'
    if 13<=age<=18:
        return '13-18'
    if 19<=age<=25:
        return '19-25'
    if 26<=age<=40:
        return '26-40'
    if 41<=age<=60:
        return '41-60'
    else:
        return '>60'

df['group_age'] = df['age'].apply(group_age)
df
group_age_cat = ['0-4','5-12','13-18','19-25','26-40','41-60','>60']

age_cat_type = CategoricalDtype(categories=group_age_cat, ordered=True)
df['group_age'] = df['group_age'].astype(age_cat_type)
# trying to discover if there's any pattern in people with missing bmi
df_bmi_by_groupage = df.groupby('group_age')['bmi'].mean().apply(lambda x: round(x,2))
df_bmi_by_groupage
df.loc[(df['group_age']=='0-4'),'bmi'] = df.loc[(df['group_age']=='0-4'),'bmi'].fillna(df_bmi_by_groupage.loc['0-4'])
df.loc[(df['group_age']=='5-12'),'bmi'] = df.loc[(df['group_age']=='5-12'),'bmi'].fillna(df_bmi_by_groupage.loc['5-12'])
df.loc[(df['group_age']=='13-18'),'bmi'] = df.loc[(df['group_age']=='13-18'),'bmi'].fillna(df_bmi_by_groupage.loc['13-18'])
df.loc[(df['group_age']=='19-25'),'bmi'] = df.loc[(df['group_age']=='19-25'),'bmi'].fillna(df_bmi_by_groupage.loc['19-25'])
df.loc[(df['group_age']=='26-40'),'bmi'] = df.loc[(df['group_age']=='26-40'),'bmi'].fillna(df_bmi_by_groupage.loc['26-40'])
df.loc[(df['group_age']=='41-60'),'bmi'] = df.loc[(df['group_age']=='41-60'),'bmi'].fillna(df_bmi_by_groupage.loc['41-60'])
df.loc[(df['group_age']=='>60'),'bmi'] = df.loc[(df['group_age']=='>60'),'bmi'].fillna(df_bmi_by_groupage.loc['>60'])
# ensure there's no more missing value in the column 'bmi'
df['bmi'].isna().sum()
print(df['smoking_status'].unique())
print(df['smoking_status'].value_counts())
# 13292 NaN
print(df.groupby('group_age')['id'].count())
plt.bar(group_age_cat,df.groupby('group_age')['id'].count())
plt.xlabel('Age Group')
df[df['group_age']=='0-4']['smoking_status'].isna().sum()
df[df['group_age']=='5-12']['smoking_status'].isna().sum()
# for children below age of 12, we will assume that they never smoked:
df.loc[(df['group_age']=='0-4'),'smoking_status'] = df.loc[(df['group_age']=='0-4'),'smoking_status'].fillna('never smoked')
df.loc[(df['group_age']=='5-12'),'smoking_status'] = df.loc[(df['group_age']=='5-12'),'smoking_status'].fillna('never smoked')
# however for the rest of the missing value, we are unable to make this assumption
# instead we will accept the flaw and changed it as 'no record'
df.loc[:,['smoking_status']] = df.loc[:,['smoking_status']].fillna('no record')
df
"""
### changing column datatypes
"""
df.dtypes
# there are quite a number of columns with object type
# look into these columns
df_obj = df.select_dtypes(include=['object'])
df_obj.describe()
df_obj = df_obj.apply(lambda x: x.str.lower())
df_obj
# converted object datatype to category to reduce memory space
df_obj = df_obj.astype('category')
print(df_obj.dtypes)
df_obj['work_type'].value_counts()
df_obj['smoking_status'].value_counts()
df_obj['gender'].value_counts()
df_obj['Residence_type'].value_counts()
# looking into columns that is of type int64
df_num = df.select_dtypes(include=['int64'])
df_num.sample(5)
print(df_num['hypertension'].value_counts())
print(df_num['heart_disease'].value_counts())
print(df_num['stroke'].value_counts())
df_num = df_num[['hypertension','heart_disease','stroke']].astype('category')
print(df_num.dtypes)
#compare both dataframes
print(df_obj.shape)
print(df_num.shape)
# merging into new dataframe
df.columns
df_new = pd.concat([df[['age','avg_glucose_level','bmi']], df_obj, df_num], axis=1)
df_new
df_new.info()
# reducing the memory usage by changing data type