# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# + [markdown] id="47_7sBsU8JYt"
# ## UmojaHack Africa 2021 #3: Financial Resilience Challenge (BEGINNER) by UmojaHack Africa
#
# Can you predict if an individual will be able to make a payment in an emergency situation?
#
# The objective of this challenge is to build a machine learning model to predict which individuals across Africa and around the world are most likely to be financially resilient.
#
#
# ![Umoja Hack](https://zindpublic.blob.core.windows.net/public/uploads/competition/image/151/thumb_cdf71374-9857-44bb-8216-5f8c67afb51f.png)
#
# This is a simple Python starter notebook to get you started with the Financial Resilience Challenge.
#
# This notebook covers:
# - Loading the data
# - Simple EDA and an example of feature enginnering
# - Data preprocessing and data wrangling
# - Creating a simple model
# - Making a submission
# - Some tips for improving your score

# + id="GsxsCJyNFxBZ"
# Import libraries
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.metrics import auc, classification_report, roc_auc_score
from sklearn.model_selection import train_test_split
from lightgbm import LGBMClassifier
from sklearn.ensemble import RandomForestClassifier
pd.set_option('max_colwidth', 500)
# %matplotlib inline
import warnings
warnings.filterwarnings('ignore')

# + id="0_7BVS92U0Ti"
# Load files
train = pd.read_csv('Train.csv')
test = pd.read_csv('Test.csv')
samplesubmission = pd.read_csv('SampleSubmission.csv')
variable_definations = pd.read_csv('VariableDefinitions.csv')

# + colab={"base_uri": "https://localhost:8080/", "height": 635} id="f_LhTb6nU0Se" outputId="b9882c53-9626-41b1-da7e-0299c7d2a3f6"
# Preview the first five rows of the train set
train.head()

# + colab={"base_uri": "https://localhost:8080/", "height": 602} id="c0KotvwrU0PP" outputId="9eef902c-3d8e-47a9-a340-6bc9c7aae1a0"
# Preview the first five rows of the test set
test.head()

# + colab={"base_uri": "https://localhost:8080/", "height": 195} id="wU1bcDZSU0Nj" outputId="370f83aa-3fe6-475f-d2ed-d966a3431647"
# Preview the first five rows of the sample submission file
samplesubmission.head()

# + colab={"base_uri": "https://localhost:8080/", "height": 1000} id="UFh6oAKDFyws" outputId="b7801036-dfb5-4172-b045-b1155e723706"
# Preview variable definations
variable_definations

# + colab={"base_uri": "https://localhost:8080/"} id="PdVvmpG_GRxA" outputId="2efee873-8ff7-44de-8570-a7575c8cf0f4"
# Check the shape of the train and test sets
print(f'The shape of the train set is: {train.shape}\nThe shape of the test set is: {test.shape}')
# -

train.isnull().sum()

# + [markdown] id="9CO1R3hi8FJQ"
# ### Check for missing values

# + colab={"base_uri": "https://localhost:8080/", "height": 609} id="iWgHvxqYGRux" outputId="979fb804-723c-466c-81ff-7216d3e760cd"
# Check if there any missing values in train set
ax = train.isna().sum().sort_values().plot(kind = 'barh', figsize = (9, 10))
plt.title('Percentage of Missing Values Per Column in Train Set', fontdict={'size':15})
for p in ax.patches:
    percentage ='{:,.0f}%'.format((p.get_width()/train.shape[0])*100)
    width, height =p.get_width(),p.get_height()
    x=p.get_x()+width+0.02
    y=p.get_y()+height/2
    ax.annotate(percentage,(x,y))

# + colab={"base_uri": "https://localhost:8080/", "height": 609} id="6A9jdEtoGRsi" outputId="ca04afd2-6372-4b8a-b8af-8ff177185145"
# Check if there missing values in test set
ax = test.isna().sum().sort_values().plot(kind = 'barh', figsize = (9, 10))
plt.title('Percentage of Missing Values Per Column in Test Set', fontdict={'size':15})

for p in ax.patches:
    percentage ='{:,.1f}%'.format((p.get_width()/test.shape[0])*100)
    width, height =p.get_width(),p.get_height()
    x=p.get_x()+width+0.02
    y=p.get_y()+height/2
    ax.annotate(percentage,(x,y))

# + colab={"base_uri": "https://localhost:8080/"} id="PnGVLoE2GRqL" outputId="6f96e2e1-54f7-4d22-ee10-8ad69ec8bb8f"
# Check for duplicates
train.duplicated().any(), test.duplicated().any()
# -

# since the total count of 'Q7' is so small 
# so filling data will decrease the reliablity
train = train.drop('Q7', axis=1)
test = test.drop('Q7', axis=1)

# Checking each Q column's count of 1.0, 2.0, 3.0, 4.0  
for j in range(1,29):
    if j == 7:
        print(f'this is {j}') 
        continue
    o, t, th, f = 0, 0, 0, 0
    try:
        for i in train[f'Q{j}']:
            if i == 1.0: o+=1
            elif i == 2.0: t+=1
            elif i == 3.0: th+=1
            elif i == 4.0: f+=1
        print(f"this is Q{j}",len(train[f'Q{j}']))
        print(o, t, th, f)
    except:
        try:
            for i in train[f'Q{j}a']:
                if i == 1.0: o+=1
                elif i == 2.0: t+=1
                elif i == 3.0: th+=1
                elif i == 4.0: f+=1
            print(f"this is Q{j}a",len(train[f'Q{j}a']))
            print(o, t, th, f)
        except:
            try:
                for i in train[f'Q{j}a']:
                    if i == 1.0: o+=1
                    elif i == 2.0: t+=1
                    elif i == 3.0: th+=1
                    elif i == 4.0: f+=1
                print(f"this is Q{j}a",len(train[f'Q{j}a']))
                print(o, t, th, f)
            except:
                pass

# since the next large null column is Q23
# fill the data using maximum count
train['Q23'] = train['Q23'].fillna(2.0)
test['Q23'] = test['Q23'].fillna(2.0)

# Check for duplicates
train.duplicated().any(), test.duplicated().any()

# + [markdown] id="SiZR9Zu6I5xo"
# ## Distribution of the target variable

# + colab={"base_uri": "https://localhost:8080/", "height": 404} id="J5xyqTXqHCYo" outputId="a5e94eb4-1475-4144-c4ea-0281f8784acd"
plt.figure(figsize=(7, 6))
sns.countplot(train.target)
plt.title('Target Variable Distribution');

# + [markdown] id="2r3BS3IcKP7f"
# This shows us that the target (Can you make a payment if you were in an emergency) is fairly balanced. The majority class in this dataset are people who can make a payment incase of an emergency

# + colab={"base_uri": "https://localhost:8080/", "height": 523} id="izjup23WHCVr" outputId="d4fc229a-9618-40f2-99c0-7aed0dbfe7fb"
# Q1 - Has ATM/debit card
plt.figure(figsize=(8, 7))
ax =sns.countplot(train.Q1)
ax.set_xticklabels(['Yes', 'No', 'Don"t Know', 'Refused to answer'], rotation=45 )
plt.title('Distribution of Q1 - Has ATM/debit card', fontdict = {'size': 15});

# + [markdown] id="4WpL_l95Nlxq"
# This shows that most people do not own a debit or ATM card

# + [markdown] id="g9oZ14qYN1FY"
# ## Combine train and test set for easy preprocessing 

# + colab={"base_uri": "https://localhost:8080/"} id="8PH2LhntHCTj" outputId="b0c166b3-28bf-4dc4-f072-4219e99d4c0c"
# Combine train and test set
ntrain = train.shape[0] # to be used to split train and test set from the combined dataframe

all_data = pd.concat((train, test)).reset_index(drop=True)
print(f'The shape of the combined dataframe is: {all_data.shape}')

# + colab={"base_uri": "https://localhost:8080/"} id="4chstsvnHCRS" outputId="528302e0-53f9-4f9c-9289-9952fd05bb3a"
# Check the column names and datatypes
all_data.info()

# + colab={"base_uri": "https://localhost:8080/"} id="3AADdmJ6HCOr" outputId="c4b4a051-37e9-403d-f042-567cb49351e4"
# Category columns
cat_cols = ['country',	'region', 'owns_mobile'] + [x for x in all_data.columns if x.startswith('Q')]
num_cols = ['age', 'population']

# Change columns to their respective datatypes
all_data[cat_cols] = all_data[cat_cols].astype('category')

# Confirm whether the changes have been successful
all_data.info()

# + [markdown] id="_QFo9TxSkY6w"
# ### Number of unique values per categorical column

# + colab={"base_uri": "https://localhost:8080/"} id="2gc8aFu_kcU6" outputId="8eff12c2-e4c7-4ff8-cbd6-a31b66d5f592"
# Check unique values for each categorical column
for col in cat_cols:
  print(col, all_data[col].nunique())

# +
# o, t, th, f = 0, 0, 0, 0
# for i in train['Q27']:
#     if i == 1.0: o+=1
#     elif i == 2.0: t+=1
#     elif i == 3.0: th+=1
#     elif i == 4.0: f+=1
# print(len(train['Q27']))
# print(o, t, th, f)

# + [markdown] id="wUA3IqW42Oeq"
# ## Fill in missing values
# Missing values can be filled using different strategies
#  - Mean
#  - Max
#  - Min
#  - for categorical variables - mode
#  - [sklearn SimpleImputer](https://scikit-learn.org/stable/modules/generated/sklearn.impute.SimpleImputer.html)
#  - Others... do more reasearch

# + colab={"base_uri": "https://localhost:8080/"} id="VQ4t3UbC2Nsy" outputId="176e804c-12a8-4f76-8a12-af4de9a2665f"
# Fill in missing values
# For cat cols and date cols fill in with mode and for num cols fill in with 9999
from sklearn.impute import SimpleImputer

imp = SimpleImputer(strategy='mean')

for col in all_data.columns:
  if col in cat_cols:
    all_data[col] = all_data[col].fillna(all_data[col].mode()[0])
  elif col in num_cols:
    all_data[col] = imp.fit_transform(np.array(all_data[col]).reshape(-1, 1))
#     all_data[col] = all_data[col].fillna(all_data[col].fillna(9999))

# Confirm that there aren't any missing values
all_data[all_data.columns.difference(['target'])].isna().sum().any()
# -

# fill age column
imp = SimpleImputer(strategy='mean')
train['age'] = imp.fit_transform(np.array(train['age']).reshape(-1, 1))

train.isnull().sum()

# + [markdown] id="3Hd9viOb2_c3"
# ### Feature Engineering
# #### Try different strategies of dealing with categorical variables
#  - One hot encoding
#  - Label encoding
#  - Target encoding
#  - Reduce the number of unique values...

# + colab={"base_uri": "https://localhost:8080/"} id="X2DUKqTb3UY-" outputId="428bb9c0-2d5c-4d68-b206-4ac1069a613d"
# Shape of data before encoding
all_data.shape

# + colab={"base_uri": "https://localhost:8080/", "height": 278} id="vQtjAQw92Np4" outputId="782fe9cb-276c-45f1-de3a-861f918fac54"
# Use one hot encoding to turn categorical features to numerical features
# Encode categorical features
all_data = pd.get_dummies(data = all_data, columns = cat_cols)
all_data.head()

# + colab={"base_uri": "https://localhost:8080/"} id="xijxGVyB3Z3K" outputId="aaeda529-cd48-4d0d-e3d3-941e7e518c26"
# Shape of data after encoding
all_data.shape

# + colab={"base_uri": "https://localhost:8080/"} id="xXRt7bHS3fhH" outputId="c21bc672-e690-4c0e-a106-304978846a4f"
# Separate train and test data from the combined dataframe
train_df = all_data[:ntrain]
test_df = all_data[ntrain:]

# Check the shapes of the split dataset
train_df.shape, test_df.shape

# + [markdown] id="eDBSVM-S4E-h"
# ### Training and making predictions
#
# - Is lgbm the best model for this challenge?
# - Parameter tuning
#   - Grid search, random search, perhaps bayesian search works better...

# + colab={"base_uri": "https://localhost:8080/"} id="MzMVccR34ARy" outputId="b74bc80f-ec2a-41cb-de11-7e1793bb92c3"
# Select main columns to be used in training
main_cols = all_data.columns.difference(['ID', 'target'])
X = train_df[main_cols]
y = train_df.target.astype(int)

# Split data into train and test sets
X_train, X_test, y_train, y_test = train_test_split(X,y,test_size=0.3, random_state=42)

# Train model
model = RandomForestClassifier()
model.fit(X_train, y_train)

# Make predictions
y_pred = model.predict_proba(X_test)[:, 1]

# Check the auc score of the model
print(f'RandomForest AUC score on the X_test is: {roc_auc_score(y_test, y_pred)}\n')

# print classification report
print(classification_report(y_test, [1 if x >= 0.5 else 0 for x in y_pred]))

# + [markdown] id="tFULqvbM659e"
# ### Train different model and compare results

# +
# Select main columns to be used in training
main_cols = all_data.columns.difference(['ID', 'target'])
X = train_df[main_cols]
y = train_df.target.astype(int)

# Split data into train and test sets
X_train, X_test, y_train, y_test = train_test_split(X,y,test_size=0.13, random_state=41)

# +
# Train model
model2 = LGBMClassifier()
model2.fit(X_train.values, y_train.values)

# Make predictions
y_pred = model2.predict_proba(X_test)[:, 1]

# Check the auc score of the model
print(f'LGBM AUC score on the X_test is: {roc_auc_score(y_test, y_pred)}\n')

# print classification report
print(classification_report(y_test, [1 if x >= 0.5 else 0 for x in y_pred]))

# +
# Select main columns to be used in training
main_cols = all_data.columns.difference(['ID', 'target'])
X = train_df[main_cols]
y = train_df.target.astype(int)

# Split data into train and test sets
X_train, X_test, y_train, y_test = train_test_split(X,y,test_size=0.3, random_state=41)

# +
from xgboost import XGBClassifier
# Train model
model3 = XGBClassifier()
model3.fit(X_train.values, y_train.values)

# Make predictions
y_pred = model3.predict_proba(X_test)[:, 1]

# Check the auc score of the model
print(f'XGB AUC score on the X_test is: {roc_auc_score(y_test, y_pred)}\n')

# print classification report
print(classification_report(y_test, [1 if x >= 0.5 else 0 for x in y_pred]))

# +
# Select main columns to be used in training
main_cols = all_data.columns.difference(['ID', 'target'])
X = train_df[main_cols]
y = train_df.target.astype(int)

# Split data into train and test sets
X_train, X_test, y_train, y_test = train_test_split(X,y,test_size=0.3, random_state=41)

# +
from catboost import CatBoostClassifier
# Train model
seed = 42
model = CatBoostClassifier(iterations=1000, verbose=0)
model.fit(X_train.values, y_train.values)

# Make predictions
y_pred = model.predict_proba(X_test)[:, 1]

# Check the auc score of the model
print(f'CatBoostClassifier AUC score on the X_test is: {roc_auc_score(y_test, y_pred)}\n')

# print classification report
print(classification_report(y_test, [1 if x >= 0.5 else 0 for x in y_pred]))

# +
# Make prediction on the test set
test_df = test_df[main_cols]
# model.fit(X,y)
model2.fit(X.values,y)
# model3.fit(X,y)
predictions1  = model.predict_proba(test_df)[:, 1]
predictions2 = model2.predict_proba(test_df)[:, 1]
# predictions3 = model3.predict_proba(test_df)[:, 1]

predictions =  (predictions1*0.6 + predictions2*0.3) #+ predictions3*0.1)

# # Create a submission file
sub_file = samplesubmission.copy()
sub_file.target = predictions

# # Check the distribution of your predictions
sns.countplot([1 if x >= 0.5 else 0 for x in sub_file.target])
plt.title('Predicted Variable Distribution');

# + [markdown] id="bXKU5avW6uaT"
# - There is a light improvement when using LGBMClassifier, XGBoostClassifier, CatBoostClassifier
#
# [More on AUC score](https://developers.google.com/machine-learning/crash-course/classification/roc-and-auc#:~:text=AUC%20represents%20the%20probability%20that,has%20an%20AUC%20of%201.0.)

# + [markdown] id="2L7-CX1c7l-u"
# ### Making predictions of the test set and creating a submission file

# + colab={"base_uri": "https://localhost:8080/", "height": 281} id="2CY70-KV5lYO" outputId="a928aaf6-c5c9-4e76-eda1-377e999617a1"
# Make prediction on the test set
test_df = test_df[main_cols]
predictions = model.predict_proba(test_df)[:, 1]

# # Create a submission file
sub_file = samplesubmission.copy()
sub_file.target = predictions

# # Check the distribution of your predictions
sns.countplot([1 if x >= 0.5 else 0 for x in sub_file.target])
plt.title('Predicted Variable Distribution');

# + colab={"base_uri": "https://localhost:8080/", "height": 195} id="oroCrCK27v-4" outputId="6de7be09-47f7-41d7-e59e-1c30726c8c3f"
# Create a csv file and upload to zindi 
sub_file.to_csv('Baseline.csv', index = False)
sub_file.head()

# + [markdown] id="co8AT8lL7y5s"
# ###More Tips
# - Thorough EDA and domain knowledge sourcing
# - Re-group Categorical features 
# - More Feature Engineering 
# - Dataset balancing - oversampling, undersampling, SMOTE...
# - Ensembling of models 
# - Cross-validation: Group folds, Stratified...

# + [markdown] id="ih0ejC9w7617"
# # ******************* GOOD LUCK!!! ***************************

StarcoderdataJupyterScriptsDedupFiltered