"""
# Deep Learning - Hyper Parameter Tuning

* We can use Hyper-Parameer to decide Number of Hidden Layers required in a Neural Network.


"""
"""
Using the dataset of Churn Modelling. Refer to https://www.kaggle.com/dskagglemt/churn-modelling-using-keras 
"""
# This Python 3 environment comes with many helpful analytics libraries installed
# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python
# For example, here's several helpful packages to load

import numpy as np # linear algebra
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)

# Input data files are available in the read-only "../input/" directory
# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory

import os
for dirname, _, filenames in os.walk('/kaggle/input'):
    for filename in filenames:
        print(os.path.join(dirname, filename))

# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using "Save & Run All" 
# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session
df = pd.read_csv('/kaggle/input/churn-modelling/Churn_Modelling.csv')
df.head()
X = df.iloc[:,3:13]
y = df.iloc[:, 13]

X.Geography.unique()

X['Gender'].unique()

geo_cat = pd.get_dummies(X["Geography"], drop_first = True)
gender_cat = pd.get_dummies(X['Gender'], drop_first = True)

# merge geo_cat and gender_cat into our X.

X = pd.concat([X, geo_cat, gender_cat], axis = 1)

X = X.drop(['Geography',"Gender"], axis = 1)
from sklearn.model_selection import train_test_split
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.2, random_state = 9)
from sklearn.preprocessing import StandardScaler

sc = StandardScaler()

X_train = sc.fit_transform(X_train)
X_test = sc.fit_transform(X_test)
"""
# Perform Hyperparameter Optimization
"""
from keras.wrappers.scikit_learn import KerasClassifier
from sklearn.model_selection import GridSearchCV

# import keras
from keras.models import Sequential
from keras.layers import Dense, Activation, Embedding, Flatten, BatchNormalization
from keras.layers import LeakyReLU, PReLU, ELU
from keras.layers import Dropout
from keras.activations import relu, sigmoid
# Define a method to create a model.
def create_model(layers, activation):
    model = Sequential()
    
    for i, nodes in enumerate(layers):
        if i == 0:
            model.add(Dense(nodes, input_dim = X_train.shape[1]))
            model.add(Activation(activation))
            model.add(Dropout(0.3))
        else:
            model.add(Dense(nodes))
            model.add(Activation(activation))
            model.add(Dropout(0.3))
            
    model.add(Dense(units = 1, kernel_initializer = 'glorot_uniform', activation = 'sigmoid'))
    
    model.compile(optimizer ='adam', loss = 'binary_crossentropy', metrics = ['accuracy'])
    
    return model
model = KerasClassifier(build_fn = create_model, verbose = 0)
# layers = [[20], [40,20], [45,30,15]]
# In case your have runtime error, you may need to change the list of lists that specify the layers to a list of tuples
# RuntimeError: Cannot clone object <tensorflow.python.keras.wrappers.scikit_learn.KerasClassifier object at 0x7fb8700b6f90>, as the constructor either does not set or modifies parameter layers
# https://stackoverflow.com/questions/59818584/cannot-clone-object-keras-wrappers

layers = [(20,), (40,20), (45,30,15)] 

# it indicates that we need a layers as 1 hidden layer with 20 neurons.
# [40,20] will indicates 2 hidden layer, with 40 neurns at first layer, and 20 neurons on 2nd hidden layer
# and so on

# Dont forget to add (,) in (20,) otherwise it will throw an error like : TypeError: 'int' object is not iterable This is because single tuple without comma(,) is treated as int.

activations = ['sigmoid', 'relu']

param_grid =dict(layers = layers, activation = activations, batch_size = [128, 256], epochs = [30])

grid = GridSearchCV(estimator = model, param_grid = param_grid, cv = 5)
grid_result = grid.fit(X_train, y_train)
"""
# Best Model 
"""
grid_result.best_score_, grid_result.best_params_
"""
Above is the Best parameters and activation function and epochs to be use for better result.
"""
y_pred = grid.predict(X_test)
y_pred = (y_pred > 0.5)
# Lets see the accuracy of our Test Dataset.
from sklearn.metrics import confusion_matrix
cm = confusion_matrix(y_test, y_pred)
cm
from sklearn.metrics import accuracy_score
score = accuracy_score(y_pred, y_test)

score