# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: tf_env
#     language: python
#     name: tf_env
# ---

# ![eo_logo.png](eo_logo.png)

# # Tuning a Neural Network's arhcitecture and hyperparameters with a genetic algorithm

# This notebook will show how a neural network's architecture and hyperparameters can be tuned using a **genetic algorithm**. The NN will be evaluated on the sonar dataset. 
#
# The optimisation process includes three main steps:<br>
# 1) Coding the **evaluation function** - taking a dictionar of parameters as argument and returning a scalar <br>
# 2) Defining the **search space**, a list of integer, real or categorical parameters<br>
# 3) Running the **optimisation function**<br>
#
# The datasety can be found in the [UCI Machine Learning Repository](https://archive.ics.uci.edu/ml/datasets/Connectionist+Bench+(Sonar,+Mines+vs.+Rocks))
#
# The NN evaluation function was informed by this [web page](https://machinelearningmastery.com/binary-classification-tutorial-with-the-keras-deep-learning-library/)
#
# General evolution_opt documentation can be found [here](https://eliottkalfon.github.io/evolution_opt/)

# ## Importing the main packages

# +
import pandas as pd
from evolution_opt.genetic import *

from keras.models import Sequential
from keras.layers import Dense, Dropout
from keras.wrappers.scikit_learn import KerasClassifier

from sklearn.model_selection import cross_val_score
from sklearn.preprocessing import LabelEncoder
from sklearn.model_selection import StratifiedKFold
from sklearn.preprocessing import StandardScaler
from sklearn.pipeline import Pipeline

# -

# ## Reading and preparing the sonar dataset

# load dataset
dataframe = pd.read_csv('sonar_dataset.txt', header=None)
dataset = dataframe.values
# split into input (X) and output (Y) variables
X = dataset[:,0:60].astype(float)
Y = dataset[:,60]
# encode class values as integers
encoder = LabelEncoder()
encoder.fit(Y)
encoded_Y = encoder.transform(Y)


# ## Coding the evaluation function

def evaluate_nn(param_dict):
    '''
    This function will create and evaluate a NN based on the parameters of a given individual
    '''
    def model_build():
        model = Sequential()
        model.add(Dense(param_dict['n_neurons_layer1'], input_dim=60, activation=param_dict['activation_fn']))
        #The NN architecture will not set a dropout superior to 0.5
        #If a dropout parameter has a value superior to 0.5, no dropout is applied
        if param_dict['dropout_1'] < 0.5:
        model.add(Dropout(param_dict['dropout_1']))
        if param_dict['n_hidden_layers'] >= 2:
            model.add(Dense(param_dict['n_neurons_layer2'], activation=param_dict['activation_fn']))
            if param_dict['dropout_2'] < 0.5:
            model.add(Dropout(param_dict['dropout_2']))
        if param_dict['n_hidden_layers'] >= 3:
            model.add(Dense(param_dict['n_neurons_layer3'], activation=param_dict['activation_fn']))
            if param_dict['dropout_3'] < 0.5:
            model.add(Dropout(param_dict['dropout_3']))
        model.add(Dense(1, activation='sigmoid'))
        #Compile model
        model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
        return model
    #Evaluate the model on the sonar dataset
    estimators = []
    estimators.append(('standardize', StandardScaler()))
    estimators.append(('mlp', KerasClassifier(build_fn=model_build, epochs=param_dict['n_epochs'], batch_size=param_dict['batch_size'], verbose=0)))
    pipeline = Pipeline(estimators)
    kfold = StratifiedKFold(n_splits=3, shuffle=True)
    results = cross_val_score(pipeline, X, encoded_Y, cv=kfold)
    #Returns the average accuracy across the cross validation splits
    return np.mean(results)


# ## Defining the search space

#Search space parameters can be Real, Integers or Categorical
search_space = [
    Real(0.001, 0.8, 'learning_rate', precision = 2, prior = 'lognormal', mean = 0.1, stdev = 1),
    Integer(1,3, 'n_hidden_layers'),
    Integer(20, 140, 'n_epochs', step = 20),
    Integer(2, 32, 'batch_size', step = 2),
    Categorical(['relu', 'sigmoid', 'tanh'], 'activation_fn'),
    Integer(20, 150, 'n_neurons_layer1', step = 10),
    Integer(20, 150, 'n_neurons_layer2', step = 10),
    Integer(20, 150, 'n_neurons_layer3', step = 10),
    Real(0, 1, 'dropout_1', precision = 2),
    Real(0, 1, 'dropout_2', precision = 2),
    Real(0, 1, 'dropout_3', precision = 2),
]

# **Note**: Following our configuration, no dropout will be applied if the selected rate is above 0.5

# ## Running the optimisation function

best_params = optimise(evaluate_nn, search_space,
             minimize=False, population_size=10,
             n_rounds=5, n_children=10, verbose=True)

best_params

# **Note**: the best architecture has a single hidden layer. All of the parameters containing the suffix "2" or "3" do not have any implication on the model



StarcoderdataJupyterScriptsDedupFiltered