# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# +
import matplotlib.pyplot as plt
import numpy as np
import sklearn
import sklearn.datasets

import neural_net
import optimization
import performance


# -

# ## Model with different optimization algorithms
#

# +
def load_dataset():
    np.random.seed(3)
    train_X, train_Y = sklearn.datasets.make_moons(n_samples=300, noise=.2) #300 #0.2 
    # Visualize the data
    plt.scatter(train_X[:, 0], train_X[:, 1], c=train_Y, s=40, cmap=plt.cm.Spectral)
    train_X = train_X.T
    train_Y = train_Y.reshape((1, train_Y.shape[0]))
    
    return train_X, train_Y

def plot_decision_boundary(clf, X, y):

    # Set min and max values and give it some padding
    x_min, x_max = X[0, :].min() - 1, X[0, :].max() + 1
    y_min, y_max = X[1, :].min() - 1, X[1, :].max() + 1
    h = 0.01
    # Generate a grid of points with distance h between them

    xx, yy = np.meshgrid(np.arange(x_min, x_max, h), np.arange(y_min, y_max, h))
    Z = np.c_[xx.ravel(), yy.ravel()].T
    Z = clf.predict(Z)
    Z = Z.reshape(xx.shape)
    plt.contourf(xx, yy, Z, cmap=plt.cm.Spectral)
    plot_data(X.T, y.T.ravel())

def plot_data(X, y):
    plt.scatter(X[y == 0, 0], X[y == 0, 1], color="red", s=30, label="Cluster1")
    plt.scatter(X[y == 1, 0], X[y == 1, 1], color="blue", s=30, label="Cluster2")


# -

train_X, train_Y = load_dataset()

# ### MINI-BATCH GRADIENT DESCENT

layers_dims = [train_X.shape[0], 5, 2, 1]
init_method = 'he'
activations = ('relu', 'sigmoid')
lambd = 0
optimizer_name = 'gd'
learning_rate = 0.0007
num_epochs = 10000

clf = neural_net.MLNN(layers_dims, init_method, activations, lambd, optimizer_name, learning_rate, num_epochs)

clf.train(train_X, train_Y)

predictions = clf.predict(train_X)
gd_accuracy = performance.compute_accuracy(train_Y, predictions)
print('Gradient descent optimization accuracy = ', gd_accuracy, '%')

# **GETTING THE MLNN PARAMS**

params = clf.get_params()

plt.plot(params['costs'])
plt.title('Cost 100 per epoch')

plot_decision_boundary(clf, train_X, train_Y)

# ### MINI-BATCH GRADIENT DESCENT WITH MOMENTUM

layers_dims = [train_X.shape[0], 5, 2, 1]
init_method = 'he'
activations = ('relu', 'sigmoid')
lambd = 0
optimizer_name = 'momentum'
learning_rate = 0.0007
num_epochs = 10000

clf = neural_net.MLNN(layers_dims, init_method, activations, lambd,
                      optimizer_name, learning_rate, num_epochs)

clf.train(train_X, train_Y)

predictions = clf.predict(train_X)
momentum_accuracy = performance.compute_accuracy(train_Y, predictions)
print('Momentum optimization accuracy = ', momentum_accuracy, '%')

plot_decision_boundary(clf, train_X, train_Y)

params = clf.get_params()
plt.plot(params['costs'])
plt.title('Cost 100 per epoch')

# ### MINI-BATCH GRADIENT DESCENT WITH RMSPROP

layers_dims = [train_X.shape[0], 5, 2, 1]
init_method = 'he'
activations = ('relu', 'sigmoid')
lambd = 0
optimizer_name = 'rmsprop'
learning_rate = 0.0007
num_epochs = 10000

clf = neural_net.MLNN(layers_dims, init_method, activations, lambd, optimizer_name, learning_rate, num_epochs)

clf.train(train_X, train_Y)

predictions = clf.predict(train_X)
rmsprop_accuracy = performance.compute_accuracy(train_Y, predictions)
print('RMSProp optimization accuracy = ', rmsprop_accuracy, '%')

plot_decision_boundary(clf, train_X, train_Y)

params = clf.get_params()
plt.plot(params['costs'])
plt.title('Cost 100 per epoch')

# ### MINI-BATCH GRADIENT DESCENT WITH ADAM

layers_dims = [train_X.shape[0], 5, 2, 1]
init_method = 'xavier'
activations = ('relu', 'sigmoid')
lambd = 0
optimizer_name = 'adam'
learning_rate = 0.0007
num_epochs = 10000

clf = neural_net.MLNN(layers_dims, init_method, activations, lambd, optimizer_name, learning_rate, num_epochs)

clf.train(train_X, train_Y)

predictions = clf.predict(train_X)
adam_accuracy = performance.compute_accuracy(train_Y, predictions)
print('ADAM optimization accuracy = ', adam_accuracy, '%')

plot_decision_boundary(clf, train_X, train_Y)

params = clf.get_params()
plt.plot(params['costs'])
plt.title('Cost 100 per epoch')

# + [markdown] variables={"np.round(adam_accuracy, 2)": "94.33", "np.round(gd_accuracy, 2)": "87.33", "np.round(momentum_accuracy, 2)": "85.0", "np.round(rmsprop_accuracy, 2)": "94.33"}
# ### Summary
#
# <table> 
#     <tr>
#         <td>
#             <b>optimization method</b>
#         </td>
#         <td>
#         <b>accuracy</b>
#         </td>
#     </tr>
#         <td>
#         Gradient descent
#         </td>
#         <td>
#         87.33%
#         </td>
#     <tr>
#         <td>
#         Momentum
#         </td>
#         <td>
#         85%
#         </td>
#     </tr>
#     <tr>
#         <td>
#         RMSProp
#         </td>
#         <td>
#         94.33%
#         </td>
#     </tr>
#     <tr>
#         <td>
#         Adam
#         </td>
#         <td>
#         94.33%
#         </td>
#     </tr>
# </table> 
#
# -



StarcoderdataJupyterScriptsDedupFiltered