# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernel_info:
#     name: python3
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# <img align="left" src="https://lever-client-logos.s3.amazonaws.com/864372b1-534c-480e-acd5-9711f850815c-1524247202159.png" width=200>
# <br></br>
# <br></br>
#
# ## *Data Science Unit 4 Sprint 2*
#
# # Sprint Challenge - Neural Network Foundations
#
# Table of Problems
#
# 1. [Defining Neural Networks](#Q1)
# 2. [Chocolate Gummy Bears](#Q2)
#     - Perceptron
#     - Multilayer Perceptron
# 4. [Keras MMP](#Q3)

# <a id="Q1"></a>
# ## 1. Define the following terms:
#
# - **Neuron:** units in a neural network that recieve inputs from other nodes or an external source to compute an output
# - **Input Layer:** data collected for each of the independent variables. X1,X2,X3 and so on
# - **Hidden Layer:** layers between inputs and out put with different itterations definging the weights a biases for selected input to achieve desired output
# - **Output Layer:** information displayed to us after defining the input and adjusting how the hidden layers will work.
# - **Activation:** a non-linier function that take the inputs of a NN and defines the oputput
# - **Backpropagation:** helps weights adjust effectively after learning the mistake enhance to a better score
#

# ## 2. Chocolate Gummy Bears <a id="Q2"></a>
#
# Right now, you're probably thinking, "yuck, who the hell would eat that?". Great question. Your candy company wants to know too. And you thought I was kidding about the [Chocolate Gummy Bears](https://nuts.com/chocolatessweets/gummies/gummy-bears/milk-gummy-bears.html?utm_source=google&utm_medium=cpc&adpos=1o1&gclid=Cj0KCQjwrfvsBRD7ARIsAKuDvMOZrysDku3jGuWaDqf9TrV3x5JLXt1eqnVhN0KM6fMcbA1nod3h8AwaAvWwEALw_wcB). 
#
# Let's assume that a candy company has gone out and collected information on the types of Halloween candy kids ate. Our candy company wants to predict the eating behavior of witches, warlocks, and ghosts -- aka costumed kids. They shared a sample dataset with us. Each row represents a piece of candy that a costumed child was presented with during "trick" or "treat". We know if the candy was `chocolate` (or not chocolate) or `gummy` (or not gummy). Your goal is to predict if the costumed kid `ate` the piece of candy. 
#
# If both chocolate and gummy equal one, you've got a chocolate gummy bear on your hands!?!?!
# ![Chocolate Gummy Bear](https://ed910ae2d60f0d25bcb8-80550f96b5feb12604f4f720bfefb46d.ssl.cf1.rackcdn.com/3fb630c04435b7b5-2leZuM7_-zoom.jpg)

# + inputHidden=false jupyter={"outputs_hidden": false} outputHidden=false
import pandas as pd
candy = pd.read_csv('chocolate_gummy_bears.csv')

# + inputHidden=false jupyter={"outputs_hidden": false} outputHidden=false
candy.head()
# -

# ### Perceptron
#
# To make predictions on the `candy` dataframe. Build and train a Perceptron using numpy. Your target column is `ate` and your features: `chocolate` and `gummy`. Do not do any feature engineering. :P
#
# Once you've trained your model, report your accuracy. You will not be able to achieve more than ~50% with the simple perceptron. Explain why you could not achieve a higher accuracy with the *simple perceptron* architecture, because it's possible to achieve ~95% accuracy on this dataset. Provide your answer in markdown (and *optional* data anlysis code) after your perceptron implementation. 

# + inputHidden=false jupyter={"outputs_hidden": false} outputHidden=false
# Start your candy perceptron here

X = candy[['chocolate', 'gummy']].values
y = candy['ate'].values

# +
import numpy as np

def sigmoid(x):
    return 1 / (1 + np.exp(-x))

def sigmoid_derivate(x):
    sx = sigmoid(x)
    return( sx * (1-sx))

class Perceptron(object):
    
    def __init__(self, niter = 1):
        self.niter = niter
    

    def fit(self, X, y):
        
        inputs = X
        correct_outputs = y.reshape(-1,1)

        weights = 2 * np.random.random((X.shape[1],1)) - 1
       # bias = 2 * np.random.random((len(X),1)) - 1
        
        for i in range(self.niter):

            weighted_sum = np.dot(inputs, weights)# + bias

            activated_output = sigmoid(weighted_sum)

   
            error = correct_outputs - activated_output

            adjustments = error * sigmoid_derivate(activated_output)

  
            #bias += error
            weights += np.dot(inputs.T, adjustments)
        self.activated_output = activated_output
        self.weights = weights
        #self.bias = bias
        

    def predict(self, X):
        weighted_sum = np.dot(X, self.weights) #+ self.bias
        
        return sigmoid(weighted_sum)
# -

nn = Perceptron()
nn.fit(X,y)

# +
pred = nn.predict(X)

n_pred = []
for i in pred:
    n_pred.append(int(i))
from sklearn.metrics import accuracy_score as acs

acs(y, n_pred)


# -

# a simple perceptron does not have the bias nor does it have weights updated effictively.

# ### Multilayer Perceptron <a id="Q3"></a>
#
# Using the sample candy dataset, implement a Neural Network Multilayer Perceptron class that uses backpropagation to update the network's weights. Your Multilayer Perceptron should be implemented in Numpy. 
# Your network must have one hidden layer.
#
# Once you've trained your model, report your accuracy. Explain why your MLP's performance is considerably better than your simple perceptron's on the candy dataset. 

# + inputHidden=false jupyter={"outputs_hidden": false} outputHidden=false
class NeuralNetwork:
    def __init__(self):

        self.inputs = 2
        self.hiddenNodes = 3
        self.outputNodes = 1


        self.weights1 = np.random.rand(self.inputs, self.hiddenNodes)
        self.weights2 = np.random.rand(self.hiddenNodes, self.outputNodes)
        
    def sigmoid(self, s):
        return 1 / (1+np.exp(-s))
    
    def sigmoidPrime(self, s):
        return s * (1 - s)
    
    def feed_forward(self, X):

        self.hidden_sum = np.dot(X, self.weights1)

        self.activated_hidden = self.sigmoid(self.hidden_sum)
        self.output_sum = np.dot(self.activated_hidden, self.weights2)
        self.activated_output = self.sigmoid(self.output_sum)
        
        return self.activated_output
        
    def backward(self, X,y,o):

        self.o_error = y - o
        self.o_delta = self.o_error * self.sigmoidPrime(o)
        self.z2_error = self.o_delta.dot(self.weights2.T)
        self.z2_delta = self.z2_error * self.sigmoidPrime(self.activated_hidden)
        self.weights1 += X.T.dot(self.z2_delta)
        self.weights2 += self.activated_hidden.T.dot(self.o_delta)
        

    def train(self, X, y):
        o = self.feed_forward(X)
        self.backward(X,y,o)

nn = NeuralNetwork()

y.reshape(-1,1)
# -

for i in range(10000):
    if (i in [a for a in range(7)]) or ((i + 1) % 5000 == 0):
        print('+' + '---' * 3 + f'EPOCH {i+1}' + '---'*3 + '+')
        print('Input: \n', X)
        print('Actual Output: \n', y.reshape(-1,1))
        print('Predicted Output: \n', str(nn.feed_forward(X)))
        print("Loss: \n", str(np.mean(np.square(y.reshape(-1,1) - nn.feed_forward(X)))))
    nn.train(X,y.reshape(-1,1))


# +
def sigmoid(x):
    return 1 / (1 + np.exp(-x))

def sigmoid_derivate(x):
    sx = sigmoid(x)
    return( sx * (1-sx))

class Perceptron(object):
    
    def __init__(self, niter = 500):
        self.niter = niter
    

    def fit(self, X, y):
        
        inputs = X
        correct_outputs = y.reshape(-1,1)

        weights = 2 * np.random.random((X.shape[1],1)) - 1
        bias = 2 * np.random.random((len(X),1)) - 1
        
        for i in range(self.niter):

            weighted_sum = np.dot(inputs, weights) + bias

            activated_output = sigmoid(weighted_sum)

   
            error = correct_outputs - activated_output

            adjustments = error * sigmoid_derivate(activated_output)

  
            bias += error
            weights += np.dot(inputs.T, adjustments)
        self.activated_output = activated_output
        self.weights = weights
        self.bias = bias
        

    def predict(self, X):
        weighted_sum = np.dot(X, self.weights) + self.bias
        
        return sigmoid(weighted_sum)


nn = Perceptron()
nn.fit(X,y)


pred = nn.predict(X)
n_pred = []
for i in pred:
    n_pred.append(int(i))

acs(y,n_pred)
# -

# Backpropogation helps weights adjust effectively after learning the mistake enhance to a better score (+95%)

# P.S. Don't try candy gummy bears. They're disgusting. 

# ## 3. Keras MMP <a id="Q3"></a>
#
# Implement a Multilayer Perceptron architecture of your choosing using the Keras library. Train your model and report its baseline accuracy. Then hyperparameter tune at least two parameters and report your model's accuracy.
# Use the Heart Disease Dataset (binary classification)
# Use an appropriate loss function for a binary classification task
# Use an appropriate activation function on the final layer of your network.
# Train your model using verbose output for ease of grading.
# Use GridSearchCV or RandomSearchCV to hyperparameter tune your model. (for at least two hyperparameters)
# When hyperparameter tuning, show you work by adding code cells for each new experiment.
# Report the accuracy for each combination of hyperparameters as you test them so that we can easily see which resulted in the highest accuracy.
# You must hyperparameter tune at least 3 parameters in order to get a 3 on this section.

# + inputHidden=false jupyter={"outputs_hidden": false} outputHidden=false
import pandas as pd
from sklearn.preprocessing import StandardScaler

df = pd.read_csv('https://raw.githubusercontent.com/ryanleeallred/datasets/master/heart.csv')
df = df.sample(frac=1)
print(df.shape)
df.head()
# -

# !pip install keras

# !pip install tensorflow

# +
import sklearn
from sklearn.model_selection import GridSearchCV, train_test_split
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Dropout
from tensorflow.keras.wrappers.scikit_learn import KerasClassifier

X = df.iloc[:,0:14]
y = df.values[:,-1].astype('bool')

print(X.shape,y.shape)
# -

names = X.columns# Create the Scaler object
scaler = StandardScaler()# Fit your data on the scaler object
scaled_X = scaler.fit_transform(X)
X = pd.DataFrame(scaled_X, columns=names)

df.target.value_counts(normalize = True)

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=.2)
print(X_train.shape, y_train.shape, X_test.shape, y_test.shape)

# +
#Frame the model

import numpy as np
# Plant a random seed for reproducibility
seed = 1
np.random.seed(seed)

# Function to create model, required for KerasClassifier
def create_model():
# create model - Sigmoid activation function for binary/boolean type
    model = Sequential()
    model.add(Dense(24, input_dim=14, activation='sigmoid'))
    model.add(Dropout(rate = .3))
    model.add(Dense(12, activation='sigmoid'))
    model.add(Dense(1, activation='sigmoid'))
    
    # Compile model
    model.compile(loss='binary_crossentropy',
                  optimizer='adam',
                  metrics=['accuracy'])
    return model

# create model; verbose? Yes
model = KerasClassifier(build_fn=create_model, verbose=1)

# define the grid search parameters; Bests: batch_size = , epochs = 
param_grid = {'batch_size': [1,10,30,60,120],  
              'epochs': [10]
             } 

# Create Grid Search
grid = GridSearchCV(estimator=model, param_grid=param_grid, n_jobs=-1)

grid_result = grid.fit(X_train, y_train)

# Report Results - best accuracy = 
print(f"Best: {grid_result.best_score_} using {grid_result.best_params_}")
means = grid_result.cv_results_['mean_test_score']
stds = grid_result.cv_results_['std_test_score']
params = grid_result.cv_results_['params']
for mean, stdev, param in zip(means, stds, params):
    print(f"Means: {mean}, Stdev: {stdev} with: {param}")

# +
#trying a higher batch size for better accuracy

# define the grid search parameters; Bests: batch_size = , epochs = 
param_grid = {'batch_size': [120],  
              'epochs': [10, 20, 40, 80, 160]
             }

# Create Grid Search
grid = GridSearchCV(estimator=model, param_grid=param_grid, n_jobs=-1)

grid_result = grid.fit(X_train, y_train)

# Report Results - best accuracy = 
print(f"Best: {grid_result.best_score_} using {grid_result.best_params_}")
means = grid_result.cv_results_['mean_test_score']
stds = grid_result.cv_results_['std_test_score']
params = grid_result.cv_results_['params']
for mean, stdev, param in zip(means, stds, params):
    print(f"Means: {mean}, Stdev: {stdev} with: {param}")

# +
#refining epoch values

# define the grid search parameters; Bests: batch_size = , epochs = 
param_grid = {'batch_size': [120],  
              'epochs': [30, 40, 50, 60, 70]
             }

# Create Grid Search
grid = GridSearchCV(estimator=model, param_grid=param_grid, n_jobs=-1)

grid_result = grid.fit(X_train, y_train)

# Report Results - best accuracy = 
print(f"Best: {grid_result.best_score_} using {grid_result.best_params_}")
means = grid_result.cv_results_['mean_test_score']
stds = grid_result.cv_results_['std_test_score']
params = grid_result.cv_results_['params']
for mean, stdev, param in zip(means, stds, params):
    print(f"Means: {mean}, Stdev: {stdev} with: {param}")
# -





StarcoderdataJupyterScriptsDedupFiltered