# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     name: python3
# ---

# + [markdown] id="view-in-github" colab_type="text"
# <a href="https://colab.research.google.com/github/cxbxmxcx/TensorFlow_Certification/blob/master/TF_certification_raw.ipynb" target="_parent"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/></a>

# + [markdown] id="92_Lp4UfSj9L" colab_type="text"
# Import and setup

# + id="CYQrqc_vRpUp" colab_type="code" colab={}
import numpy as np
import matplotlib.pyplot as plt
from pylab import rcParams

# %matplotlib inline

rcParams['figure.figsize'] = 12, 6

RANDOM_SEED = 56

np.random.seed(RANDOM_SEED)

# + [markdown] id="cE4nVJrhScBM" colab_type="text"
# Environment Data

# + id="ZEA-9Y8eR1Dd" colab_type="code" colab={}
X = np.array([[1], [2], [3]])
y = np.array([ [0], [1], [1]])


# + [markdown] id="zoKbdrylSWhD" colab_type="text"
# Activation Functions

# + id="NETV1Jy8SVnJ" colab_type="code" colab={}
def sigmoid(x):
  """
  Computes the sigmoid function sigm(input) = 1/(1+exp(-input))
  """
  return 1 / (1 + np.exp(-x))

def sigmoid_(y):
  """
  Computes the derivative of sigmoid funtion. sigmoid(y) * (1.0 - sigmoid(y)). 
  The way we implemented this requires that the input y is already sigmoided
  """
  return y * (1-y)


# + [markdown] id="UVV1Pj_3SwXt" colab_type="text"
# Multiply and Sum inputs*weights

# + id="NeNySocuSvdJ" colab_type="code" colab={}
def dense(inputs, weights):
    """A simple dense layer."""
    return np.matmul(inputs, weights)


# + [markdown] id="-hCa08t5S8XU" colab_type="text"
# Initialization

# + id="P_DImmW7S7WB" colab_type="code" colab={}
input_size = 1
hidden_size = 3
output_size = 1

def initialize_weights():
    # weights for hidden layer, shape: 1x3
    w1 = np.random.uniform(size=(input_size, hidden_size))
    # weights for output layer, shape: 3x1
    w2 = np.random.uniform(size=(hidden_size, output_size))
    return w1, w2
    
w1, w2 = initialize_weights()


# + [markdown] id="UKuXJjGKTVfE" colab_type="text"
# Forward Pass

# + id="ticUNSK5TUwp" colab_type="code" colab={}
def forward_pass(X):
    # Step 1: Calculate weighted average of inputs (output shape: 1x3)
    net_hidden = dense(X, w1)
    
    # Step 2: Calculate the result of the sigmoid activation function (shape: 1x3)
    act_hidden = sigmoid(net_hidden)
    
    # Step 3: Calculate output of neural network (output shape: 4x1)
    y_hat = dense(act_hidden, w2)
    
    return act_hidden, y_hat


# + [markdown] id="-S5dUTTkTjyR" colab_type="text"
# Loss Function

# + id="UMpOVl8eTmHx" colab_type="code" colab={}
def mse(y_hat, y):
    residual = y_hat - y
    error = np.mean(0.5 * (residual ** 2))
    return residual, error


# + [markdown] id="KkNndE3HTuMs" colab_type="text"
# Backpropagation

# + id="qXysczNGTx0e" colab_type="code" colab={}
def backward_pass(X, y_hat, act_hidden):
    # Step 1: Calculate error
    residual, error = mse(y_hat, y)
    
    # Step 2: calculate gradient wrt w2
    N = X.shape[0]
    dL_dy = 1.0 / N * residual  # shape (4, 1)
    dy_dw2 = act_hidden  # shape (4, 3)
    dL_dw2 = np.matmul(dL_dy.T, dy_dw2)  # shape (1, 3)
    
    # According to the math, `dL_dw2` is a row-vector, however, `w2` is a column-vector.
    # To prevent erroneous numpy broadcasting during the gradient update, we must make
    # sure that `dL_dw2` is also a column-vector.
    dL_dw2 = dL_dw2.T
    
    # Step 3: calculate gradient wrt w1
    dL_dw1 = 1.0 / N * np.matmul(X.T, np.matmul(residual, w2.T) * sigmoid_(act_hidden))
    
    return dL_dw2, dL_dw1, error


# + [markdown] id="_J2ZY3QMUKVi" colab_type="text"
# Optimization

# + id="kG67xDqQTz9_" colab_type="code" outputId="59570b4c-a48a-41e3-9f4d-908f488af8c4" colab={"base_uri": "https://localhost:8080/", "height": 1000}
n_epochs = 10000
learning_rate = 0.1
training_errors = []

# re-initialize the weights to be sure we start fresh
w1, w2 = initialize_weights()

for epoch in range(n_epochs):
 
    # Step 1: forward pass
    act_hidden, y_hat = forward_pass(X)
    
    # Step 2: backward pass
    dw2, dw1, error = backward_pass(X, y_hat, act_hidden)
    
    # Step 3: apply gradients scaled by learning rate
    w2 = w2 - learning_rate * dw2
    w1 = w1 - learning_rate * dw1
    
    # Step 4: some book-keeping and print-out
    if epoch % 200 == 0:
        print('Epoch %d> Training error: %f' % (epoch, error))
    training_errors.append([epoch, error])

# Plot training error progression over time
training_errors = np.asarray(training_errors)
plt.plot(training_errors[:, 0], training_errors[:, 1]);
plt.xlabel('Epochs');
plt.ylabel('Training Error');

# + [markdown] id="dzycmZJeUuQC" colab_type="text"
# Predict

# + id="aOEZSxLaUs2N" colab_type="code" outputId="8396340d-4888-4146-c42f-672dd0107244" colab={"base_uri": "https://localhost:8080/", "height": 35}
print(forward_pass([3]))

StarcoderdataJupyterScriptsDedupFiltered