<filename>recurrent_neuron_network.py
#https://nbviewer.org/github/amanchadha/coursera-deep-learning-specialization/blob/master/C5%20-%20Sequence%20Models/Week%201/Building%20a%20Recurrent%20Neural%20Network%20-%20Step%20by%20Step/Building_a_Recurrent_Neural_Network_Step_by_Step.ipynb


#Dimensions of prediction ŷ 
#Similar to the inputs and hidden states, ŷ  is a 3D tensor of shape (ny,m,Ty)
#ny: number of units in the vector representing the prediction
#m: number of examples in a mini-batch
#Ty: number of time steps in the prediction
#For a single time step t, a 2D slice ŷ ⟨t⟩ has shape (ny,m)
#In the code, the variable names are:
#y_pred: ŷ 
#yt_pred: ŷ ⟨t⟩



# pakcage
import numpy as np
from rnn_utils import *
from public_tests import *


# RNN.png

###########################################
# exercise 1 - rnn_cell_forward
#Instructions:

#Compute the hidden state with tanh activation: a⟨t⟩=tanh(Waaa⟨t−1⟩+Waxx⟨t⟩+ba)
#Using your new hidden state a⟨t⟩, compute the prediction ŷ ⟨t⟩=softmax(Wyaa⟨t⟩+by). (The function softmax is provided)
#Store (a⟨t⟩,a⟨t−1⟩,x⟨t⟩,parameters) in a cache
#Return a⟨t⟩ , ŷ ⟨t⟩ and cache
#Additional Hints
#A little more information on numpy.tanh
#In this assignment, there's an existing softmax function for you to use. It's located in the file 'rnn_utils.py' and has already been imported.
#For matrix multiplication, use numpy.dot



#Input with nx number of units
#For a single time step of a single input example, x(i)⟨t⟩ is a one-dimensional input vector
#Using language as an example, a language with a 5000-word vocabulary could be one-hot encoded into a vector that has 5000 units. So x(i)⟨t⟩ would have the shape (5000,)
#The notation nx is used here to denote the number of units in a single time step of a single training example

#Time steps of size Tx
#A recurrent neural network has multiple time steps, which you'll index with t.
#In the lessons, you saw a single training example x(i) consisting of multiple time steps Tx. In this notebook, Tx will denote the number of timesteps in the longest sequence.

#Batches of size m
#Let's say we have mini-batches, each with 20 training examples
#To benefit from vectorization, you'll stack 20 columns of x(i) examples
#For example, this tensor has the shape (5000,20,10)
#You'll use m to denote the number of training examples




# ? need to think why we need tanh function

###########################################

def rnn_cell_forward(xt, a_prev, parameters):
    """
    Implements a single forward step of the RNN-cell as described in Figure (2)

    Arguments:
    xt -- your input data at timestep "t", numpy array of shape (n_x, m).
    a_prev -- Hidden state at timestep "t-1", numpy array of shape (n_a, m)
    parameters -- python dictionary containing:
                        Wax -- Weight matrix multiplying the input, numpy array of shape (n_a, n_x)
                        Waa -- Weight matrix multiplying the hidden state, numpy array of shape (n_a, n_a)
                        Wya -- Weight matrix relating the hidden-state to the output, numpy array of shape (n_y, n_a)
                        ba --  Bias, numpy array of shape (n_a, 1)
                        by -- Bias relating the hidden-state to the output, numpy array of shape (n_y, 1)
    Returns:
    a_next -- next hidden state, of shape (n_a, m)
    yt_pred -- prediction at timestep "t", numpy array of shape (n_y, m)
    cache -- tuple of values needed for the backward pass, contains (a_next, a_prev, xt, parameters)
    """
    
    # Retrieve parameters from "parameters"
    Wax = parameters["Wax"]
    Waa = parameters["Waa"]
    Wya = parameters["Wya"]
    ba = parameters["ba"]
    by = parameters["by"]
    
    ### START CODE HERE ### (≈2 lines)
    # compute next activation state using the formula given above
    a_next = np.tanh(np.dot(Waa,a_prev) + np.dot(Wax,xt) + ba)
    # compute output of the current cell using the formula given above
    yt_pred = softmax( np.dot(Wya,a_next) + by)
    ### END CODE HERE ###
    
    # store values you need for backward propagation in cache
    cache = (a_next, a_prev, xt, parameters)
    
    return a_next, yt_pred, cache
  
  
 # calling rnn_cell_forward
p.random.seed(1)
xt_tmp = np.random.randn(3, 10)
a_prev_tmp = np.random.randn(5, 10)
parameters_tmp = {}
parameters_tmp['Waa'] = np.random.randn(5, 5)
parameters_tmp['Wax'] = np.random.randn(5, 3)
parameters_tmp['Wya'] = np.random.randn(2, 5)
parameters_tmp['ba'] = np.random.randn(5, 1)
parameters_tmp['by'] = np.random.randn(2, 1)

a_next_tmp, yt_pred_tmp, cache_tmp = rnn_cell_forward(xt_tmp, a_prev_tmp, parameters_tmp)
print("a_next[4] = \n", a_next_tmp[4])
print("a_next.shape = \n", a_next_tmp.shape)
print("yt_pred[1] =\n", yt_pred_tmp[1])
print("yt_pred.shape = \n", yt_pred_tmp.shape)

# UNIT TESTS
rnn_cell_forward_tests(rnn_cell_forward)


# rnn_seq.png
###########################################
#Exercise 2 - rnn_forward
#Implement the forward propagation of the RNN described in Figure 3.

#Instructions:

#Create a 3D array of zeros, a of shape (na,m,Tx) that will store all the hidden states computed by the RNN
#Create a 3D array of zeros, ŷ , of shape (ny,m,Tx) that will store the predictions
#Note that in this case, Ty=Tx (the prediction and input have the same number of time steps)
#Initialize the 2D hidden state a_next by setting it equal to the initial hidden state, a0
#At each time step t:
#Get x⟨t⟩, which is a 2D slice of x for a single time step t
#x⟨t⟩ has shape (nx,m)
#x has shape (nx,m,Tx)
#Update the 2D hidden state a⟨t⟩ (variable name a_next), the prediction ŷ ⟨t⟩ and the cache by running rnn_cell_forward
#a⟨t⟩ has shape (na,m)
#Store the 2D hidden state in the 3D tensor a, at the tth position
#a has shape (na,m,Tx)
#Store the 2D ŷ ⟨t⟩ prediction (variable name yt_pred) in the 3D tensor ŷ pred at the tth position
#ŷ ⟨t⟩ has shape (ny,m)
#ŷ  has shape (ny,m,Tx)
#Append the cache to the list of caches
#Return the 3D tensor a and ŷ , as well as the list of caches
#Additional Hints
#Some helpful documentation on np.zeros
#If you have a 3 dimensional numpy array and are indexing by its third dimension, you can use array slicing like this: var_name[:,:,i]
  
###########################################
def rnn_forward(x, a0, parameters):
    """
    Implement the forward propagation of the recurrent neural network described in Figure (3).

    Arguments:
    x -- Input data for every time-step, of shape (n_x, m, T_x).
    a0 -- Initial hidden state, of shape (n_a, m)
    parameters -- python dictionary containing:
                        Waa -- Weight matrix multiplying the hidden state, numpy array of shape (n_a, n_a)
                        Wax -- Weight matrix multiplying the input, numpy array of shape (n_a, n_x)
                        Wya -- Weight matrix relating the hidden-state to the output, numpy array of shape (n_y, n_a)
                        ba --  Bias numpy array of shape (n_a, 1)
                        by -- Bias relating the hidden-state to the output, numpy array of shape (n_y, 1)

    Returns:
    a -- Hidden states for every time-step, numpy array of shape (n_a, m, T_x)
    y_pred -- Predictions for every time-step, numpy array of shape (n_y, m, T_x)
    caches -- tuple of values needed for the backward pass, contains (list of caches, x)
    """
    
    # Initialize "caches" which will contain the list of all caches
    caches = []
    
    # Retrieve dimensions from shapes of x and parameters["Wya"]
    n_x, m, T_x = x.shape
    n_y, n_a = parameters["Wya"].shape
    
    ### START CODE HERE ###
    
    # initialize "a" and "y_pred" with zeros (≈2 lines)
    a = np.zeros((n_a, m, T_x))
    y_pred = np.zeros((n_y, m, T_x))
    
    # Initialize a_next (≈1 line)
    a_next = a0
    
    # loop over all time-steps
    for t in range(T_x):
        # Update next hidden state, compute the prediction, get the cache (≈1 line)
        a_next, yt_pred, cache = rnn_cell_forward(x[:,:,t], a_next, parameters)
        # Save the value of the new "next" hidden state in a (≈1 line)
        a[:,:,t] = a_next
        # Save the value of the prediction in y (≈1 line)
        y_pred[:,:,t] = yt_pred
        # Append "cache" to "caches" (≈1 line)
        caches.append(cache)
    ### END CODE HERE ###
    
    # store values needed for backward propagation in cache
    caches = (caches, x)

    return a, y_pred, caches

# calling rnn_forward
np.random.seed(1)
x_tmp = np.random.randn(3, 10, 4)
a0_tmp = np.random.randn(5, 10)
parameters_tmp = {}
parameters_tmp['Waa'] = np.random.randn(5, 5)
parameters_tmp['Wax'] = np.random.randn(5, 3)
parameters_tmp['Wya'] = np.random.randn(2, 5)
parameters_tmp['ba'] = np.random.randn(5, 1)
parameters_tmp['by'] = np.random.randn(2, 1)

a_tmp, y_pred_tmp, caches_tmp = rnn_forward(x_tmp, a0_tmp, parameters_tmp)
print("a[4][1] = \n", a_tmp[4][1])
print("a.shape = \n", a_tmp.shape)
print("y_pred[1][3] =\n", y_pred_tmp[1][3])
print("y_pred.shape = \n", y_pred_tmp.shape)
print("caches[1][1][3] =\n", caches_tmp[1][1][3])
print("len(caches) = \n", len(caches_tmp))

#UNIT TEST    
rnn_forward_test(rnn_forward)

#Situations when this RNN will perform better:
#This will work well enough for some applications, but it suffers from vanishing gradients.
#The RNN works best when each output ŷ ⟨t⟩ can be estimated using "local" context.
#"Local" context refers to information that is close to the prediction's time step t.
#More formally, local context refers to inputs x⟨t′⟩ and predictions ŷ ⟨t⟩ where t′ is close to t.
#What you should remember:

#The recurrent neural network, or RNN, is essentially the repeated use of a single cell.
#A basic RNN reads inputs one at a time, and remembers information through the hidden layer activations (hidden states) that are passed from one time step to the next.
#The time step dimension determines how many times to re-use the RNN cell
#Each cell takes two inputs at each time step:
#The hidden state from the previous cell
#The current time step's input data
#Each cell has two outputs at each time step:
#A hidden state
#A prediction </font>
