# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     name: python3
# ---

# + [markdown] id="view-in-github" colab_type="text"
# <a href="https://colab.research.google.com/github/swap-10/Dense-DNN-From-Scratch/blob/main/DenseDNNScratch.ipynb" target="_parent"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/></a>

# + [markdown] id="sx-MqOmAqjMR"
# # Fully Connected Deep Neural Network with NumPy

# + [markdown] id="PT-veG8Jqsyn"
# # Constructing the Neural Net

# + id="qJ6XSCYxCsL0"
import numpy as np
import matplotlib.pyplot as plt
import h5py
import scipy
import sys
import glob
import h5py
import cv2
import csv

np.random.seed()


# + id="w7p3oYdqHEb4"
def sigmoid(Z):

  s = 1 / np.exp(Z)
  s = 1 / (1 + s) 
  cache_act = Z

  return s, cache_act


def sigmoid_derivative(Z):

  d_sigmoid = np.multiply(Z, (1 - Z))

  return d_sigmoid


def sigmoid_backprop(dA, cache_act):
  
  Z = cache_act
  dZ = np.multiply(dA, sigmoid_derivative(Z))

  return dZ



def relu(Z):

  r = np.maximum(0, Z)
  cache_act = Z

  return r, cache_act


def relu_derivative(Z):

  d_relu = Z
  d_relu[d_relu<=0] = 0
  d_relu[d_relu>0] = 1

  return d_relu


def relu_backprop(dA, cache_act):
  
  Z = cache_act
  dZ = np.multiply(dA, relu_derivative(Z))

  return dZ



def softmax(Z):
  
  cache_act = Z
  s = np.exp(Z) / (np.sum(np.exp(Z), axis=0))

  return s, cache_act



# + colab={"base_uri": "https://localhost:8080/"} id="bcF-BUFAPiWy" outputId="aa192c8d-6511-4cbb-927e-ac60533e0543"
a = [[1, 2],
     [3, 4],
     [5, 6]]

b = [[11, 12],
      [13, 14],
      [15, 16]]

print(np.multiply(a,b))


# + id="mpiMcZp1ELmW"
def init_params(layer_dims):
  
  params = {}
  L = len(layer_dims)

  for i in range(1, L):
    params[f"W%d" %i] = (np.random.randn(layer_dims[i], layer_dims[i-1]) + (1e-9)) * 0.03
    params[f"b%d" %i] = np.zeros((layer_dims[i],1))
  
  return params


# + id="q50TbswZFqZ0"
def forward_nonact(A, W, b):
  Z = np.dot(W, A) + b
  cache = (A, W, b)

  return Z, cache

def forward_act(A_pre, W, b, activation):
  
  if activation == "sigmoid":
    Z, cache_nonact = forward_nonact(A_pre, W, b)
    A, cache_act = sigmoid(Z)
  
  elif activation == "softmax":
    Z, cache_nonact = forward_nonact(A_pre, W, b)
    A, cache_act = softmax(Z)

  elif activation == "relu":
    Z, cache_nonact = forward_nonact(A_pre, W, b)
    A, cache_act = relu(Z)

  cache = (cache_nonact, cache_act)

  return A, cache


# + id="5fQMd2T5IRg0"
def forward_prop(X, params):

  L = len(params) // 2
  cache_accumulate = []
  A = X  # First layer

  for i in range(1, L):
    A_pre = A
    A, cache = forward_act(
                          A_pre,
                          params[f"W%d" %i], params[f"b%d" %i],
                          activation="relu"
                          )
    cache_accumulate.append(cache)

  A_L, cache = forward_act(
                           A,
                           params[f"W%d" %L],
                           params[f"b%d" %L],
                           activation="softmax"
                          )
  cache_accumulate.append(cache)

  return A_L, cache_accumulate



# + id="WPgbEF2eKReg"
def compute_cost(A_L, Y):

  m = Y.shape[1]  # Assuming Y is in form of (1, num_of_training_examples)

  cost = (-1/m) * np.sum(np.multiply(Y, np.log(A_L)), axis=1, keepdims=True)   # Softmax cost
  cost = np.sum(cost)
  cost = np.squeeze(cost)

  return cost


# + id="gZW8MRMxLsci"
def backward_nonact(dZ, cache_nonact):

  A_pre, W, b = cache_nonact
  m = A_pre.shape[1]  # All A.shape[1] will have same value (m)

  dW = (1/m) * np.dot(dZ, A_pre.T)
  db = (1/m) *  np.sum(dZ, axis=1, keepdims=True)
  dA_pre = np.dot(W.T, dZ)

  return dA_pre, dW, db


def backward_act(dA, cache, activation):

  cache_nonact, cache_act = cache

  if activation == "relu":
    dZ = relu_backprop(dA, cache_act)
    dA_pre, dW, db = backward_nonact(dZ, cache_nonact)

  elif activation == "sigmoid":
    dZ = sigmoid_backprop(dA, cache_act)
    dA_pre, dW, db = backward_nonact(dZ, cache_nonact)
  
  # Softmax is only used in the final layer and it's required derivative: derivative of loss w.r.t Z is much easier and more convenient to compute
  # For this reason it is computed seperately in the beginning of the backward propagation step and not here

  return dA_pre, dW, db



# + id="-2PXQlnIupJl"
def backward_prop(A_L, Y, cache_accumulate):

  L = len(cache_accumulate)
  m = A_L.shape[1]
  Y = Y.reshape(A_L.shape)
  grads = {}

  dA_L = - (np.divide(Y, A_L))  # Softmax derivative of Y wrt activated output

  cache_cur = cache_accumulate[-1]
  dZ = A_L - Y  # This combined with the next line gives the backward_act() for the last layer (softmax)
  dA_pre_cur, dW_cur, db_cur = backward_nonact(dZ, cache_cur[0])  # cache_cur[0] is cache_nonact.
  grads[f"dA%d" %(L-1)] = dA_pre_cur
  grads[f"dW%d" %L] = dW_cur
  grads[f"db%d" %L] = db_cur

  for i in range(L-2, -1, -1):

    cache_cur = cache_accumulate[i]
    dA_pre_cur, dW_cur, db_cur = backward_act(dA_pre_cur, cache_cur, activation="relu")
    grads[f"dA%d" %i] = dA_pre_cur
    grads[f"dW%d" %(i+1)] = dW_cur
    grads[f"db%d" %(i+1)] = db_cur

  return grads



# + id="v76TCDib0beA"
def update_params(params, grads, learning_rate):

  parameters = params.copy()
  L = len(parameters) // 2

  for i in range(L):

    parameters[f"W%d" %(i+1)] = parameters[f"W%d" %(i+1)] - (learning_rate*grads[f"dW%d" %(i+1)])
    parameters[f"b%d" %(i+1)] = parameters[f"b%d" %(i+1)] - (learning_rate*grads[f"db%d" %(i+1)])

  return parameters


# + [markdown] id="b_fS0Nw91sqc"
#
#
# ---
# ---
# ---

# + [markdown] id="JcRbMBO91ybV"
# <br>
# <br>
# <br>
# <br>

# + [markdown] id="AGYMzjW6rFU3"
# [Dataset used](https://https://data.mendeley.com/datasets/4drtyfjtfy/1)

# + [markdown] id="0SOt0bGzq8OI"
# # Using the Neural Net

# + [markdown] id="G_8tB7HNrCSh"
# # Data Preparation

# + [markdown] id="wYn6vSFgrEnk"
# <h2> Careful! <h2>

# + id="4hbgSBXO43IV" language="bash"
# rm -rf downloaddata
# rm -rf dataset2

# + id="fZnTy2iI2COs" colab={"base_uri": "https://localhost:8080/"} outputId="4ce42be6-fd86-4399-d34e-c94a3f43d912" language="bash"
# mkdir downloaddata
# wget -O ./downloaddata/images.zip https://data.mendeley.com/public-files/datasets/4drtyfjtfy/files/a03e6097-f7fb-4e1a-9c6a-8923c6a0d3e0/file_downloaded | head
# unzip ./downloaddata/images.zip -d ./downloaddata

# + id="OlI6hBLWSB85" language="bash"
# touch filenames.txt
# ls ./downloaddata/dataset2/ > filenames.txt

# + id="gZ_1her5HutL" language="bash"
# a=1
# for i in ./downloaddata/dataset2/cloudy*.jpg;
# do
#   new=$(printf "./downloaddata/dataset2/cloudy%04d.jpg" "$a") #04 pad to length of 4
#   mv -i -- "$i" "$new"
#   let a=a+1
# done
#
# a=1
# for i in ./downloaddata/dataset2/rain*.jpg;
# do
#   new=$(printf "./downloaddata/dataset2/rain%04d.jpg" "$a") #04 pad to length of 4
#   mv -i -- "$i" "$new"
#   let a=a+1
# done
#
# a=1
# for i in ./downloaddata/dataset2/shine*.jpg;
# do
#   new=$(printf "./downloaddata/dataset2/shine%04d.jpg" "$a") #04 pad to length of 4
#   mv -i -- "$i" "$new"
#   let a=a+1
# done
#
# a=1
# for i in ./downloaddata/dataset2/sunrise*.jpg;
# do
#   new=$(printf "./downloaddata/dataset2/sunrise%04d.jpg" "$a") #04 pad to length of 4
#   mv -i -- "$i" "$new"
#   let a=a+1
# done
#

# + id="r36V1XW9czSt" language="bash"
# touch filenames.csv
# cd downloaddata/dataset2/
# printf "%s\n" * > ../../filenames.csv
# cd ../../
# shuf filenames.csv > filenames_shuf.csv
# head -n 901 filenames_shuf.csv > train_names.csv
# tail -n 225 filenames_shuf.csv > test_names.csv

# + [markdown] id="9GryzbWjmMRH"
# csv files containing names of shuffled data -- done

# + colab={"base_uri": "https://localhost:8080/"} id="wMEf524o8yc3" outputId="32b77f50-15fb-4f69-d2fc-3f7bd85085bf"
IMG_WIDTH = 128
IMG_HEIGHT = 128

hf = 'import_images.h5'

with open('train_names.csv', newline='\n') as f:
    reader = csv.reader(f)
    data = list(reader)

print(data[:100])
nfiles = len(data)
print(f'count of image files nfiles={nfiles}')
data = enumerate(data)

# resize all images and load into a single dataset
with h5py.File(hf,'w') as  h5f:
    img_ds = h5f.create_dataset('images_train',shape=(nfiles, IMG_WIDTH, IMG_HEIGHT,3), dtype=int)
    img_labels = h5f.create_dataset('labels_train', shape=(nfiles, 1), dtype=int)
    for (cnt, ifile) in data:
        ifile = "./downloaddata/dataset2/" + ifile[0]
        img = cv2.imread(ifile, cv2.IMREAD_COLOR)
        if img is None:
          continue
        img_resize = cv2.resize( img, (IMG_WIDTH, IMG_HEIGHT))
        img_ds[cnt:cnt+1:,:,:] = img_resize

        if "cloudy" in str(ifile):
          classnum = 0
        elif "rain" in str(ifile):
          classnum = 1
        elif "shine" in str(ifile):
          classnum = 2
        elif "sunrise" in str(ifile):
          classnum = 3
        img_labels[cnt, :] = classnum
        
classnames = ["cloudy", "rain", "shine", "sunrise"]

# + colab={"base_uri": "https://localhost:8080/", "height": 475} id="1rJ7sp1kJEYr" outputId="a84307f5-b8dd-4623-e55a-6228ced5715b"
hfile = h5py.File(hf, 'r')
n = hfile.get('images_train')
l = hfile.get('labels_train')
print(type(n))
plt.figure(figsize=(10,10))
for i, img in enumerate(n[:10,:,:,:]):
  plt.subplot(2,5,i+1)
  plt.title(classnames[np.squeeze(l[i,:])])
  plt.imshow(img)
hfile.close()

# + colab={"base_uri": "https://localhost:8080/"} id="oJXgvEz9rhoo" outputId="a9592ffc-1efe-4882-b165-220f31b20337"
# https://stackoverflow.com/questions/28170623/how-to-read-hdf5-files-in-python

hfile.close()
hfile = h5py.File(hf, 'r')
i=0
for key in hfile.keys():
  print(key)

group1 = hfile['images_train']
train_images = group1[:]
train_images = np.array(train_images)
print(type(train_images))
print(train_images.shape)
train_images = train_images.reshape(IMG_WIDTH*IMG_HEIGHT*3, -1)
print(train_images.shape)
print(train_images)

group2 = hfile['labels_train']
train_labels = group2[:]
train_labels = np.array(train_labels)
print(type(train_labels))
print(train_labels.shape)
train_labels = train_labels.reshape(1, 901)
print(train_labels.shape)
print(train_labels)
hfile.close()

# + colab={"base_uri": "https://localhost:8080/"} id="ouvp-yNO52-8" outputId="9367a9a1-0cc5-4c0f-9327-6d26cb66a9bb"
layer_dims = [49152, 50, 20, 5, 4]
train_labels = np.multiply(train_labels, np.ones((4, train_labels.shape[1])))
print(train_labels.shape)
print(type(train_labels))
print(train_labels)

# + colab={"base_uri": "https://localhost:8080/"} id="UN9I76skFaLJ" outputId="9b3d847b-b27d-4540-9270-be3cb7b5ed8c"
for i in range(train_labels.shape[0]):
  train_labels[i] = [(i==val) for val in train_labels[i]]
print(train_labels.shape)
print(train_labels[0])


# + id="-UA8k7q37Knu"
def model(X, Y, layer_dims, learning_rate=0.001, num_iters=3000, print_cost=False):
  np.random.seed(0)
  costs = []

  params = init_params(layer_dims)

  for i in range(0, num_iters):

    A_L, cache_accumulate = forward_prop(X, params)

    cost = compute_cost(A_L, Y)

    grads = backward_prop(A_L, Y, cache_accumulate)

    params = update_params(params, grads, learning_rate)

    if print_cost and i % 10 == 0 or i == num_iters - 1:
      print("Cost after iteration {}: {}".format(i+1, np.squeeze(cost)))

    if i % 100 == 0 or i == num_iters - 1:
      costs.append(cost)
    

  return params, costs

# + colab={"base_uri": "https://localhost:8080/"} id="OGR8J3q58xqb" outputId="26534e3e-fe5a-4c36-cdbc-5149e73f43a8"
params, costs = model(train_images, train_labels, layer_dims, num_iters=1, print_cost=False)

# + id="lBrmY3EXqXlQ" colab={"base_uri": "https://localhost:8080/"} outputId="3b1634c0-f907-4c1e-dcf7-8ccb567605f9"
params, costs = model(train_images, train_labels, layer_dims, learning_rate=0.002, num_iters=1500, print_cost=True)

# + id="GfYdegzWdkjf"


StarcoderdataJupyterScriptsDedupFiltered