# -*- coding: utf-8 -*-
"""hw3_4.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/1nR9Fw0b0bDqb14CJp-XsMqNqAnxQahZY
"""

# Commented out IPython magic to ensure Python compatibility.
# %matplotlib inline
import numpy as np
import matplotlib
import matplotlib.pyplot as plt
import pandas as pd
from numpy import shape
from numpy import mat
from numpy import arange

def sigmoid(x):
  return 1 / (1 + np.exp(-x))

def net_input(theta, x):
  return np.dot(x, theta)

def probability(theta, x):
  return sigmoid(net_input(theta, x))

def cost_func(x, y):
  # if transfer to matrix, there is no need to use np.sum
  total_cost = -(y * np.log(sigmoid(x)) + (1 - y) * np.log(1 - sigmoid(x)))
  return total_cost

"""create gd function"""

def gradient_descent(init, steps, learning_rate, y):
  # transfer init to matrix
  row, col = shape(init)
  init = mat(init)
  # create w vector
  xs = [np.ones((col, 1))]
  # x[m,n] * w[n,1] = y[m, 1]
  for step in range(steps):
    err = sigmoid(init * xs[-1]) - y
    xs.append(xs[-1] - learning_rate * init.T * err)
  return xs

"""create sgd function"""

def stochastic_gradient_descent(init, steps, learning_rate, y):
  row, col = shape(init)
  init = mat(init)
  xs = [np.ones((col, 1))]
  for step in range(steps):
    # use stochastic to get value y, need to generate y
    for index in range(row):
      err = sigmoid(init[index] * xs[-1]) - y[index]
      xs.append(xs[-1] - learning_rate * init[index].T * err)
  return xs

"""a. generate data points"""

standard_deviation = [[0.03,0],[0,0.03]]

mean1 = [0.5,0.5]
data1 = np.random.multivariate_normal(mean1, standard_deviation,100)
# print(data1.shape)

mean2 = [-0.5,-0.5]
data2 = np.random.multivariate_normal(mean2, standard_deviation, 100)

# cut the data set
# data1_x = data1[:, 0]
# data1_y = data1[:, 1]
# data2_x = data2[:, 0]
# data2_y = data2[:, 1]

"""b. use GD to train logisitc regression model"""

init = np.concatenate((data1, data2))
y = mat(([1] * 100 + [0] * 100)).T
gd = gradient_descent(init, 2000, 0.01, y)
sgd = stochastic_gradient_descent(init, 10, 0.1, y)

# get y index from x
temp_x = arange(-1,1,0.1)
x_node = []
y_gd_node = []
y_sgd_node = []
# print(len(temp_x))

for k in range(len(temp_x)):
  x_node.append(temp_x[k])

gd_w = gd[-1]
sgd_w = sgd[-1]

gd_y = (-gd_w[0] * temp_x) / gd_w[1]
sgd_y = (-sgd_w[0] * temp_x) / sgd_w[1]

temp_y_row, temp_y_col = gd_y.shape

for k in range(temp_y_col):
  y_gd_node.append(gd_y[0, k])
  y_sgd_node.append(sgd_y[0, k])

plt.plot(x_node, y_gd_node, c='r')
plt.plot(x_node, y_sgd_node, c='b')

print(y_gd_node)
print(y_sgd_node)

plt.ylim(-1.0, 1.0)
plt.xlim(-1.0, 1.0)

plt.scatter(data1[:, 0], data1[:, 1], c='blue')
plt.scatter(data2[:, 0], data2[:, 1], c='red')
plt.show()

"""c.get loss diagram of gd and sgd"""

cost_gd = []
cost_sgd = []

# print(type(gd_y))
# print(init.shape)
# print(y.shape)
# print(gd[0])

# transfer init to matrix so that we can perform matrix calculation directly
init = mat(init)
Y = np.squeeze(y)

for _ in gd:
  temp = init * _
  x = np.squeeze(temp).T
  cost_gd.append(np.asarray(cost_func(x, Y))[0][0])

# print(type(cost_gd))

for _ in sgd:
  temp = init * _
  x = np.squeeze(temp).T
  cost_sgd.append(np.asarray(cost_func(x, Y))[0][0])

plt.plot(range(len(cost_gd)), cost_gd, c = 'r')
plt.plot(range(len(cost_sgd)), cost_sgd, c = 'b')
plt.show()

# print(len(cost_sgd))
# print(len(cost_gd))
print('the blue line is cost result of sgd')
print('the red line is cost result of gd')