import numpy as np
from load_auto import load_auto
import matplotlib.pyplot as plt
import math

def initialize_parameters(observation_dimension):
    # observation_dimension: number of features taken into consideration of the input
    # returns weights as a vector and offset as a scalar
    weights = np.zeros((observation_dimension, 1))
    offset_b = 0
    return weights, offset_b

def model_forward(train_dataset, weights, offset_b):
    # train_dataset: input data points
    # weights and offset_b: model parameters
    # returns the output predictions as a vector corresponding to each input data point
    number_observations = np.size(train_dataset, axis = 1)
    predictions = np.zeros((1, number_observations))
    for observation in range(0, number_observations):
        with np.errstate(over='raise', invalid='raise'):
            try:
                predictions[0, observation] = weights.T @ train_dataset[:, observation] + offset_b
            except:
                predictions[0, observation] = np.inf
    return predictions

def compute_cost(predictions, train_labels):
    # predictions: computed output values
    # train_labels: true output values (ground truth)
    # returns the cost function value
    number_observations = np.size(predictions, axis = 1)
    sum = 0
    with np.errstate(over='raise', invalid='raise'):
        try:
            for observation in range(0, number_observations):
                sum += (train_labels[observation, 0] - predictions[0, observation])**2
        except:
            return np.inf
    return sum/number_observations

def model_backward(observation_dimension, train_dataset, predictions, train_labels):
    # observation_dimension: number of features taken into consideration of the input
    # train_dataset: input data points
    # predictions: computed output values
    # train_labels: true output values (ground truth)
    # returns the gradient of the cost function with respect to all parameters
    number_observations = np.size(train_dataset, axis = 1)
    sum_weights = np.zeros((observation_dimension, 1))
    sum_offset = 0
    for observation in range(0, number_observations):
        diff = predictions[0, observation] - train_labels[observation, 0]
        with np.errstate(over='raise', invalid='raise'):
            try:
                sum_weights += train_dataset[:, observation].reshape(observation_dimension,-1) * diff
                sum_offset += diff
            except:
                return np.full(sum_weights.shape, np.inf), np.inf
    gradient_weights = sum_weights * (2/number_observations)
    gradient_offset = sum_offset * (2/number_observations)
    return gradient_weights, gradient_offset

def update_parameters(weights, offset_b, gradient_weights, gradient_offset, learning_rate):
    # weights and offset_b: parameters computed (or initialised) in this iteration
    # gradient_weights and gradient_offset: gradients of the cost function
    # learning_rate: step size
    # returns the updated parameters for the next iteration
    updated_weights = weights - (learning_rate * gradient_weights)
    updated_offset = offset_b - (learning_rate * gradient_offset)
    return updated_weights, updated_offset

def predict(train_dataset, weights, offset_b):
    return model_forward(train_dataset, weights, offset_b)

def train_linear_model(train_dataset, train_labels, number_iterations, learning_rate):
    # train_dataset: input data points
    # train_labels: true output values (ground truth)
    # number_iterations and learning_rate: user-defined hyperparameters
    # returns the model parameters and cost function values as a vector
    cost = []
    observation_dimension = np.size(train_dataset, axis = 0)
    weights, offset_b = initialize_parameters(observation_dimension)
    while number_iterations > 0:
        predictions = predict(train_dataset, weights, offset_b)
        cost.append(compute_cost(predictions, train_labels))
        gradient_weights, gradient_offset = model_backward(observation_dimension, train_dataset, predictions, train_labels)
        weights, offset_b = update_parameters(weights, offset_b, gradient_weights, gradient_offset, learning_rate)
        number_iterations -= 1
    return weights, offset_b, cost

def plotting_cost_iteration(learning_rates, cost_consolidated):
    for counter in range(0, cost_consolidated.shape[0]):
        plt.plot(np.arange(start=1, stop = (cost_consolidated.shape[1] + 1), step= 1), cost_consolidated[counter,:], label=r'$\alpha = $' + str(learning_rates[counter]))
    plt.xlabel('Iteration')
    plt.ylabel('Cost')
    plt.title('Cost per Iteration')
    plt.ylim(0,720)
    plt.legend()
    plt.show()

def plotting_horsepower_mpg(train_dataset, train_labels, weights, offset_b):
    plt.scatter(train_dataset[0,:], train_labels[:,0], label='Data points')
    plt.plot(train_dataset[0,:], np.array(train_dataset[0,:]*weights + offset_b).reshape(train_labels.shape),'r-', label='Linear Regression')
    plt.xlabel('(normalised) Horsepower')
    plt.ylabel('MPG')
    plt.title('MPG vs (normalised) Horsepower')
    plt.legend()
    plt.show()

PATH_DATASET = '/Users/marcellovendruscolo/Documents/vscode-workspace/DeepLearningForImageAnalysis/linearRegression_gradientDescent/Auto.csv'
train_dataset, train_labels = load_auto(PATH_DATASET)
train_dataset = np.array(train_dataset)
non_normalised_dataset = np.array(np.transpose(train_dataset))
non_normalised_horsepower = non_normalised_dataset[2,:].reshape(1,-1)
train_labels = np.array(train_labels)

mean = np.mean(train_dataset, axis=0)
sd = np.std(train_dataset, axis=0)
for col in range(0, train_dataset.shape[1]):
    train_dataset[:,col] = (train_dataset[:,col] - mean[col])/sd[col]
normalised_dataset = np.transpose(train_dataset)
horsepower_dataset = normalised_dataset[2,:].reshape(1,-1)

# Exercise 1.4.1 and Exercise 1.4.2:
# learning_rate = 0.1
# number_iterations = 1000

# print('\nChoice of input dataset: (i) Only horsepower feature.')
# weights, offset_b, cost_function_value = train_linear_model(horsepower_dataset, train_labels, number_iterations, learning_rate)
# print('Number of iterations: ' +str(number_iterations))
# print('Learning rate: ' +str(learning_rate))
# print('Cost function value: ' +str(cost_function_value[len(cost_function_value) - 1]))
# print('Weights: ' +str(weights))
# print('Offset: ' +str(offset_b))

# print('\nChoice of input dataset: (ii) All features except name.')
# weights, offset_b, cost_function_value = train_linear_model(normalised_dataset, train_labels, number_iterations, learning_rate)
# print('Number of iterations: ' +str(number_iterations))
# print('Learning rate: ' +str(learning_rate))
# print('Cost function value: ' +str(cost_function_value[len(cost_function_value) - 1]))
# print('Weights: ' +str(weights))
# print('Offset: ' +str(offset_b) + '\n')

# Exercise 1.4.3:
# learning_rates = [1, 1e-1, 1e-2, 1e-3, 1e-4]
# number_iterations = 1000
# cost_consolidated = np.ndarray(shape=(len(learning_rates), number_iterations))

# for counter in range(0, len(learning_rates)):
#     weights, offset_b, cost_consolidated[counter,:] = train_linear_model(normalised_dataset, train_labels, number_iterations, learning_rates[counter])

# plotting_cost_iteration(learning_rates, cost_consolidated)

# Exercise 1.4.4:
# learning_rate = [1, 1e-1, 1e-2, 1e-3, 1e-4]
# number_iterations = 1000
# cost_consolidated = np.ndarray(shape=(len(learning_rate), number_iterations))
# for counter in range(0, len(learning_rate)):
#     weights, offset_b, cost_consolidated[counter,:] = train_linear_model(non_normalised_dataset, train_labels, number_iterations, learning_rate[counter])
# plotting_cost_iteration(learning_rate, cost_consolidated)

# Exercise 1.4.5:
# learning_rate = 0.1
# number_iterations = 1000
# weights, offset_b, cost_function_value = train_linear_model(horsepower_dataset, train_labels, number_iterations, learning_rate)
# plotting_horsepower_mpg(horsepower_dataset, train_labels, weights, offset_b)
[((5298, 5321), 'load_auto.load_auto', 'load_auto', (['PATH_DATASET'], {}), '(PATH_DATASET)\n', (5307, 5321), False, 'from load_auto import load_auto\n'), ((5338, 5361), 'numpy.array', 'np.array', (['train_dataset'], {}), '(train_dataset)\n', (5346, 5361), True, 'import numpy as np\n'), ((5510, 5532), 'numpy.array', 'np.array', (['train_labels'], {}), '(train_labels)\n', (5518, 5532), True, 'import numpy as np\n'), ((5541, 5571), 'numpy.mean', 'np.mean', (['train_dataset'], {'axis': '(0)'}), '(train_dataset, axis=0)\n', (5548, 5571), True, 'import numpy as np\n'), ((5577, 5606), 'numpy.std', 'np.std', (['train_dataset'], {'axis': '(0)'}), '(train_dataset, axis=0)\n', (5583, 5606), True, 'import numpy as np\n'), ((5743, 5770), 'numpy.transpose', 'np.transpose', (['train_dataset'], {}), '(train_dataset)\n', (5755, 5770), True, 'import numpy as np\n'), ((303, 339), 'numpy.zeros', 'np.zeros', (['(observation_dimension, 1)'], {}), '((observation_dimension, 1))\n', (311, 339), True, 'import numpy as np\n'), ((638, 668), 'numpy.size', 'np.size', (['train_dataset'], {'axis': '(1)'}), '(train_dataset, axis=1)\n', (645, 668), True, 'import numpy as np\n'), ((689, 723), 'numpy.zeros', 'np.zeros', (['(1, number_observations)'], {}), '((1, number_observations))\n', (697, 723), True, 'import numpy as np\n'), ((1253, 1281), 'numpy.size', 'np.size', (['predictions'], {'axis': '(1)'}), '(predictions, axis=1)\n', (1260, 1281), True, 'import numpy as np\n'), ((2000, 2030), 'numpy.size', 'np.size', (['train_dataset'], {'axis': '(1)'}), '(train_dataset, axis=1)\n', (2007, 2030), True, 'import numpy as np\n'), ((2051, 2087), 'numpy.zeros', 'np.zeros', (['(observation_dimension, 1)'], {}), '((observation_dimension, 1))\n', (2059, 2087), True, 'import numpy as np\n'), ((3689, 3719), 'numpy.size', 'np.size', (['train_dataset'], {'axis': '(0)'}), '(train_dataset, axis=0)\n', (3696, 3719), True, 'import numpy as np\n'), ((4550, 4573), 'matplotlib.pyplot.xlabel', 'plt.xlabel', (['"""Iteration"""'], {}), "('Iteration')\n", (4560, 4573), True, 'import matplotlib.pyplot as plt\n'), ((4578, 4596), 'matplotlib.pyplot.ylabel', 'plt.ylabel', (['"""Cost"""'], {}), "('Cost')\n", (4588, 4596), True, 'import matplotlib.pyplot as plt\n'), ((4601, 4632), 'matplotlib.pyplot.title', 'plt.title', (['"""Cost per Iteration"""'], {}), "('Cost per Iteration')\n", (4610, 4632), True, 'import matplotlib.pyplot as plt\n'), ((4637, 4653), 'matplotlib.pyplot.ylim', 'plt.ylim', (['(0)', '(720)'], {}), '(0, 720)\n', (4645, 4653), True, 'import matplotlib.pyplot as plt\n'), ((4657, 4669), 'matplotlib.pyplot.legend', 'plt.legend', ([], {}), '()\n', (4667, 4669), True, 'import matplotlib.pyplot as plt\n'), ((4674, 4684), 'matplotlib.pyplot.show', 'plt.show', ([], {}), '()\n', (4682, 4684), True, 'import matplotlib.pyplot as plt\n'), ((4767, 4844), 'matplotlib.pyplot.scatter', 'plt.scatter', (['train_dataset[(0), :]', 'train_labels[:, (0)]'], {'label': '"""Data points"""'}), "(train_dataset[(0), :], train_labels[:, (0)], label='Data points')\n", (4778, 4844), True, 'import matplotlib.pyplot as plt\n'), ((4985, 5022), 'matplotlib.pyplot.xlabel', 'plt.xlabel', (['"""(normalised) Horsepower"""'], {}), "('(normalised) Horsepower')\n", (4995, 5022), True, 'import matplotlib.pyplot as plt\n'), ((5027, 5044), 'matplotlib.pyplot.ylabel', 'plt.ylabel', (['"""MPG"""'], {}), "('MPG')\n", (5037, 5044), True, 'import matplotlib.pyplot as plt\n'), ((5049, 5092), 'matplotlib.pyplot.title', 'plt.title', (['"""MPG vs (normalised) Horsepower"""'], {}), "('MPG vs (normalised) Horsepower')\n", (5058, 5092), True, 'import matplotlib.pyplot as plt\n'), ((5097, 5109), 'matplotlib.pyplot.legend', 'plt.legend', ([], {}), '()\n', (5107, 5109), True, 'import matplotlib.pyplot as plt\n'), ((5114, 5124), 'matplotlib.pyplot.show', 'plt.show', ([], {}), '()\n', (5122, 5124), True, 'import matplotlib.pyplot as plt\n'), ((5396, 5423), 'numpy.transpose', 'np.transpose', (['train_dataset'], {}), '(train_dataset)\n', (5408, 5423), True, 'import numpy as np\n'), ((1305, 1347), 'numpy.errstate', 'np.errstate', ([], {'over': '"""raise"""', 'invalid': '"""raise"""'}), "(over='raise', invalid='raise')\n", (1316, 1347), True, 'import numpy as np\n'), ((791, 833), 'numpy.errstate', 'np.errstate', ([], {'over': '"""raise"""', 'invalid': '"""raise"""'}), "(over='raise', invalid='raise')\n", (802, 833), True, 'import numpy as np\n'), ((2248, 2290), 'numpy.errstate', 'np.errstate', ([], {'over': '"""raise"""', 'invalid': '"""raise"""'}), "(over='raise', invalid='raise')\n", (2259, 2290), True, 'import numpy as np\n'), ((4393, 4456), 'numpy.arange', 'np.arange', ([], {'start': '(1)', 'stop': '(cost_consolidated.shape[1] + 1)', 'step': '(1)'}), '(start=1, stop=cost_consolidated.shape[1] + 1, step=1)\n', (4402, 4456), True, 'import numpy as np\n'), ((4872, 4924), 'numpy.array', 'np.array', (['(train_dataset[(0), :] * weights + offset_b)'], {}), '(train_dataset[(0), :] * weights + offset_b)\n', (4880, 4924), True, 'import numpy as np\n'), ((2489, 2523), 'numpy.full', 'np.full', (['sum_weights.shape', 'np.inf'], {}), '(sum_weights.shape, np.inf)\n', (2496, 2523), True, 'import numpy as np\n')]