"""
## ***Libraries***
"""
# Importing Libraries(Technologies)
from math import sqrt
from pandas_datareader import data 
import matplotlib.pyplot as plt
import scipy
import pandas as pd
import datetime as dt
import urllib.request,json
import os
import numpy as np
import tensorflow as tf
from sklearn.preprocessing import MinMaxScaler
from sklearn.metrics import mean_squared_error

from keras.models import Sequential
from keras.layers import Dense,LSTM
"""
Technologies used: Google colab, Python, Json, tf keras  

APIs used: wwo-hist, api.eia.gov
"""
"""
# **Data Loading**
"""
dataframe2=pd.read_csv('../input/homesteadus-electricity-consumption/electric_consumption_till_25jun.csv')
dataframe2=dataframe2.drop('Unnamed: 0',axis=1)
fig,ax = plt.subplots()                       # Create a figure and a set of subplots.
plt.ylabel('Electricity Consumption[in MWh]') # labelling Y-axis
plt.xlabel('Date')                            # labelling X-axis
ax.plot(dataframe2['Date'],dataframe2['Consumption'])
plt.show()
"""
> You can see the periodicity so we can use time features like day,month,year.
"""
DataFrame=dataframe2.copy()
"""
# ***FEATURE SELECTION***
Statistical tests can be used to select those features that have the strongest relationship with the output variable.
"""
"""
## **Correlation Matrix with Heatmap**

Correlation states how the features are related to each other or the target variable.

Correlation can be positive (increase in one value of feature increases the value of the target variable) or negative (increase in one value of feature decreases the value of the target variable)

Heatmap makes it easy to identify which features are most related to the target variable, we will plot heatmap of correlated features using the seaborn library.
"""
import seaborn as sns
#get correlations of each features in dataset
corrmat = DataFrame.corr()
top_corr_features = corrmat.index
plt.figure(figsize=(20,20))
#plot heat map
g=sns.heatmap(DataFrame[top_corr_features].corr(),annot=True,cmap="RdYlGn")
"""
## ***Pearson Correlation Coefficient***

Pearson's Correlation Coefficient helps to find out the relationship between two quantities. It gives the measure of the strength of association between two variables. The value of Pearson's Correlation Coefficient can be between -1 to +1. 1 means that they are highly correlated and 0 means no correlation.
"""
pearsoncorr = DataFrame.corr(method='pearson')
pearsoncorr
pearsoncorr_list = pearsoncorr.values.tolist()                     # Dataframe 'pearsoncorr' with pearson correlation values among features converted to list

features = DataFrame.columns
features = features[1:-1]                                    # features of DataFrame except Date

a = list(pearsoncorr_list[-1])                              # Pearson correlation value of features vs consumption
pf = pd.DataFrame(columns=['Features','ratio'])            # Dataframe to visualise list 'a' values relation to features

print('CONSUMPTION CORRELATIONS (PEARSON) FOR HOMESTEAD')

for i in range(len(a)-1):
  d = [features[i],a[i]]
  ln = len(pf)
  pf.loc[ln] = d

pf
"""
## ***Kendall rank correlation coefficient***

It is a measure of rank correlation: the similarity of the orderings of the data when ranked by each of the quantities.

The Kendall correlation between two variables will be high when observations have a similar (or identical for a correlation of 1) rank (i.e. relative position label of the observations within the variable: 1st, 2nd, 3rd, etc.) between the two variables, and low when observations have a dissimilar (or fully different for a correlation of −1) rank between the two variables.
"""
corr = DataFrame.corr(method='kendall')
corr
DataFrame
"""
## ***R Squared Correlation***

Correlation (otherwise known as “R”) is a number between 1 and -1 where a value of +1 implies that an increase in x results in some increase in y, -1 implies that an increase in x results in a decrease in y, and 0 means that there isn’t any relationship between x and y. Like correlation, R² tells you how related two things are. However, we tend to use R² because it’s easier to interpret. R² is the percentage of variation (i.e. varies from 0 to 1) explained by the relationship between two variables.

### ***R² = (var(mean) - var(line)) / var(mean)***
"""
RConsumption = {}
h = ['Date','Consumption']
for col in DataFrame.columns:
  if( col not in h):
    slope, intercept, r_value, p_value, std_err = scipy.stats.stats.linregress(DataFrame['Consumption'], DataFrame[col])
    RConsumption[col] = r_value**2
print('DEMAND CORRELATIONS (r^2) FOR HOMESTEAD')

cons_df = pd.DataFrame(RConsumption.items(), columns=['Features', 'r**2'])
cons_df
"""
## ***Features Drop-out***
"""
DataFrame.columns
DataFrame
dataframe2 = DataFrame.copy()
dataframe2
dataframe2.columns
"""
We have extracted data from above module and store in .csv file, data until 24th june 2020
"""
dataframe2.columns
dataframe2['Day']= pd.DatetimeIndex(dataframe2['Date']).day
dataframe2['month']=pd.DatetimeIndex(dataframe2['Date']).month
dataframe2['Year']=pd.DatetimeIndex(dataframe2['Date']).year
pearsoncorr = dataframe2.corr(method='pearson')
pearsoncorr['Consumption']
f = ['Homestead_cloudcover','Homestead_visibility','Homestead_winddirDegree','Day','Year']
dataframe2=dataframe2.drop(f,axis = 1)
df.head()
cnsm=dataframe2.pop('Consumption')
dataframe2['Consumption']=cnsm
dataframe2
print(dataframe2.columns)
dataframe2.shape
"""
# ***DATA NORMALISATION***
"""
Date = dataframe2.pop('Date')           # Removing column 'Date' from dataframe2, need not to be normalized
values = dataframe2.values              #converting to np array
"""
## **MinMaxScaler**

Standardizes features by scaling each feature to a given range.

This estimator scales and translates each feature individually such that it is in the given range on the training set, i.e. between zero and one.

The standardization is given by:
```
X_std = (X - X.min(axis=0)) / (X.max(axis=0) - X.min(axis=0))
X_scaled = X_std * (max - min) + min
```



where min, max = feature_range.
"""
scaler = MinMaxScaler(feature_range=(0,1))
scaled = scaler.fit_transform(values)           # Fit to data, then transform it.
scaled.shape
dataframe2['Date'] = Date                       #adding date back
dataframe2.shape
scaled                                         #normalized values
DataFrame1 = dataframe2.copy(deep = False)
DataFrame1.head()
"""
# ***Training and Testing***
"""
 # convert series to supervised learning
def series_to_supervised(data, n_in=1, n_out=1, dropnan=True):
	n_vars = 1 if type(data) is list else data.shape[1]
	df = pd.DataFrame(data)
	cols, names = list(), list()
	# input sequence (t-n, ... t-1)
	for i in range(n_in, 0, -1):
		cols.append(df.shift(i))
		names += [('var%d(t-%d)' % (j+1, i)) for j in range(n_vars)]
	# forecast sequence (t, t+1, ... t+n)
	for i in range(0, n_out):
		cols.append(df.shift(-i))
		if i == 0:
			names += [('var%d(t)' % (j+1)) for j in range(n_vars)]
		else:
			names += [('var%d(t+%d)' % (j+1, i)) for j in range(n_vars)]
	# put it all together
	agg = pd.concat(cols, axis=1)
	agg.columns = names
	# drop rows with NaN values
	if dropnan:
		agg.dropna(inplace=True)
	return agg
# frame as supervised learning
reframed = series_to_supervised(scaled, 1, 1)
reframed.drop(reframed.columns[range(14,27)],axis=1,inplace=True)
reframed.head()
# split into train and test sets(68:32 ratio)
values = reframed.values
n_train_hours = int(len(values)*0.68)
train = values[:n_train_hours, :]
test = values[n_train_hours:, :]
# split into input and outputs
train_X, train_y = train[:, :-1], train[:, -1]
test_X, test_y = test[:, :-1], test[:, -1]
# reshape input to be 3D [samples, timesteps, features]
train_X = train_X.reshape((train_X.shape[0], 1, train_X.shape[1]))
test_X = test_X.reshape((test_X.shape[0], 1, test_X.shape[1]))
print(train_X.shape, train_y.shape, test_X.shape, test_y.shape)
print(values)
"""
Sequential class model is a linear stack of Layers. You can create a Sequential model and define all of the layers in the constructor.

We have used

1.LSTM Layer

2.Dense Layer

Why we are using LSTM Layer?

Typical RNN uses information from the previous step to predict the output.But if only the previous step is not enough,that is long term dependency.If we use RNN using all previous steps ,the **explosion/vanishing gradient problem** is encountered.

LSTM can solve this problem, because it uses gates to control the memorizing process. 

**Model Compliation**

Model optimizer is the search technique used to update weights in your model.(uses gradient descent methods to minimize the loss function)

The loss function, also called the objective function is the evaluation of the model used by the optimizer to navigate the weight space.

"""
# design network
model = Sequential()
model.add(LSTM(50, input_shape=(train_X.shape[1], train_X.shape[2])))
model.add(Dense(1))
model.compile(loss='mae', optimizer='adam')
# fit network
history = model.fit(train_X, train_y, epochs=250, batch_size=72, validation_data=(test_X, test_y), verbose=2, shuffle=False)
# plot history
plt.plot(history.history['loss'], label='train')
plt.plot(history.history['val_loss'], label='test')
plt.legend()
plt.show()
print(test_X.shape)
# make a prediction
yhat = model.predict(test_X)
print(yhat.shape)
#reshaping as the shape when scaling was done
test_x = test_X.reshape((test_X.shape[0], test_X.shape[2]))
# invert scaling for forecast
inv_yhat = np.concatenate((test_x[:,:-1],yhat), axis=1)
print(inv_yhat.shape)
inv_yhat = scaler.inverse_transform(inv_yhat)
inv_yhat = inv_yhat[:,-1]
#reshaping as the shape when scaling was done
test_y = test_y.reshape((len(test_y), 1))
inv_y = np.concatenate((test_x[:,:-1],test_y), axis=1)
# invert scaling for actual
inv_y = scaler.inverse_transform(inv_y)
inv_y = inv_y[:,-1]
# calculate RMSE
rmse = sqrt(mean_squared_error(inv_y, inv_yhat))
print('Test RMSE: %.3f' % rmse)
 
 from sklearn.metrics import r2_score
 r2_score(inv_y, inv_yhat)
"""
In scholarly research that focuses on marketing issues, R2 values of *0.75, **0.50**, or 0.25* can, as a rough rule of thumb, be respectively described as substantial, **moderate**, or weak. Sarstedt, M., & Mooi, E. (2014,p.211). 

https://people.duke.edu/~rnau/rsquared.htm
"""
com=pd.DataFrame(list(zip(inv_y,inv_yhat)),columns=['Actual','Predicted'])
com.iloc[15:35,:]

y=inv_y[:100]
predicted=inv_yhat[:100]
plt.plot(y,label='actual')
plt.plot(predicted,label='predicted')
plt.ylabel('Electricity unit')
plt.title('Actual vs Predicted')
plt.legend()
plt.show()