"""
# Stacking Ensemble of Deep earning Neural Networks
"""
"""
![image.png](attachment:image.png)
"""
"""
### Importing the libraries
"""
import numpy as np
from keras.layers import Dense, Activation
from keras.models import Sequential
from sklearn.model_selection import train_test_split
import matplotlib.pyplot as plt
import pandas as pd 
"""
## Part 1 - Data Preprocessing
"""
"""
### Importing the dataset
"""
df = pd.read_csv('../input/datamahedi/data.csv',delimiter=';')
df.columns
df.head(7)
"""
### Dropping the missing values 
"""
df.isnull().sum()
df = df.dropna()
df.describe(include='all')
"""
We do a pairplot for all the variable sin the dataset
"""
#import seaborn as sns 
#sns.pairplot(df)
"""
### Splitting the dataset into the Training set and Test set
"""
"""
We create input features and target variables
"""
df['Y']
x=df.drop(['Y'], axis=1)
y=df['Y'].values
y
#if you write the code y = df['y'] you will get a series of values, but if you add (.values) you will get an array of values which can be reshaped after that. 
"""
Creating the training and test dataset
"""
from sklearn.model_selection import train_test_split
X_train, X_test, y_train, y_test = train_test_split(x,y,test_size=0.25,shuffle=False)
"""
### Feature Scaling
"""
"""
All input features are numerical so we need to scale them. StandardScaler works well when the data is normally distributed. Based on the pair plot we see that the data is not normally distributed. Hence we use MinMaxScaler to scale the data
"""
from sklearn.preprocessing import StandardScaler
sc = StandardScaler()
X_train = sc.fit_transform(X_train)
X_test = sc.transform(X_test)


y= y.reshape(-1,1)
y=sc.fit_transform(y)
"""
# NN for regression 
"""
"""
Creating the neural network for the regressor. We have the input nodes, we create one hidden layer with the nodes and an output layer.

As this a regression problem, the loss function we use is mean squared error and the metrics against which we evaluate the performance of the model is mean absolute error and accuracy.
Mean absolute error is the absolute difference between the predicted value and the actual value.

we define a function build_regressor to use these wrappers. build_regressor creates and returns the Keras sequential model.
"""
from keras import Sequential
from keras.layers import Dense
def build_regressor():
    regressor = Sequential()
    regressor.add(Dense(units=380, input_dim=380))
    regressor.add(Dense(units=1))
    regressor.compile(optimizer='adam', loss='mean_squared_error',  metrics=['mae','accuracy'])
    return regressor
"""
We pass build_regressor function to the build_fn argument when constructing the KerasRegressor class. Batch_size is 32 and we run 100 epochs
"""
from keras.wrappers.scikit_learn import KerasRegressor
regressor = KerasRegressor(build_fn=build_regressor, batch_size=32,epochs=100)
results=regressor.fit(X_train,y_train)
y_pred= regressor.predict(X_test)
fig, ax = plt.subplots()
ax.scatter(y_test, y_pred)
ax.plot([y_test.min(), y_test.max()], [y_test.min(), y_test.max()], 'k--', lw=4)
ax.set_xlabel('Measured')
ax.set_ylabel('Predicted')
plt.show()
"""
Black broken line is the predicted values and we can see that it encompasses most of the values

Couple of tips for better accuracy
* Always standardize both input features and target variable. If we only standardize input feature then we will get incorrect predictions
* Data may not be always normally distributed so check the data and then based on the distribution apply StandardScaler, MinMaxScaler, Normalizer or RobustScaler
"""
"""
# **MLP for regression **
"""
"""
(MLPs) can be used successfully for classification, albeit that state-of-the-art methods may yield better performance for some datasets.

and it can also be used for a regression problem.
"""
"""
Salient points of Multilayer Perceptron (MLP) in Scikit-learn
* There is no activation function in the output layer.
* For regression scenarios, the square error is the loss function, and cross-entropy is the loss function for the classification
* It can work with single as well as multiple target values regression.
* Unlike other popular packages, likes Keras the implementation of MLP in Scikit doesn’t support GPU.
* We cannot fine-tune the parameters like different activation functions, weight initializers etc. for each layer.
"""
"""Import the required modules"""
from sklearn.neural_network import MLPRegressor
from sklearn.metrics import r2_score
"""
In the below code, three hidden layers are modelled, with 380 neurons in each layer. Considering the input and output layer, we have a total of 5 layers in the model. In case any optimiser is not mentioned then “Adam” is the default optimiser and it can manage pretty large dataset.
"""
"""
you play with the hidden layer sizes to get a better r2_score !
here few notes to take into consideration : 
* The number of hidden neurons should be between the size of the input layer and the size of the output layer.
* The number of hidden neurons should be 2/3 the size of the input layer, plus the size of the output layer.
* The number of hidden neurons should be less than twice the size of the input layer.
* see this https://www.heatonresearch.com/2017/06/01/hidden-layers.html

"""
reg = MLPRegressor(hidden_layer_sizes=(380,100,10),activation="relu" ,random_state=1, max_iter=2000).fit(X_train,y_train)
"""
In addition to “RELU” activation, MLPRegressor supports the “sigmoid” and “hyperbolic tan” function.
the trained model is used to predict the target values of the reserved test dataset, which model has not seen before.
"""
y_pred=reg.predict(X_test)
print("The Score with ", (r2_score(y_pred, y_test)))
fig, ax = plt.subplots()
ax.scatter(y_test, y_pred)
ax.plot([y_test.min(), y_test.max()], [y_test.min(), y_test.max()], 'k--', lw=4)
ax.set_xlabel('Measured')

ax.set_ylabel('Predicted')
plt.show()