from datetime import datetime,timedelta, time
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from data_helper_functions import *
from IPython.display import display
pd.options.display.max_columns = 999
%matplotlib inline

desired_channel = 'BAND_01'
desired_date = datetime(2014, 4, 1)
desired_timedelta = timedelta(hours = 15)
desired_datetime = desired_date + desired_timedelta
satellite_filefolder = '../../data/satellite/colorado/summer6months/data/'
sensor_filefolder = '../../data/sensor_data/colorado6months/'
pvoutput_filefolder = '../../data/pvoutput/pvoutput6months/'

#satellite data
satellite_filename = find_filename(desired_datetime, desired_channel, satellite_filefolder)
lons, lats, data = return_satellite_data(satellite_filename, satellite_filefolder)


plt.figure(figsize=(8, 8))
imgplot = plt.imshow(data)
imgplot.set_interpolation('none')
plt.savefig('foo.png')
plt.show()

#sensor data
sensor_filename = find_file_from_date(desired_date, sensor_filefolder)
df_sensor = return_sensor_data(sensor_filename, sensor_filefolder)
df_sensor[df_sensor.index == desired_datetime]
display(df_sensor[df_sensor.index == desired_datetime])

#pvoutput data
pvoutput_filename = find_file_from_date(desired_date, pvoutput_filefolder)
df_pvoutput = return_pvoutput_data(pvoutput_filename, pvoutput_filefolder)
display(df_pvoutput[df_pvoutput.index == desired_datetime])

#saving df to image

# a = Image(data=df_sensor)
# type(a)

"""
Explanation: Summon any data
I want to make a single query and have it return data across the datasets
End of explanation
"""


from datetime import datetime,timedelta, time
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from data_helper_functions import *
from IPython.display import display
pd.options.display.max_columns = 999
%matplotlib inline

#iterate over datetimes:
mytime = datetime(2014, 4, 1, 13)
times = make_time(mytime)

# Now that we can call data up over any datetime and we have a list of interested datetimes,
# we can finally construct an X matrix and y vector for regression.

sensor_filefolder = 'data/sensor_data/colorado6months/'
pvoutput_filefolder = 'data/pvoutput/pvoutput6months/'

X = [] #Sensor values
y = [] #PVOutput

for desired_datetime in times:
    
    try: #something wrong with y on last day
        desired_date = (desired_datetime - timedelta(hours=6)).date() #make sure correct date
        desired_date = datetime.combine(desired_date, time.min) #get into datetime format

        sensor_filename = find_file_from_date(desired_date, sensor_filefolder)
        df_sensor = return_sensor_data(sensor_filename, sensor_filefolder).ix[:,-15:-1]
        df_sensor[df_sensor.index == desired_datetime]

        pvoutput_filename = find_file_from_date(desired_date, pvoutput_filefolder)
        df_pvoutput = return_pvoutput_data(pvoutput_filename, pvoutput_filefolder)
        
        y.append(df_pvoutput[df_pvoutput.index == desired_datetime].values[0][0])
        X.append(df_sensor[df_sensor.index == desired_datetime].values[0])
    except:
        pass

X = np.array(X)
y = np.array(y)

print X.shape
print y.shape

"""
Explanation: Build up sensor to pvoutput model
End of explanation
"""


from sklearn.cross_validation import train_test_split
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=99)

from sklearn.ensemble import RandomForestRegressor
rfr = RandomForestRegressor(oob_score = True)

rfr.fit(X_train,y_train)

y_pred = rfr.predict(X_test)

rfr.score(X_test,y_test)

df_sensor.columns.values.shape

sorted_mask = np.argsort(rfr.feature_importances_)

for i in zip(df_sensor.columns.values,rfr.feature_importances_[sorted_mask])[::-1]:
    print i

"""
Explanation: ...finally ready to model!
Random Forest
End of explanation
"""


#now do a linear model and compare:
from sklearn.linear_model import LinearRegression
lr = LinearRegression()
lr.fit(X_train,y_train)
lr.score(X_test,y_test)

sorted_mask = np.argsort(lr.coef_)

for i in zip(df_sensor.columns.values,lr.coef_[sorted_mask])[::-1]:
    print i

df_sensor.ix[:,-15:-1].head() #selects photometer and AOD, 
# useful in next iteration of using sensor data to fit

"""
Explanation: Linear model
End of explanation
"""


import pandas as pd
import numpy as np
from sklearn.preprocessing import scale
from lasagne import layers
from lasagne.nonlinearities import  softmax, rectify, sigmoid, linear, very_leaky_rectify, tanh
from lasagne.updates import nesterov_momentum, adagrad, momentum
from nolearn.lasagne import NeuralNet
import theano
from sklearn.cross_validation import train_test_split
from sklearn.preprocessing import StandardScaler

y = y.astype('float32')
x = X.astype('float32')
scaler = StandardScaler()
scaled_x = scaler.fit_transform(x)
x_train, x_test, y_train, y_test = train_test_split(scaled_x, y, test_size = 0.2, random_state = 12)

nn_regression = NeuralNet(layers=[('input', layers.InputLayer),
#                                   ('hidden1', layers.DenseLayer),
#                                    ('hidden2', layers.DenseLayer),
                                  ('output', layers.DenseLayer)
                                ],

               # Input Layer
               input_shape=(None, x.shape[1]),

               # hidden Layer
#                hidden1_num_units=512,
#                hidden1_nonlinearity=softmax,
                          
             # hidden Layer
#                hidden2_num_units=128,
#                hidden2_nonlinearity=linear,

              # Output Layer
              output_num_units=1,
              output_nonlinearity=very_leaky_rectify,

              # Optimization
              update=nesterov_momentum,
              update_learning_rate=0.03,#0.02
              update_momentum=0.8,#0.8
              max_epochs=600, #was 100

              # Others
              #eval_size=0.2,
              regression=True,
              verbose=0,
         )

nn_regression.fit(x_train, y_train)
y_pred = nn_regression.predict(x_test)
nn_regression.score(x_test, y_test)

val = 11
print y_pred[val][0]
print y_test[val]

plt.plot(y_pred,'ro')

plt.plot(y_test,'go')

"""
Explanation: When only keeping the photometer data, random forest and linear model do pretty similar.  When I added all of the sensor instruments to the fit, rfr scored 0.87 and lr scored negative!
Also, I threw away the mysterious "Research 2" sensor, that was probably just a solar panel!  I asked NREL what it is, so we'll see.  If it turns out to be a solar panel, then I can do some feature engineering with the sensor data by simulating a solar panel!
Neural Net Exploration
End of explanation
"""


from sklearn.ensemble import ExtraTreesRegressor
etr = ExtraTreesRegressor(oob_score=True, bootstrap=True,
                          n_jobs=-1, n_estimators=1000) #nj_obs uses all cores!
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=99)

etr.fit(X_train, y_train)

print etr.score(X_test,y_test)
print etr.oob_score_

y_pred = etr.predict(X_test)

from random import randint
val = randint(0,y_test.shape[0])
print y_pred[val]
print y_test[val]

print X.shape
print y.shape

"""
Explanation: Extra Trees!
End of explanation
"""


from sklearn.externals import joblib
joblib.dump(etr, 'data/sensor-to-power-model/sensor-to-power-model.pkl') 

np.savez_compressed('data/y.npz',y=y) #save y

"""
Explanation: Save this thing and try it out on the simulated sensors!
End of explanation
"""


