import pandas as pd
import numpy as np

train=pd.read_csv('post_stockout_train_candidate.csv',parse_dates=['time_stamp_utc'])
pre_stock = pd.read_csv('pre_stockout_train_candidate.csv',parse_dates=['time_stamp_utc'])
order =pd.read_csv('pre_and_post_orders_data_train.csv')
order_test = pd.read_csv('test/pre_orders_data_test.csv',parse_dates=['device_acknowledge_at'])
pre_stock_test =pd.read_csv('test/pre_stockout_test_candidate.csv')
test =pd.read_csv('test/post_stockout_test_candidate.csv')
driver_log=pd.read_csv('pre_and_post_driver_log_train_candidate.csv')
driver_log_test=pd.read_csv('test/pre_driver_log_test_candidate.csv')

pd.set_option('display.max_columns', 500)

train.head()

test.head()

pre_stock.head()

pre_stock_test.head()

"""
Explanation: --Kanav Anand(IIT kharagpur)
                                                                                        Inustrial and Systems Engineering


<a href='#intro'>1. Content:</a>  

<a href='#rtd'>2. Retrieving the Data</a>

<a href='#ll'>2.1 Load libraries</a>
<a href='#td'>2.2 Read the Data</a>
<a href='#ll2'>2.3 Which data to use?</a>



<a href='#god'>3. Feature Engineering</a>

<a href='#ot'>3.1 Drivers Engaged</a>
<a href='#ot2'>3.2 Backbone of my Analysis(Feature Engineering)</a>
<a href='#ot3'>3.3 Exploring hidden hints and making features.</a>
<a href='#ot4'>3.4 Making Features using order file</a>
<a href='#ot5'>3.5 Aggregate features on order file.</a>
<a href='#ot6'>3.6 Number of months a person visit.</a>
<a href='#ot7'>3.7 Approximate month interval after which person make visits.</a>
<a href='#ot6'>3.8 Date-time based features.</a>


<a href='#nlp'>4.Making Model.</a>

Initially I have done some preprocessing in excel, So please use all the files in ZIP.
<a id='rtd'>2. Retrieving the Data</a>
<a id='ll'>2.1 Load libraries</a>
End of explanation
"""


pre_stock.groupby(by='dt')['stockout'].sum().sort_values(ascending=False)

%matplotlib inline
import matplotlib.pyplot as plt
pre_stock.groupby(by='dt')['stockout'].sum().plot(figsize=(10,4))
plt.xticks(rotation=40)

plt.annotate("1St Jan",xy=(1,8000))

order['created_at']=pd.to_datetime(order.created_at)
order['complete_at']=pd.to_datetime(order.complete_at)

order_test['created_at']=pd.to_datetime(order_test.created_at)
order_test['complete_at']=pd.to_datetime(order_test.complete_at)

order['date_create']=order.created_at.dt.date
order_test['date_create']=order_test.created_at.dt.date

order['unique']=1

pre_stock_test.columns

col = [ 'time_stamp_utc', 'dt', 'Latitude', 'Longitude', 
       'stockout', 'hour', 'minute', 'second', 'weekday']
alldata =pd.concat([train[col],pre_stock[col],pre_stock_test[col]],axis=0).reset_index(drop=True)

alldata.dt.value_counts().index

alldata =pd.concat([train[col],pre_stock[col],pre_stock_test[col]],axis=0).reset_index(drop=True)
alldata_out =  alldata.loc[alldata.dt != '1-Jan-18'].reset_index(drop=True)
alldata_out.shape

"""
Explanation: <a id='ll'>2.3 Which data to use?</a>
Here i am combining all the 3 data so as to collect the data of all files and removing 1st jan which is an outlier

Here i have removed the data of 1st Jan because it was new year, and as we can see the number of stockouts were too high  so we should remove it from our current analysis because .We are analysing for a month and it was an outlier because our test dataset does not contain any "Holiday or festival"

See here, there are around 8000 stockouts on single day!
End of explanation
"""


df = order.loc[(order.state=='COMPLETE')&(order.created_at.dt.day>29)].reset_index()

df.head()

df2 = order_test.loc[(order_test.state=='COMPLETE')&(order_test.created_at.dt.day>29)].reset_index()

order_test.loc[(order_test.state=='COMPLETE')&(order_test.created_at.dt.day>25)].reset_index().shape

from tqdm import tqdm

"""
Explanation: <a id='god'>3. Feature Engineering</a>
<a id='ot'>3.1  Drivers Engaged</a>

Here i am calculating how many drivers are engaged in given train-test combination of times. This would give us average engagement of a driver at particular time interval.

** Note: this would take 5-6 hours.
End of explanation
"""


train.head()

train=pd.read_csv('train_driver.csv')
test =pd.read_csv('test/post_stockout_test_candidate.csv')


train.head()

train.head()

"""
Explanation: i saved a  backup here(train_driver.csv which contains driver engaged)
End of explanation
"""



#train.head()
def upd(train,pre_stock):
    train = train.merge(pd.DataFrame(alldata_out.groupby(by=['weekday'])['stockout'].sum()).reset_index(),on='weekday',how='left',suffixes=('','_week'))
    train = train.merge(pd.DataFrame(alldata_out.groupby(by=['hour'])['stockout'].sum()).reset_index(),on='hour',how='left',suffixes=('','_hour'))
    train = train.merge(pd.DataFrame(alldata_out.groupby(by=['second'])['stockout'].sum()).reset_index(),on='second',how='left',suffixes=('','_second'))

    train.head()


    train = train.merge(pd.DataFrame(alldata_out.groupby(by=['weekday','hour'])['stockout'].sum()).reset_index(),on=['weekday','hour'],how='left',suffixes=('','_week_hour'))
    train = train.merge(pd.DataFrame(alldata_out.groupby(by=['hour','minute'])['stockout'].sum()).reset_index(),on=['hour','minute'],how='left',suffixes=('','_hour_minute'))

    train.fillna(0,inplace=True)

    train = train.merge(pd.DataFrame(alldata_out.groupby(by=['weekday','hour','minute'])['stockout'].sum()).reset_index(),on=['weekday','hour','minute'],how='left',suffixes=('','_hour_week_minute'))
    train = train.merge(pd.DataFrame(pre_stock.groupby(by=['res_id'])['stockout'].sum()).reset_index(),on='res_id',how='left',suffixes=('','_x'))




    train = train.merge(pd.DataFrame(pre_stock.groupby(by=['res_id','hour'])['stockout'].sum()).reset_index(),on=['res_id','hour'],how='left',suffixes=('','_hour_res'))

    train.fillna(0,inplace=True)

    train = train.merge(pd.DataFrame(pre_stock.groupby(by=['res_id','hour','minute'])['stockout'].sum()).reset_index(),on=['res_id','hour','minute'],how='left',suffixes=('','_hour_res_minute'))
#;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
    
    train = train.merge(pd.DataFrame(pre_stock.groupby(by=['res_id'])['stockout'].count()).reset_index(),on='res_id',how='left',suffixes=('','_countx'))




    train = train.merge(pd.DataFrame(pre_stock.groupby(by=['res_id','hour'])['stockout'].count()).reset_index(),on=['res_id','hour'],how='left',suffixes=('','_counthour_res'))

    train.fillna(0,inplace=True)

    train = train.merge(pd.DataFrame(pre_stock.groupby(by=['res_id','hour','minute'])['stockout'].count()).reset_index(),on=['res_id','hour','minute'],how='left',suffixes=('','_counthour_res_minute'))

    train.fillna(0,inplace=True)
    return train

"""
Explanation: <a id='ot2'>3.2 Backbone of my Analysis(Feature Engineering)</a>
Making Aggregate features like
Concat the train and test and make features like<br>

1) Stockout on day of week<br>
2)stockout in the hour of a day<br>
3)stockout in the hour of second<br>
4)stockout in the hour of every day  of week<br>
5)stockout in the minute of an hour of every day  of week<br>
6)Stockout in particular residential id.<br>
7)Stockout in particular residential id for a particular hour<br>
8)Stockout in particular residential id for a particular hour in that minute<br>
End of explanation
"""


from datetime import datetime
import datetime
def dat1(X):
    return(datetime.datetime.strptime(X, "%d%b%Y:%H:%M:%S"))

tm=train.time_stamp_utc.apply(dat1)

tm2=test.time_stamp_utc.apply(dat1)

train =upd(train,pre_stock)
train.head()

test =upd(test,pre_stock_test)
test= test.merge(pd.DataFrame(alldata_out.groupby(by=['weekday'])['stockout'].sum()).reset_index(),on='weekday',how='left',suffixes=('','_week'))

test.head()

"""
Explanation: Processing date
End of explanation
"""


import seaborn as sns
%matplotlib inline
import matplotlib.pyplot as plt
plt.figure(figsize=(20,5))
sns.heatmap(pre_stock.pivot_table(index='weekday',columns='minute',values='stockout',aggfunc=sum),linecolor='black')

min_graph=pre_stock.pivot_table(index='weekday',columns='minute',values='stockout',aggfunc=sum)
min_graph

plt.figure(figsize=(20,5))
min_graph.loc[1].plot()

sec_graph=pre_stock.pivot_table(index='weekday',columns='second',values='stockout',aggfunc=sum)
sec_graph

"""
Explanation: <a id='ot3'>3.3 Exploring hidden hints and making features.</a>
This is the minute stockout  history on every day-of-week basis
End of explanation
"""


plt.figure(figsize=(20,5))

sns.heatmap(pre_stock.pivot_table(index='weekday',columns='second',values='stockout',aggfunc=sum),linecolor='black')

"""
Explanation: This is the one of the most shocking observation here, as you can see there is very high probability of getting a stockout during between 11-25 and 41-55 seconds of every minute.
End of explanation
"""


plt.figure(figsize=(20,5))
sec_graph.loc[1].plot()

"""
Explanation: you can see that is somewhat perfect normal
End of explanation
"""


a=[]
for i in train.second:
    if i<15:
        a.append(i)
    elif i<30:
        a.append(30-i)
    elif i<45:
        a.append(i-30)
    else:
        a.append(60-i)

train['sec_fun']=a
a=[]
for i in test.second:
    if i<15:
        a.append(i)
    elif i<30:
        a.append(30-i)
    elif i<45:
        a.append(i-30)
    else:
        a.append(60-i)
test['sec_fun']=a
        

train.columns

cat_vars =[  'res_id',
       'hour', 'minute', 'second', 'weekday', ]
cont_vars =['Latitude', 'Longitude','stockout_week',
       'stockout_hour', 'stockout_second', 'stockout_week_hour',
       'stockout_hour_minute', 'stockout_hour_week_minute', 'stockout_x',
       'stockout_hour_res', 'stockout_hour_res_minute', 'stockout_countx',
       'stockout_counthour_res', 'stockout_counthour_res_minute']

for v in cat_vars: train[v] = train[v].astype('category').cat.as_ordered()
for v in cont_vars: train[v] = train[v].astype('float32')
for v in cat_vars: test[v] = test[v].astype('category').cat.as_ordered()
for v in cont_vars: test[v] = test[v].astype('category').astype('float32')        

"""
Explanation: Inference:

So we can infer that there could be two possible case:<br>
1)As stockout starts occuring the Zomato's iternal system may alert the Part time driver.<br>
2)The data-organisers may have generated the random time samples where it was not uniform but somewhat normal.

Using the Above fact lets discover a new feature based on Seconds
End of explanation
"""


order_comp=order.loc[order.state=='COMPLETE']
order_comp_test=order_test.loc[order.state=='COMPLETE']

"""
Explanation: <a id='ot4'>3.4 Making Features using order file.</a>
Using  the order file , Let's try to calculate the total number of orders and aggregate features
End of explanation
"""


order_comp['day']=order_comp.created_at.dt.day
order_comp['hour']=order_comp.created_at.dt.hour
order_comp['weekday']=order_comp.created_at.dt.dayofweek
order_comp['geography']=order_comp.pickup_locality

order_comp_test['day']=order_comp_test.created_at.dt.day
order_comp_test['hour']=order_comp_test.created_at.dt.hour
order_comp_test['weekday']=order_comp_test.created_at.dt.dayofweek
order_comp_test['geography']=order_comp_test.pickup_locality

order_comp['count_order']=1
order_comp_test['count_order']=1



"""
Explanation: Getting day time features for the order_file.
End of explanation
"""


def lower(x):
    return x.lower()
order_comp.geography=order_comp.geography.apply(lower)
train.geography=train.geography.apply(lower)

train.head()

order_comp.replace({'hsr layout':'hsr_layout'},inplace=True)

from sklearn.preprocessing import LabelEncoder
le_geo=LabelEncoder()
order_comp.geography=le_geo.fit_transform(order_comp.geography)
train.geography=le_geo.fit_transform(train.geography)

order_comp_test.geography=le_geo.fit_transform(order_comp_test.geography)
test.geography=le_geo.fit_transform(test.geography)

"""
Explanation: <a id='nlp'>4.Preprocessing the data. </a>
End of explanation
"""


train.head()

"""
Explanation: Authors of the data have very cleverly organised data by only removing the data of 2nd Jan to 12th Jan and remaining data is now used for the predicting the orders per hour,order per week and orders per  day-hour combinaiton
End of explanation
"""


order_comp.head()

train.weekday=train.weekday-1
test.weekday=test.weekday-1

"""
Explanation: correct the weekday
End of explanation
"""


order_comp.loc[order_comp.day>1].groupby(by=['geography','weekday','hour'],as_index=False).count_order.sum()
order_comp.loc[order_comp.day>1].groupby(by=['weekday'],as_index=False).count_order.sum()
order_comp.loc[order_comp.day>1].groupby(by=['day','hour'],as_index=False).count_order.sum()

order_comp_test.loc[order_comp_test.day>1].groupby(by=['weekday','hour'],as_index=False).count_order.sum()
order_comp_test.loc[order_comp_test.day>1].groupby(by=['weekday'],as_index=False).count_order.sum()
order_comp_test.loc[order_comp_test.day>1].groupby(by=['day','hour'],as_index=False).count_order.sum()

train=train.merge(order_comp.loc[order_comp.day>1].groupby(by=['geography','weekday','hour'],as_index=False).count_order.sum(),on=['geography','weekday','hour']
            ,how='left')
train=train.merge(order_comp.loc[order_comp.day>1].groupby(by=['geography','weekday'],as_index=False).count_order.sum(),on=['geography','weekday']
            ,how='left')

train=train.merge(order_comp.loc[order_comp.day>1].groupby(by=['weekday','hour'],as_index=False).count_order.sum(),on=['weekday','hour']
            ,how='left',suffixes=('','_1'))
train=train.merge(order_comp.loc[order_comp.day>1].groupby(by=['weekday'],as_index=False).count_order.sum(),on=['weekday']
            ,how='left',suffixes=('','_2'))
#train=train.merge(order_comp.loc[order_comp.day>1].groupby(by=['day','hour'],as_index=False).count_order.sum(),on=['day','hour']
#            ,how='left')
train.head()

test=test.merge(order_comp_test.loc[order_comp_test.day>1].groupby(by=['geography','weekday','hour'],as_index=False).count_order.sum(),on=['geography','weekday','hour']
            ,how='left')
test=test.merge(order_comp_test.loc[order_comp_test.day>1].groupby(by=['geography','weekday'],as_index=False).count_order.sum(),on=['geography','weekday']
            ,how='left')

test=test.merge(order_comp_test.loc[order_comp_test.day>1].groupby(by=['weekday','hour'],as_index=False).count_order.sum(),on=['weekday','hour']
            ,how='left',suffixes=('','_1'))
test=test.merge(order_comp_test.loc[order_comp_test.day>1].groupby(by=['weekday'],as_index=False).count_order.sum(),on=['weekday']
            ,how='left',suffixes=('','_2'))

def dat1(X):
    return(datetime.datetime.strptime(X, "%Y-%b-%d %H:%M:%S"))
from dateutil.parser import parse

def date1(x):
    return parse(x)
driver_log.login_time=driver_log.login_time.apply(date1)
driver_log.logout_time=driver_log.logout_time.apply(date1)

driver_log_test.login_time=driver_log_test.login_time.apply(date1)
driver_log_test.logout_time=driver_log_test.logout_time.apply(date1)

driver_log_test.head()

(driver_log_test.logout_time-driver_log_test.login_time).dt.seconds.head()

"""
Explanation: <a id='ot5'>3.5 Aggregate features on order file.</a>
Aggregate features.

1) Total orders within a geography in an hour of a weekday<br>
2) Total orders in a weekday<br>
3) Total orders in a day of hour.<br>
Combiing all of them with train
End of explanation
"""


valid =train.loc[train.dt == '31-Jan-18'].reset_index(drop=True)
train_val =train.loc[train.dt != '31-Jan-18'].reset_index(drop=True)

len(test.columns)

len(train.columns)

test.columns

train.columns

col = [  'Latitude', 'Longitude', 'res_id',
         'minute', 'geography',
       'stockout_hour',  'stockout_week_hour',
       'stockout_hour_minute', 'stockout_hour_week_minute','stockout_x',
       'stockout_hour_res', 'stockout_hour_res_minute',
       'stockout_counthour_res_minute',  'count_order_x',]

"""
Explanation: <a id='nlp'>4.Making Model. </a>
<a id='nlp'>4.Preparing validation set. </a>
Making validation set !
Here normal train and test won't work so divide according to date and treat it as a test.
End of explanation
"""


from sklearn.ensemble import RandomForestClassifier,GradientBoostingClassifier,BaggingClassifier,AdaBoostClassifier
import xgboost as xgb


X=train[col]

y=train.stockout

train_X=train_val[col]

train_y=train_val.stockout

test_X=valid[col]

test_y=valid.stockout

"""
Explanation: using RandomForest  model
End of explanation
"""


clf=RandomForestClassifier(n_estimators=30,max_depth=7)#highest_accuracy
clf=RandomForestClassifier(n_estimators=30,max_depth=7,)#highest_accuracy

#clf=AdaBoostClassifier(base_estimator=clf)
import lightgbm as lgb
from sklearn.tree import ExtraTreeClassifier
#clf = lgb.LGBMClassifier(max_depth=8,n_estimators=1000,random_state=5)
#clf=ExtraTreeClassifier(max_depth=7)
#clf=Dec
#clf=GradientBoostingClassifier(n_estimators=100,max_depth=7)
clf.fit(train_X[col],train_y,)

#a=test_y.copy()
a=np.zeros(test_y.shape)

from sklearn.metrics import accuracy_score,confusion_matrix
print('prediciton->',accuracy_score(test_y,clf.predict(test_X[col])))
print('zeroes->',accuracy_score(test_y,a))
confusion_matrix(test_y,clf.predict(test_X[col]))

valid.loc[valid.stockout==1]

valid.iloc[clf.predict(test_X[col])==1]

test

"""
Explanation: <a id='nlp2'>4.2 Applying the RandomForestclassifier</a>
End of explanation
"""


import matplotlib.pyplot as plt
import seaborn as sns
%matplotlib inline

plt.figure(figsize=(20,5))
sns.barplot(col,clf.feature_importances_)

pd.DataFrame(clf.feature_importances_,index=col).sort_values(by=0)

sub=pd.read_csv('test/submission_online_testcase.csv')

clf.fit(X,y)

sub['stockout']=clf.predict(test[X.columns])[:len(sub)]

sub.to_csv('submi/hmsub_allfeat_count.csv',index=None)

sub.stockout.sum()

"""
Explanation: Feature Importance chart
End of explanation
"""


