# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# + [markdown] id="pRH3HuFMDkob"
# # Team Name - Blitzkreig
# ## Indian Institute of Technology Kharagpur
# ### <NAME>, <NAME>, <NAME>
#
# - >Github Repo: https://github.com/Gauranshi/AbInBev-Maverick-2.0/tree/main
# - >Website: https://share.streamlit.io/arusarkabose/discount-predicter/main/app.py
#
# Pipeline:
# <img src = 'https://drive.google.com/uc?id=1jXansDiG4LQ2S4mNilg843ImXxpJvXjQ' >

# + [markdown] id="pKmywbFMERUp"
# ## Importing Library and Data

# + id="h9GkqAESCfYx"
import pandas as pd
import numpy as np
from sklearn import preprocessing
from sklearn.model_selection import train_test_split
from sklearn.tree import DecisionTreeRegressor, DecisionTreeClassifier
from sklearn.metrics import mean_squared_error, mean_absolute_error, r2_score
from sklearn.utils import check_array
import matplotlib.pyplot as plt
from xgboost import XGBRegressor
from xgboost import XGBClassifier
#from lazypredict.Supervised import LazyRegressor
from sklearn import metrics
from sklearn.ensemble import RandomForestClassifier
#import autosklearn.classification
#import lightgbm as lgb
import pickle
import urllib
import seaborn as sns
# -

# !pip install pandas --upgrade

# + colab={"base_uri": "https://localhost:8080/"} id="4GCqOGlijCH9" outputId="d8f5d1cc-9d72-4db1-d33a-d46aa9064b21"
import gdown
url = 'https://drive.google.com/uc?id=13celyxcevwAaKGcNXjJhHZAHMp0Yl_u8'
output = 'data.xlsx'
gdown.download(url, output, quiet=False)
path  ='data.xlsx'
data = pd.read_excel(path, sheet_name = 'Data',engine='openpyxl')
df = data.copy()


# + [markdown] id="-T5AcXTAEXmy"
# ## Pre-processing 

# + [markdown] id="nQOCmCs_I8K4"
# > - Replace missing data with the mode of Attribute
# > - Numerical values are replaced with 0 if less than a threshold (0.0005), to prevent Numerical Overflow
# > - Drop non-significant attributes (Ship-to ID, Product Set, Discount_Total)

# + id="4znUlonzDVZP"
def basic_preprocessing(df):
    for i in range(len(df)):
        if (abs(df.loc[i,'Volume_2019 Product'])<0.0005):
            df.loc[i,'Volume_2019 Product']=0
        if (abs(df.loc[i,'OffInvoice Discount(LCU)'])<0.0005):
            df.loc[i,'OffInvoice Discount(LCU)']=0
        if (abs(df.loc[i,'OnInvoice Discount(LCU)'])<0.0005):
            df.loc[i,'OnInvoice Discount(LCU)']=0

        if (df.loc[i,'poc_image']==0):
            df.loc[i,'poc_image']='Mainstream'

    df.loc[9,'segment']='Institutional'                            # REPLACED A SINGLE MISSING VALUE
    drop_cols=['Ship-to ID','Product Set','Discount_Total']
    df=df.drop(drop_cols,axis=1)
    df=df.iloc[0:-4]                                               # REPLACED A SINGLE MISSING VALUE
    return df
df = basic_preprocessing(df)


# + [markdown] id="AMyM3UbzJcSV"
# ### Outlier removal
# > - Remove data points with Z-score above a threshold. Formula for Z-score is given as:
#
# $$ 
# \text{Z-Score} = \frac{X-\text{Mean}}{\text{Standard Deviation}}
# $$
#
# Threshold is chosen as 3 as it indicates a spread of 3 Standard Deviation which covers 99.7% of the gaussian distribution.
#
# > - Remove data points with Negative values of discount, volume or GTO

# + id="0xDogZZJI0gw"
def outlier_removal(df):
    cols = ['Volume_2019', 'GTO_2019', 'OnInvoice Discount(LCU)', 'OffInvoice Discount(LCU)', 'Tax']
    mask=np.ones(len(df),dtype=bool)               # CREATING A MASK TO STORE ROWS
    threshold=3
    for col in cols:
        z_score=(df[col]-df[col].mean())/df[col].std(ddof=0)
        for i,j in enumerate(df[col]):
            if (j<0):
                mask[i]=0
        for i,j in enumerate(z_score):
            if (j>threshold):
                mask[i]=0
    df = df[mask] 
    df=df.reset_index().drop(['index'],axis=1)
    return df
df = outlier_removal(df)

# + [markdown] id="ntaj11rsm2bI"
# ## Exploratory Data Analysis

# + [markdown] id="1EXEHwaZi86g"
# * Correlation heatmap for the numeric columns illustrates that the OnInvoice Discount is more correlated with the GTO_2019 as compared to the OffInvoice Discount. Though through line plots it seems that the opposite is true, the high number of zero values in the OffInvoice Discount column reduces its correlation.

# + colab={"base_uri": "https://localhost:8080/", "height": 402} id="1h-MYQIHnX9c" outputId="ce30be07-f11a-4a72-b6c1-937502d845c7"
cmap = sns.diverging_palette(0, 230, 90, 60, as_cmap=True)
sns.heatmap(df.corr(), cmap=cmap, vmax=1, vmin=-1, center=0,
            square=True, linewidths=.5)

# + [markdown] id="-B4kdtsZdr_t"
#
# * Through violinplots of the sales volumes for the financial years 2018 and 2019, we infer that the mean volume sold in 2018 is slightly higher than that in 2019.

# + colab={"base_uri": "https://localhost:8080/", "height": 449} id="_-HNgyeGs-cY" outputId="a1c48d4b-d4d4-4973-bdd0-c5993ccfc477"
fig, ax = plt.subplots(figsize =(9, 7))
sns.violinplot(ax = ax, data = df.iloc[:, 0:2])

# + [markdown] id="xPyXrSE9eECV"
# * Through regression model plots of the turnover against the OffInvoice and OnInvoice discounts, we see that outliers are more prominent in the OffInvoice column as is visible by the relatively low slope of the mean regression line.

# + colab={"base_uri": "https://localhost:8080/", "height": 401} id="kSeeeCc8vaCt" outputId="d580965e-af43-46ef-eb47-4b111c067ba2"
sns.lmplot(x='GTO_2019', y='OffInvoice Discount(LCU)', data=df)

# + colab={"base_uri": "https://localhost:8080/", "height": 401} id="UaZsWak9v1UA" outputId="08f06887-e133-4fd4-8262-980e9a1f364a"
sns.lmplot(x='GTO_2019', y='OnInvoice Discount(LCU)', data=df)

# + [markdown] id="EbQz9EQTe6YE"
# * The percentage of zeros in the OffInvoice column is much higher than that in the OnInvoice column as is exposed by the histogram plots.

# + colab={"base_uri": "https://localhost:8080/", "height": 296} id="AHj-OEflxzyZ" outputId="663fc432-71d7-4f88-9540-c3457b48b1d4"
eda_df=df.copy()
for i in range(len(eda_df)):
    if(eda_df.loc[i,"OnInvoice Discount(LCU)"]):
        eda_df.loc[i,"OnInvoice Discount(LCU)"]="Non-Zero"
    else:
        eda_df.loc[i,"OnInvoice Discount(LCU)"]="Zero"
sns.histplot(data=eda_df, x="OnInvoice Discount(LCU)", kde=False)

# + colab={"base_uri": "https://localhost:8080/", "height": 296} id="GT-qFb16zlp-" outputId="02c51ffa-d619-45f2-8aa2-0c41c923d03a"
eda_df=df.copy()
for i in range(len(eda_df)):
    if(eda_df.loc[i,"OffInvoice Discount(LCU)"]):
        eda_df.loc[i,"OffInvoice Discount(LCU)"]="Non-Zero"
    else:
        eda_df.loc[i,"OffInvoice Discount(LCU)"]="Zero"
sns.histplot(data=eda_df, x="OffInvoice Discount(LCU)", kde=False)

# + [markdown] id="5k4RRafEfIwZ"
# * We observe that around 85% of the OffInvoice Discount values are within 34.65% of the respective GTO_2019 values

# + colab={"base_uri": "https://localhost:8080/", "height": 297} id="tRyw2YWbIrHr" outputId="5bf1e2e0-8ef8-4539-9d07-46cd204d9b07"
eda_df=df.copy()
eda_df['marker']=0
for i in range(len(eda_df)):
    eda_df.loc[:,'marker']=0.3465*eda_df.loc[:,'GTO_2019']
plt.plot(eda_df.marker,eda_df['OffInvoice Discount(LCU)'],label='OffInvoice')
plt.plot(eda_df.marker,eda_df['marker'],label='marker')
plt.legend()
plt.xlabel('0.3465*GTO_2019')
plt.ylabel('Value')

# + [markdown] id="X843Fez4faCf"
# * We calculate the Cramer's V correlation matrix for the categorical columns of the dataset

# + id="KTeYsk6jLa_a"
data=df[['sfdc_tier','poc_image','segment','sub_segment','Brand','Sub-Brand','Pack_Type','Returnalility','province']]
label = preprocessing.LabelEncoder()
data_encoded = pd.DataFrame() 

for i in data.columns :
    data_encoded[i]=label.fit_transform(data[i])

# + id="EXKt_r5qOO40"
from scipy.stats import chi2_contingency
def cramers_V(var1,var2) :
    crosstab =np.array(pd.crosstab(var1,var2, rownames=None, colnames=None)) # Cross table building
    stat = chi2_contingency(crosstab)[0] # Keeping of the test statistic of the Chi2 test
    obs = np.sum(crosstab) # Number of observations
    mini = min(crosstab.shape)-1 # Take the minimum value between the columns and the rows of the cross table
    return (stat/(obs*mini))


# + id="FnwR81KHOYrV"
rows= []

for var1 in data_encoded:
    col = []
    for var2 in data_encoded :
        cramers =cramers_V(data_encoded[var1], data_encoded[var2]) # Cramer's V test
        col.append(round(cramers,2)) # Keeping of the rounded value of the Cramer's V  
    rows.append(col)
  
cramers_results = np.array(rows)
eda_df = pd.DataFrame(cramers_results, columns = data_encoded.columns, index =data_encoded.columns)

# + [markdown] id="jRE3xu7LkCID"
# Apart from the obvious correlations between Brand and Sub-Brand, and Segment and Sub-Segment, we observe that there is high correlation between the Pack_Type and Returnability columns. 
#
# All of the CAN and BULK are OW, along with 7.74% of the BOTTLE category, while all of the data points in the rest of the Pack_Type categories are RETURNABLE.
#

# + colab={"base_uri": "https://localhost:8080/", "height": 343} id="JBEWzOowPBJx" outputId="97515890-a239-4328-bb6d-29260c4b17fb"
mask = np.zeros_like(eda_df, dtype=np.bool)
mask[np.triu_indices_from(mask)] = True
sns.heatmap(eda_df, mask=mask, vmin=0., vmax=1, square=True)


# + [markdown] id="2ReGH6kuZJQT"
# ## Feature Engineering
# 1. **Volume Difference**: Discount multiplier based on the sales volume difference between the financial years 2019 and 2018. 
#
# > - For reduced volumes, a multiplier factor of 1.24 is provided to incentivise higher sales.
# > - For stagnant volumes, a multiplier factor of 1.12 is provided to incentivise higher sales.
# > - For increased volumes, the multiplier factor is 0.95 in order to increase the sales profits.
#
# 2. **Tiering Structure**: Modified tiering structure clubbing the lower tier premiums with the higher tier mainstreams, in accordance with similarity trends between the same:
#
# <center> Tier 2 Mainstream &xrarr; Tier 3  </center>
# <center> Tier 1 Mainstream + Tier 2 Premium &xrarr; Tier 2 </center>
# <center> Tier 0 Mainstream + Tier 1 Premium &xrarr; Tier 1 </center>
# <center> Tier 0 Premium &xrarr; Tier 0  </center>
#
# 3. **Discount Multiplier**: Discount multipier factor based on the average sales volume. The multiplier has been defined through the creation of a slab system which would encourage the PoS to increase the order volumes to move higher up the discount slabs:
#
# | Volume Slab | Discount Multiplier |
# | ----------- | ----------- |
# | > 60 percentile and < 70 percentile | 1.1 |
# | > 70 percentile and < 80 percentile | 1.15 |
# | > 80 percentile and < 90 percentile | 1.2 |
# | > 90 percentile and < 95 percentile | 1.3 |
# | > 95 percentile and < 99 percentile | 1.35 |
# | > 99 percentile and < 99.9 percentile | 1.4 |
# | > 99.9 percentile | 1.5 |
# <br>
#
# 4. **Sales Share**: For each POC, it is the ratio of volume sold by that POC to total volume of liquor sold in that province. Formula for sales share can be given as: 
# $$\frac{Volume_{ POC}}{\sum_{}Volume_{POC\in Province^{i}}} $$
#
#
# 5. **Product Cost**: It is the price of a particular product being sold at POC. Assuming tax rate to be r, total transaction value will be Tax/r, thus procuct price can be given as:
# $$ \frac{Tax}{r \times Volume}$$

# + id="yCTmPB9UY3m6"
def volume_discmult(df):
    df['vol_diff']=df['Volume_2019']-df['Volume_2018']
    df['vol_disc']=df['Discount_Total']
    df.loc[df['vol_diff']<0, 'vol_disc'] *=1.24
    df.loc[df['vol_diff']==0, 'vol_disc'] *=1.12
    df.loc[df['vol_diff']>0, 'vol_disc'] *=0.95
    return df


# + id="IYf1GxU4Yzvj"
def tiering_struct(df):
    df['new_tier']=''
    for i in range(len(df)):
        tier=df.loc[i,'sfdc_tier']
        poc=df.loc[i,'poc_image']
        if (tier=='Tier 2' and poc=='Mainstream'):
            df.loc[i,'new_tier']='Tier 3'
        elif ((tier=='Tier 1' and poc=='Mainstream') or (tier=='Tier 2' and poc=='Premium')):
            df.loc[i,'new_tier']='Tier 2'
        elif ((tier=='Tier 0' and poc=='Mainstream') or (tier=='Tier 1' and poc=='Premium')):
            df.loc[i,'new_tier']='Tier 1'
        elif (tier=='Tier 0' and poc=='Premium'):
            df.loc[i,'new_tier']='Tier 0'
    df=df.drop(['poc_image','sfdc_tier'],axis=1)
    return df


# + id="xxIiRUhPYu1p"
def engineer_discmult(df):
    df["avg_annual_vol"]=(df['Volume_2019']+df['Volume_2018'])/2
    df["discount_multiplier"]=0
    qn=dict()
    qn["60"]=df.avg_annual_vol.quantile(0.6)
    qn["70"]=df.avg_annual_vol.quantile(0.7)
    qn["80"]=df.avg_annual_vol.quantile(0.8)
    qn["90"]=df.avg_annual_vol.quantile(0.9)
    qn["95"]=df.avg_annual_vol.quantile(0.95)
    qn["97"]=df.avg_annual_vol.quantile(0.97)
    qn["99.9"]=df.avg_annual_vol.quantile(0.999)
    for i in range(len(df)):
        vol=df.loc[i,"avg_annual_vol"]
        if  (vol>=qn["60"] and vol<qn["70"]):
            df.loc[i,"discount_multiplier"]=1.1
        elif (vol>=qn["70"] and vol<qn["80"]):
            df.loc[i,"discount_multiplier"]=1.15
        elif (vol>=qn["80"] and vol<qn["90"]):
            df.loc[i,"discount_multiplier"]=1.2
        elif (vol>=qn["90"] and vol<qn["95"]):
            df.loc[i,"discount_multiplier"]=1.3
        elif (vol>=qn["95"] and vol<qn["99"]):
            df.loc[i,"discount_multiplier"]=1.35
        elif (vol>=qn["99"] and vol<qn["99.9"]):
            df.loc[i,"discount_multiplier"]=1.4
        elif (vol>=qn["99.9"]):
            df.loc[i,"discount_multiplier"]=1.5
    return df


# + id="DXbynJ63YNvu"
def sales_share(df):
    prov_dic = {}
    for i in range(len(df)):
        if(df.loc[i,'province'] in prov_dic):
            prov_dic[df.loc[i,'province']] += df.loc[i,'Volume_2019 Product']
        else:
            prov_dic[df.loc[i,'province']] = df.loc[i,'Volume_2019 Product']
    df['sales_share']=df['Volume_2019 Product']
    for i in range(len(df)):
        df.loc[i,'sales_share'] = df.loc[i,'Volume_2019'] / prov_dic[df.loc[i,'province']]
    return df


# + id="b8d3fuERYsK9"
def product_cost(df):
    df['product_cost']=df['Tax']
    prod = {}
    dx = df
    for i in range(len(df)):
        tax=df.loc[i,'Tax']
        vol=df.loc[i,'Volume_2019 Product']
        if( vol!=0):
            prod[backup.loc[i,'Product Set']] = tax/vol
            df.loc[i,'product_cost']= tax/vol
        if(vol==0):
            if backup.loc[i,'Product Set'] in prod:
                df.loc[i,'product_cost']= prod[backup.loc[i,'Product Set']]
            else:
                df.loc[i,'product_cost'] = int(25)
    return df


# + [markdown] id="lqM7edJR7MKw"
# ## One-Hot Encoding and Label Encoding
# > -   One-hot encode Nominal categorical columns using Pandas
# > -   Label encode Ordinal categorical columns using Sk-learn
#
#

# + id="r8Y7h0pKNym8"
#ONE-HOT ENCODING AND LABEL ENCODING
def encoding(df):
    oh_cols=['poc_image','segment','Brand','sub_segment','Pack_Type','Returnalility','province','Sub-Brand']
    #oh_cols=['segment','Brand','sub_segment','Pack_Type','Returnalility']
    df=pd.get_dummies(data = df,columns=oh_cols)
    le = preprocessing.LabelEncoder()
    le.fit(df.sfdc_tier)
    df.sfdc_tier=le.transform(df.sfdc_tier)
    return df
df = encoding(df)


# + [markdown] id="<KEY>"
# ## Split between training and test datasets
#
#
# > -   Drop both Discounts values from Train
# > -   Split between Train and Test datasets taking the split ratio as a user-defined parameter
#
#

# + id="hmpruGPOKAH5"
def df_train_test_split(df, target_var):
    if(target_var =='on'):
        target='OnInvoice Discount(LCU)'
        other = 'OffInvoice Discount(LCU)'
    if(target_var =='off'):
        target='OffInvoice Discount(LCU)'
        other = 'OnInvoice Discount(LCU)'
    df=df.drop([other],axis=1)
    Y=df.loc[:,target]
    X=df.drop([target],axis=1)
    X_train, X_test, y_train, y_test = train_test_split(X, Y, test_size=0.33, random_state=42)
    return X_train, X_test, y_train, y_test


# + [markdown] id="2OsPlCU0HKkZ"
# ## Model Architecture

# + [markdown] id="p9CY6RTfC4Zs"
# ### OnInvoice Discount

# + [markdown] id="iuU700PHTtWx"
# LOAD PRETRAINED MODELS
#
# This cell can be skipped and runing the subsequent cells would initialize and train the models, or load these pretrained models and comment the necessary lines in the concerned blocks.

# + colab={"base_uri": "https://localhost:8080/"} id="2jVo-2YpTHN_" outputId="42490745-bfa5-4eac-b0d2-e1bc58387f1e"
#LOAD TRAINED MODELS FROM ULRS

url = "https://www.dropbox.com/s/ml4a9jilm2vcnuj/on_xgbreg.pickle.dat?dl=1"  # dl=1 is important
urllib.request
u = urllib.request.urlopen(url)
data = u.read()
u.close()
with open('on_reg.pickle.dat', "wb") as f :
    f.write(data)

regressor=pickle.load(open("on_reg.pickle.dat", "rb"))

url = "https://www.dropbox.com/s/0aplcor3qhin2j1/on_clf1.pickle.dat?dl=1"  # dl=1 is important
urllib.request
u = urllib.request.urlopen(url)
data = u.read()
u.close()
with open('on_clf1.pickle.dat', "wb") as f :
    f.write(data)

clf=pickle.load(open("on_clf1.pickle.dat", "rb"))

url = "https://www.dropbox.com/s/4amnhgx27b7jzgf/on_clf2.pickle.dat?dl=1"  # dl=1 is important
urllib.request
u = urllib.request.urlopen(url)
data = u.read()
u.close()
with open('on_clf2.pickle.dat', "wb") as f :
    f.write(data)
clf2=pickle.load(open("on_clf2.pickle.dat", "rb"))

url = "https://www.dropbox.com/s/e46p2nc2s6xbgiz/rfreg_on.pickle.dat?dl=1"  # dl=1 is important
urllib.request
u = urllib.request.urlopen(url)
data = u.read()
u.close()
with open('on_rfr.pickle.dat', "wb") as f :
    f.write(data)
rfr=pickle.load(open("on_rfr.pickle.dat", "rb"))

# + [markdown] id="he9jEh-ijZPs"
# Training Classifier
#
# Two classifier are trained:
# - > Decision Tree Classifier: Creates a model that predicts the value of a target variable by learning simple decision rules inferred from the data features. 
# - > XGBoost Classifer: Boosting is based on weak learners and it reduces the bias error thus is useful to explain variance of the dataset.
#
# Decision Tree has high bias, low variance whereas XGBoost has low bias, high variance, thus ensemble of these two models produces the best results

# + colab={"base_uri": "https://localhost:8080/"} id="1tfAqQdZHM6b" outputId="41ec83a5-1b82-4203-819a-52df5de366c8"
X_train, X_test, y_train, y_test = df_train_test_split(df,'on')

# CLASSIFER FOR ZERO/NON-ZERO DISCOUNTS
# FIRST CLASSIFIER AS XGBOOST

y_train_cl=(y_train!=0).astype(int)
y_test_cl=(y_test!=0).astype(int)
# clf = XGBClassifier(n_estimators=3000, max_depth=4, eta=0.1, subsample=0.7, colsample_bytree=0.8)
# clf.fit(X_train,y_train_cl)
cl_preds=clf.predict(X_test)

# SECOND CLASSIFIER IS A SINGLE DECISION TREE
# clf2 = DecisionTreeClassifier(random_state=0)
# clf2.fit(X_train,y_train_cl)
cl2_preds=clf2.predict(X_test)

clf_preds = cl_preds * cl2_preds

from sklearn.metrics import classification_report
print(classification_report(y_test_cl, clf_preds))


# + [markdown] id="D8yi_BzAjWjj"
# Regressor
#
# - > The Regressor is trained on Non-Zero data points only, while the classifier models are trained on the complete dataset.
# - > The Regression results are stacked on the classification predictions for generating the final inference.

# + id="wZnpBv_AKIGO"
# FUNCTION TO FILTER DATA POINTS WITH ZERO DISCOUNT VALUE
def filer_non_zero(df, target_var):
    if(target_var =='on'):
        target='OnInvoice Discount(LCU)'
    if(target_var =='off'):
        target='OffInvoice Discount(LCU)'
    mask=np.ones(len(df),dtype=bool)
    for i in df.iterrows():
        if(i[1][target]==0):
            mask[i[0]]=0
    df = df[mask] 
    return df


# + id="Fhdq64YrrKis"
# EVALUATION METRICS FOR REGRESSION
def reg_eval_metrics(y_true, y_pred):
    error = 0
    for i,j in enumerate(y_true):
        if(j>0): error += np.abs((j - y_pred[i]) / (j +1)) * 100
        else:  error += np.abs((j - y_pred[i]) / (y_pred[i] +1)) * 100
    print('MAPE Score:  ', error/len(y_true))
    print('RMSE Score:  ', mean_squared_error(y_true, preds, squared=False))
    print('MAE Score:   ',mean_absolute_error(y_true, preds))


# + colab={"base_uri": "https://localhost:8080/"} id="KWS6DEmGjVEC" outputId="6892d382-dc5c-4a53-89ce-a8e4506c6c8f"
# TRAINING AND TESTING XGBOOST REGRESSOR 

# REMOVING ROWS WITH ZERO DISCOUNT VALUE
dx = filer_non_zero(df, 'on')
X_train_mod, X_test_mod, y_train_mod, y_test_mod = df_train_test_split(dx,'on')

# TRAINING MODEL WITH NON-ZERO VALUES
# regressor =  XGBRegressor(n_estimators=3000, max_depth=4, eta=0.1, subsample=0.7, colsample_bytree=0.8)
# regressor.fit(X_train_mod,y_train_mod)
preds=regressor.predict(X_test_mod)

print("Evaluation for non-zero only rows")
reg_eval_metrics(y_test_mod, preds)
print("")
print("")


# PREDICTING ON ENTIRE DATASET
preds=regressor.predict(X_test)
preds=preds*cl_preds*cl2_preds                         # FILTERING DATA-POINTS WHICH WERE PREDICTED ZERO BY CLASSIFICATION PIPELINE
# preds=preds*cl_preds

print("Evaluation for entire datatset")
reg_eval_metrics(y_test, preds)

# + colab={"base_uri": "https://localhost:8080/"} id="By6WYNzZ1q58" outputId="2b8dda24-21f4-4558-9cdd-82d55a3bd003"
# TRAINING AND TESTING RANDOM FOREST REGRESSOR
from sklearn.ensemble import RandomForestRegressor
# rfr = RandomForestRegressor( random_state=0)
# rfr.fit(X_train_mod,y_train_mod)
preds=rfr.predict(X_test_mod)

print("Evaluation for non-zero only rows")
reg_eval_metrics(y_test_mod, preds)
print("")
print("")

preds=rfr.predict(X_test)
preds=preds*cl_preds*cl2_preds                        # FILTERING DATA-POINTS WHICH WERE PREDICTED ZERO BY CLASSIFICATION PIPELINE

print("Evaluation for entire datatset")
reg_eval_metrics(y_test, preds)

# + [markdown] id="x1gGWG08MYPB"
# ### OffInvoice Discount

# + [markdown] id="kiva_A8PVOYi"
# LOAD PRETRAINED MODELS
#
# You can skip this cell and run the subsequent cells which would initialize and train the models, or load these pretrained models and comment the necessary lines in the concerned blocks.

# + colab={"base_uri": "https://localhost:8080/"} id="U8WUAvjbUmAh" outputId="93c75589-73a1-4978-ece1-e7683e33e624"
# LOAD TRAINED MODELS FROM ULRS
url = "https://www.dropbox.com/s/hks3pb01hc8jc0a/off_xgbreg.pickle.dat?dl=1"  # dl=1 is important
urllib.request
u = urllib.request.urlopen(url)
data = u.read()
u.close()
with open('off_reg.pickle.dat', "wb") as f :
    f.write(data)

regressor=pickle.load(open("off_reg.pickle.dat", "rb"))

url = "https://www.dropbox.com/s/tjt29nwyok3v2x0/off_clf1.pickle.dat?dl=1"  # dl=1 is important
urllib.request
u = urllib.request.urlopen(url)
data = u.read()
u.close()
with open('off_clf1.pickle.dat', "wb") as f :
    f.write(data)

clf=pickle.load(open("off_clf1.pickle.dat", "rb"))

url = "https://www.dropbox.com/s/8lv8fe46chu528g/rfreg_off.pickle.dat?dl=1"  # dl=1 is important
urllib.request
u = urllib.request.urlopen(url)
data = u.read()
u.close()
with open('off_rfr.pickle.dat', "wb") as f :
    f.write(data)
rfr=pickle.load(open("off_rfr.pickle.dat", "rb"))

# + [markdown] id="pvFLaYm2zTtL"
# Training Classifier

# + colab={"base_uri": "https://localhost:8080/"} id="GtxAGae2Masd" outputId="54cf9018-3585-4c0e-e125-b11b174aa56e"
X_train, X_test, y_train, y_test = df_train_test_split(df,'off')

# CLASSIFER FOR ZERO/NON-ZERO DISCOUNTS
# FIRST CLASSIFIER AS XGBOOST
y_train_cl=(y_train!=0).astype(int)
y_test_cl=(y_test!=0).astype(int)
# clf = XGBClassifier(n_estimators=3000, max_depth=4, eta=0.1, subsample=0.7, colsample_bytree=0.8)
# clf.fit(X_train,y_train_cl)
cl_preds=clf.predict(X_test)

from sklearn.metrics import classification_report
print(classification_report(y_test_cl, cl_preds))

# + colab={"base_uri": "https://localhost:8080/"} id="fVMxmPHvzU21" outputId="bed638ff-2fe0-4130-8565-c65b947d0fea"
# TRAINING REGRESSOR

# REMOVING ROWS WITH ZERO DISCOUNT VALUE
dx = filer_non_zero(df, 'off')
X_train_mod, X_test_mod, y_train_mod, y_test_mod = df_train_test_split(dx,'on')

# TRAINING MODEL WITH NON-ZERO VALUES
# regressor =  XGBRegressor(n_estimators=3000, max_depth=4, eta=0.1, subsample=0.7, colsample_bytree=0.8)
# regressor.fit(X_train_mod,y_train_mod)
preds=regressor.predict(X_test_mod)

print("Evaluation for non-zero only rows")
reg_eval_metrics(y_test_mod, preds)
print("")
print("")

# PREDICTING ON ENTIRE DATASET
preds=regressor.predict(X_test)
preds=preds*cl_preds                                              # FILTERING DATA-POINTS WHICH WERE PREDICTED ZERO BY CLASSIFICATION PIPELINE

print("Evaluation for entire datatset")
reg_eval_metrics(y_test, preds)

# + colab={"base_uri": "https://localhost:8080/"} id="rXPlPShZfbM8" outputId="a7267de3-bfe1-414f-ee04-2f4d59eaf780"
# TRAINING AND TESTING RANDOM FOREST REGRESSOR
# rfr = RandomForestRegressor( random_state=0)
# rfr.fit(X_train_mod,y_train_mod)
preds=rfr.predict(X_test_mod)

print("Evaluation for non-zero only rows")
reg_eval_metrics(y_test_mod, preds)
print("")
print("")

preds=rfr.predict(X_test)
preds=preds*cl_preds

print("Evaluation for entire datatset")
reg_eval_metrics(y_test, preds)

# + [markdown] id="l-RQbPJsZB_M"
# ## Extra Experiments

# + [markdown] id="HuwKiJIMaIYS"
# Ensemble using Genetic Algorithm on SVR, XGBoost, Random Forest Regressors. This is a random-based evolutionary algorithm which weights models resulting in the theoretical best possible evaluation score

# + id="zf555xh_ZRnD"
# TRAINING SVR MODEL
from sklearn.svm import SVR
svr = SVR(epsilon=0.2)
svr.fit(X_train_mod,y_train_mod)
preds=svr.predict(X_test_mod)

models = [regressor, svr, rfr]
test_pred_list = []                         # LIST CONTAINING TEST PREDICTIONS FOR ALL REGRESSORS
for i in models:
    preds = i.predict(X_test)
    test_pred_list.append(preds)

train_pred_list = []                        # LIST CONTAINING TRAIN PREDICTIONS FOR ALL REGRESSORS
for i in models:
    preds = i.predict(X_train)
    train_pred_list.append(preds)
all_test_preds = np.array(test_pred_list)
all_train_preds = np.array(train_pred_list)

# + colab={"base_uri": "https://localhost:8080/", "height": 598} id="8RblFkY7JW5L" outputId="95fac97a-43b9-453f-9933-9dbeb6b17f7a"
# !pip install geneticalgorithm
from geneticalgorithm import geneticalgorithm as ga

# FUNCTION TO TRAIN WEIGHTS
def f(X):
    softmax_w = np.exp(X)/np.sum(np.exp(X))
    weighted_all_test_preds = np.array([sw*dpreds for sw, dpreds in zip(softmax_w, all_test_preds)])
    # print(weighted_all_test_preds.shape)
    # weighted_dev_preds = np.sum(weighted_all_test_preds, axis = 0)
    # final_dev_preds = np.mean(weighted_all_test_preds, axis = 0)
    score1 = mean_squared_error(y_test, weighted_all_test_preds[0], squared=False)
    score2 = mean_squared_error(y_test, weighted_all_test_preds[1], squared=False)
    score3 = mean_squared_error(y_test, weighted_all_test_preds[2], squared=False)
    return min(score1, score2,score3)

# TRAIN GA ENSEMBLE MODEL
varbound=np.array([[0, 1]]*all_test_preds.shape[0])
model=ga(function=f,dimension=all_test_preds.shape[0],variable_type='real',variable_boundaries=varbound)
model.run()

convergence=model.report
solution=model.output_dict
solution
X = solution['variable']
# MULTIPLYING MODEL PREDICTIONS WITH WEIGHTS  
softmax_w = np.exp(X)/np.sum(np.exp(X))
weighted_all_test_preds = np.array([sw*dpreds for sw, dpreds in zip(softmax_w, all_test_preds)])
final_dev_preds = np.mean(weighted_all_test_preds, axis = 0)
score = mean_squared_error(y_test, final_dev_preds, squared=False)

# + [markdown] id="5RQX2Z8RqBA2"
# ### Prospect for Future Feature Additions:
#
# #### Country Tax Rates:
#
# Based on the province of the POC and COVID-19 regulations, we can adjust our discount rates so as to meet up the loss in sales and customers caused due to hindrances in transportation and the increased tax rates.

StarcoderdataJupyterScriptsDedupFiltered