# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# Added custom loss function base on @kyakvolev 's work. Credit to the author.

# The forum post is here: https://www.kaggle.com/c/m5-forecasting-uncertainty/discussion/139515

# + _cell_guid="b1076dfc-b9ad-4769-8c92-a6c4dae69d19" _uuid="8f2839f25d086af736a60e9eeb907d3b93b6e0e5"
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from tqdm.auto import tqdm as tqdm

from ipywidgets import widgets, interactive, interact
import ipywidgets as widgets
from IPython.display import display

import os
for dirname, _, filenames in os.walk('data/raw'):
    for filename in filenames:
        print(os.path.join(dirname, filename))
# -

# ## Reading data

# + _cell_guid="79c7e3d0-c299-4dcb-8224-4455121ee9b0" _uuid="d629ff2d2480ee46fbb7e2d37f6b5fab8052498a"
train_sales = pd.read_csv('data/raw/sales_train_validation.csv')
calendar_df = pd.read_csv('data/raw/calendar.csv')
submission_file = pd.read_csv('data/raw/sales_train_validation.csv')
sell_prices = pd.read_csv('data/raw/sample_submission.csv')
# -

# ## Variables to help with aggregation

total = ['Total']
train_sales['Total'] = 'Total'
train_sales['state_cat'] = train_sales.state_id + "_" + train_sales.cat_id
train_sales['state_dept'] = train_sales.state_id + "_" + train_sales.dept_id
train_sales['store_cat'] = train_sales.store_id + "_" + train_sales.cat_id
train_sales['store_dept'] = train_sales.store_id + "_" + train_sales.dept_id
train_sales['state_item'] = train_sales.state_id + "_" + train_sales.item_id
train_sales['item_store'] = train_sales.item_id + "_" + train_sales.store_id

# +
val_eval = ['validation', 'evaluation']

# creating lists for different aggregation levels
total = ['Total']
states = ['CA', 'TX', 'WI']
num_stores = [('CA',4), ('TX',3), ('WI',3)]
stores = [x[0] + "_" + str(y + 1) for x in num_stores for y in range(x[1])]
cats = ['FOODS', 'HOBBIES', 'HOUSEHOLD']
num_depts = [('FOODS',3), ('HOBBIES',2), ('HOUSEHOLD',2)]
depts = [x[0] + "_" + str(y + 1) for x in num_depts for y in range(x[1])]
state_cats = [state + "_" + cat for state in states for cat in cats]
state_depts = [state + "_" + dept for state in states for dept in depts]
store_cats = [store + "_" + cat for store in stores for cat in cats]
store_depts = [store + "_" + dept for store in stores for dept in depts]
prods = list(train_sales.item_id.unique())
prod_state = [prod + "_" + state for prod in prods for state in states]
prod_store = [prod + "_" + store for prod in prods for store in stores]
# -

print("Departments: ", depts)
print("Categories by state: ", state_cats)

quants = ['0.005', '0.025', '0.165', '0.250', '0.500', '0.750', '0.835', '0.975', '0.995']
days = range(1, 1913 + 1)
time_series_columns = [f'd_{i}' for i in days]


# ## Getting aggregated sales

def CreateSales(name_list, group):
    '''
    This function returns a dataframe (sales) on the aggregation level given by name list and group
    '''
    rows_ve = [(name + "_X_" + str(q) + "_" + ve, str(q)) for name in name_list for q in quants for ve in val_eval]
    sales = train_sales.groupby(group)[time_series_columns].sum() #would not be necessary for lowest level
    return sales


total = ['Total']
train_sales['Total'] = 'Total'
train_sales['state_cat'] = train_sales.state_id + "_" + train_sales.cat_id
train_sales['state_dept'] = train_sales.state_id + "_" + train_sales.dept_id
train_sales['store_cat'] = train_sales.store_id + "_" + train_sales.cat_id
train_sales['store_dept'] = train_sales.store_id + "_" + train_sales.dept_id
train_sales['state_item'] = train_sales.state_id + "_" + train_sales.item_id
train_sales['item_store'] = train_sales.item_id + "_" + train_sales.store_id

#example usage of CreateSales
sales_by_state_cats = CreateSales(state_cats, 'state_cat')
sales_by_state_cats


# ## Getting quantiles adjusted by day-of-week

def CreateQuantileDict(name_list = stores, group = 'store_id' ,X = False):
    '''
    This function writes creates sales data on given aggregation level, and then writes predictions to the global dictionary my_dict
    '''
    sales = CreateSales(name_list, group)
    sales = sales.iloc[:, 2:] #starting from d_3 because it is a monday, needed to make daily_factors work
    sales_quants = pd.DataFrame(index = sales.index)
    for q in quants:
        sales_quants[q] = np.quantile(sales, float(q), axis = 1)
    full_mean = pd.DataFrame(np.mean(sales, axis = 1))
    daily_means = pd.DataFrame(index = sales.index)
    for i in range(7):
        daily_means[str(i)] = np.mean(sales.iloc[:, i::7], axis = 1)
    daily_factors = daily_means / np.array(full_mean)

    daily_factors = pd.concat([daily_factors, daily_factors, daily_factors, daily_factors], axis = 1)
    daily_factors_np = np.array(daily_factors)

    factor_df = pd.DataFrame(daily_factors_np, columns = submission_file.columns[1:])
    factor_df.index = daily_factors.index

    for i,x in enumerate(tqdm(sales_quants.index)):
        for q in quants:
            v = sales_quants.loc[x, q] * np.array(factor_df.loc[x, :])
            if X:
                my_dict[x + "_X_" + q + "_validation"] = v
                my_dict[x + "_X_" + q + "_evaluation"] = v
            else:
                my_dict[x + "_" + q + "_validation"] = v
                my_dict[x + "_" + q + "_evaluation"] = v


my_dict = {}
#adding prediction to my_dict on all 12 aggregation levels
CreateQuantileDict(total, 'Total', X=True) #1
CreateQuantileDict(states, 'state_id', X=True) #2
CreateQuantileDict(stores, 'store_id', X=True) #3
CreateQuantileDict(cats, 'cat_id', X=True) #4
CreateQuantileDict(depts, 'dept_id', X=True) #5
CreateQuantileDict(state_cats, 'state_cat') #6
CreateQuantileDict(state_depts, 'state_dept') #7
CreateQuantileDict(store_cats, 'store_cat') #8
CreateQuantileDict(store_depts, 'store_dept') #9
CreateQuantileDict(prods, 'item_id', X=True) #10
CreateQuantileDict(prod_state, 'state_item') #11
CreateQuantileDict(prod_store, 'item_store') #12

total

# ## Creating valid prediction df from my_dict

pred_df = pd.DataFrame(my_dict)
pred_df = pred_df.transpose()
pred_df_reset = pred_df.reset_index()
final_pred = pd.merge(pd.DataFrame(submission_file.id), pred_df_reset, left_on = 'id', right_on = 'index')
del final_pred['index']
final_pred = final_pred.rename(columns={0: 'F1', 1: 'F2', 2: 'F3', 3: 'F4', 4: 'F5', 5: 'F6', 6: 'F7', 7: 'F8', 8: 'F9',
                                        9: 'F10', 10: 'F11', 11: 'F12', 12: 'F13', 13: 'F14', 14: 'F15', 15: 'F16',
                                        16: 'F17', 17: 'F18', 18: 'F19', 19: 'F20', 20: 'F21', 21: 'F22', 
                                        22: 'F23', 23: 'F24', 24: 'F25', 25: 'F26', 26: 'F27', 27: 'F28'})
final_pred = final_pred.fillna(0)

for i in range(1,29):
    final_pred['F'+str(i)] *= 1.170
final_pred.to_csv('return_of_the_blend.csv', index=False)

StarcoderdataJupyterScriptsDedupFiltered