# This Python 3 environment comes with many helpful analytics libraries installed
# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python
# For example, here's several helpful packages to load

import numpy as np # linear algebra
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)

# Input data files are available in the read-only "../input/" directory
# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory

import os
for dirname, _, filenames in os.walk('/kaggle/input'):
    for filename in filenames:
        print(os.path.join(dirname, filename))

# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using "Save & Run All" 
# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session
"""
Credits: https://www.kaggle.com/ambrosm/tpsjan22-01-eda-which-makes-sense
"""
import seaborn as sns
import matplotlib.pyplot as plt
%matplotlib inline

import warnings
# ignore warnings
warnings.filterwarnings("ignore")
train = pd.read_csv("../input/tabular-playground-series-jan-2022/train.csv")
test = pd.read_csv("../input/tabular-playground-series-jan-2022/test.csv")
for df in [train, test]:
    df['date'] = pd.to_datetime(df['date'])
    df.set_index('date', inplace=True, drop=False)
train
# for every store, country and product combination (18 in total), we have data for 1461 days
train.groupby(['country', 'store', 'product']).count()
print("Training:")
print(f"First day: {train.date.min()}")
print(f"Last day: {train.date.max()}")
print("Predictions:")
print(f"First day: {test.date.min()}")
print(f"Last day: {test.date.max()}")
# num_sold is never zero, every product is always sold every day in every store
train.groupby(['country', 'store', 'product']).num_sold.agg(['min', 'max', 'mean'])
# On average, KaggleRama sells more of every product in every country in every year
train.groupby(['country', 'store', 'product', train.date.dt.year]).num_sold.mean().unstack('store')
# group products by year
grp = train.groupby(['country', 'store', 'product', train.date.dt.year]).num_sold.mean().unstack(level='product')
grp['Hat/Mug'] = grp['Kaggle Hat'] / grp['Kaggle Mug']
grp['Mug/Sticker'] = grp['Kaggle Mug'] / grp['Kaggle Sticker']
# ratios remain about constant, the ratio of Hat:Mug ~ 1.77 and Mug/Sticker ~ 1.98
grp
# group products by month
grp = train.groupby(['product', train.date.dt.month]).num_sold.mean().unstack(level='product')
grp['Hat/Mug'] = grp['Kaggle Hat'] / grp['Kaggle Mug']
grp['Mug/Sticker'] = grp['Kaggle Mug'] / grp['Kaggle Sticker']
# the ratio doesn't remain the same for months, so seasonality is there for products
grp
# Sale is higher on weekdays, (here, 0=Monday and 6=Sunday)
train.groupby(['country', 'store', 'product', train.date.dt.weekday]).num_sold.mean().unstack('product')
"""
### Histograms
"""
plt.figure(figsize=(18, 12))
for i, (combi, df) in enumerate(train.groupby(['country', 'store', 'product'])):
    ax = plt.subplot(6, 3, i+1, ymargin=0.5)
    ax.hist(df.num_sold, bins=50, color='blue')
    ax.set_title(combi)
    
plt.suptitle('Histograms of number of products sold', y=1.0)
plt.tight_layout(h_pad=0.3)
plt.show()
"""
Refer: [this nb cell](https://www.kaggle.com/ambrosm/tpsjan22-01-eda-which-makes-sense/notebook?scriptVersionId=84561837&cellId=15)

Inference:

The histograms for every country-store-product combination show that all histograms are skewed. For every product, there are some days with sales far above the mean. For these outliers, predictions will be much less accurate than for the regular days. This is why the competition is scored by Symmetric mean absolute percentage error (SMAPE) rather than MAE or MSE.
The histograms of the Kaggle Hat looks wider than the histograms of the other two products.
"""
"""
## Time Series
"""
"""
Resume reading from here:https://www.kaggle.com/ambrosm/tpsjan22-01-eda-which-makes-sense/notebook?scriptVersionId=84561837&cellId=17 
"""
plt.figure(figsize=(18, 12))
for i, (combi, df) in enumerate(train.groupby(['country', 'store','product'])):
    ax = plt.subplot(6, 3, i+1, ymargin=0.5)
    ax.plot(df.num_sold)
    ax.set_title(combi)
    
plt.tight_layout(h_pad=3.0)
plt.suptitle('Daily sales for 2015-18', y=1.03)
plt.show()
"""
Inference: 
* 1st column: Kaggle Hats sell more in the first half of every year 
* 2nd column: Kaggle mugs sell more at the end of every year (during winters)
* 3rd column: Kaggle stickers sale remains constant throughout the year
"""
"""
We zoom in on the end of year, particularly December and plot sales average for these 31 days:
"""
# for December
plt.figure(figsize=(18, 12))
for i, (combi, df) in enumerate(train.groupby(['country', 'store', 'product'])):
    ax = plt.subplot(6, 3, i+1, ymargin=0.5)
    ax.bar(range(1, 32), df.num_sold[df.date.dt.month==12].groupby(df.date.dt.day).mean(), color = ['r']*25 + ['orange']*6)
    ax.set_title(combi)
    ax.set_xticks(ticks=range(5, 31, 5))
plt.tight_layout(h_pad=3.0)
plt.suptitle('Daily sales of December', y=1.03)
plt.show()
"""
### Trend and Seasonality
"""
plt.figure(figsize=(18, 12))
for i, (combi, df) in enumerate(train.groupby(['country', 'store', 'product'])):
    ax = plt.subplot(6, 3, i+1, ymargin=0.5)
    resampled = df.resample('MS').num_sold.sum()
    ax.bar(range(len(resampled)), resampled)
    ax.set_title(combi)
    ax.set_ylim(resampled.min(), resampled.max())
    ax.set_xticks(range(0, 48, 12), [f"Jan{y}" for y in range(2015, 2019)])
plt.suptitle('Monthly sales for 2015-2018', y=1.03)
plt.tight_layout(h_pad=3.0)
plt.show()
"""
Inference:

* We can see an upward trend
* There is seasonal variation for every product
"""
# seeing the seasonality more clearly by averaging over month
plt.figure(figsize=(18, 12))
for i, (combi, df) in enumerate(train.groupby(['country', 'store', 'product'])):
    ax = plt.subplot(6, 3, i+1, ymargin=0.5)
    resampled = df.resample('MS').sum()
    resampled = resampled.groupby(resampled.index.month).mean()
    ax.bar(range(1, 13), resampled.num_sold)
    ax.set_xticks(ticks=range(1, 13), labels=['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'])
    ax.set_title(combi)
    ax.set_ylim(resampled.num_sold.min(), resampled.num_sold.max())
plt.suptitle('Monthly sales for 2015-2018', y=1.03)
plt.tight_layout(h_pad=3.0)
plt.show()
"""
Inference:

* Kaggle Hat: max in Apr, min in Sep
* Kaggle Mug: max in Dec, min in Jun,Jul
* Kaggle Sticker: max in Dec, min in Feb
"""
# weekly variation
plt.figure(figsize=(18, 12))
for i, (combi, df) in enumerate(train.groupby(['country', 'store', 'product'])):
    ax = plt.subplot(6, 3, i+1, ymargin=0.5)
    ax.bar(range(1, 8), df.num_sold.groupby(df.date.dt.weekday).mean(), color=['r']*5+ ['orange']*2)
    ax.set_title(combi)
    ax.set_xticks(ticks=range(1, 8), labels=['Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun'])
plt.suptitle('Weeky average')
plt.tight_layout(h_pad=3.0)
plt.show()
"""
Inference:

* Saturday and Sundays experience higher sales
* Fri also has larger sales compared to other working days
"""
"""
### Easter holiday
"""
"""
The diagram shows daily sales for April of the four years and we find that during the week after Easter, sales are higher than normal. Weekends are colored orange, Easter Sunday is marked red, the week after Easter is colored light blue.
"""
import dateutil.easter as easter
plt.figure(figsize=(12, 8))
for i, (year, df) in enumerate(train.groupby(train.date.dt.year)):
    df = df.reset_index(drop=True)
    ax = plt.subplot(4, 1, i+1, ymargin=0.5)
    april = df.num_sold[(df.date.dt.month==4)].groupby(df.date.dt.day).mean()
    date_range = pd.date_range(start=f'{year}-04-01', end=f'{year}-04-30', freq='D')
    easter_date = easter.easter(year)
    color = ['r' if d == easter_date else 'lightblue' if (d.date() - easter_date).days in range(6) else 'b' if d.dayofweek < 5 else 'orange' for d in date_range]
    ax.bar(range(1, 31),
           april,
           color=color)
    ax.set_title(str(year))
    ax.set_xticks(ticks=range(5, 31, 5))
plt.tight_layout(h_pad=3.0)
plt.suptitle('Daily sales for April', y=1.03)
plt.show()