"""
# Data analysis and Forecasting of COVID19 cases in Italy
"""
"""
## <font color='white'><span style='background :black' >The notebook is updated daily. Latest update: <font color='red'>Jan 30th with Jan 30th data.</font>
"""
"""
**The following project is about modeling daily COVID19 cases in Italy by Machine Learning algorithms, with the goal of forecasting the cases for future days.<br>
The time series modeling will be performed by Prophet by Facebook, Neural Prophet and SARIMAX models.**
"""
"""
<img src="https://i.imgur.com/pnhV2Ow.png" width="900px">
"""
"""
# Main results Dashboard:
"""
"""
<img src="https://i.imgur.com/2NXrfU6.png" width="900px">
"""
"""
**All the three algorithms performed well on the last 4 weeks of data compared to the actual values, with similar predicted cases.** <br>
Moreover, **the forecasted values are also similar**.
"""
"""
<img src="https://i.imgur.com/7G22LLn.png" width="900px">
"""
"""
We can clearly see that **the number of weekly cases is getting more stable and lower in terms of Mean, Median and Std. Deviation.**<br>
"""
"""
<img src="https://i.imgur.com/oycem0Q.png" width="900px">
"""
"""
**Trend and seasionality plots analysis:**
- Trend: We can see the trends of the 5 covid waves: the steeper one is the current one (Jan 2022).
- Holidays: We can see the effects of holidays on the number of new cases.
- Weekly seasonality: We can observe the drop of cases on Monday, since on Sunday less tests are carried, and an overall increase over the week, with the highest number of positive cases found around Friday.
- Yearly seasonality: We can observe the drop in cases from May to November, seen both in 2020 and 2021
"""
"""
<img src="https://i.imgur.com/QuKpBxc.png" width="900px">
"""
"""
Overall, we can see that **the average predicted values fit well the last 4 weeks data.**
"""
"""
### Install necessary libraries: pmdarima, neuralprophet and holidays
"""
!pip install pmdarima
pip install neuralprophet
pip install holidays
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns

from sklearn.metrics import mean_squared_error

import calendar

import holidays

import fbprophet
from fbprophet import Prophet

from neuralprophet import NeuralProphet

from statsmodels.tsa.stattools import adfuller
from statsmodels.graphics.tsaplots import plot_acf, plot_pacf

import pmdarima as pm
from statsmodels.tsa.statespace.sarimax import SARIMAX

seed=42

plt.style.use("seaborn-whitegrid")
plt.rc("figure", autolayout=True)
plt.rc("axes", labelweight="bold", labelsize="large", titleweight="bold", titlesize=14, titlepad=10)
"""
## Custom defined functions
"""
def mape(y_true, y_pred): 
    y_true, y_pred = np.array(y_true), np.array(y_pred)
    return np.mean(np.abs((y_true - y_pred) / y_true)) * 100
"""
## Loading the data
"""
"""
We can fetch the data from the official government github repository, where the data is updated daily at 18:00.
"""
df = pd.read_csv('https://raw.githubusercontent.com/pcm-dpc/COVID-19/master/dati-regioni/dpc-covid19-ita-regioni.csv', parse_dates=['data'])
df.tail()
df.info()
"""
**NOTE: We can see that the dataframe contains lots of columns. For our analysis we will just focus on the columns 'data' ( 'date' in english) and 'nuovi_positivi' ('new cases' in english).<br>**
"""
"""
First, we convert the date column to pandas datetime format.
"""
df['data'] = pd.to_datetime(df['data']).dt.normalize()
"""
Moreover, we get today's and tomorrow's date, since they will be useful for future plots.
"""
today = df['data'].iloc[-1]
tomorrow = today + pd.DateOffset(days=1)
"""
Since the data is divided by region for each day, we create a new dataframe 'df_italy' where we group the rows by the column 'data' ( 'date' in english) to get the total daily data in Italy.
"""
df_italy=df.groupby('data').sum()
plt.figure(figsize=(10,4))
plt.title('COVID19 new cases in Italy', fontsize=20)
plt.plot(df_italy['nuovi_positivi'])
plt.text(df_italy.index[0],np.max(df_italy['nuovi_positivi'])-30000,
         'Todays new cases:{}'.format(df_italy['nuovi_positivi'].iloc[-1]),
         fontsize=20,
         bbox=dict(facecolor='white', alpha=1))
plt.ylabel('New cases')
plt.show()
plt.figure(figsize=(13,6))
plt.title('COVID19 new cases in Italy during the last 4 weeks', fontsize=30)
plt.plot(df_italy[-28:].index, df_italy[-28:].nuovi_positivi, marker='o', color='red')
plt.bar(df_italy[-28:].index, df_italy[-28:].nuovi_positivi, color='#000080', alpha=0.8)
plt.ylabel('New Cases', fontsize=15)
plt.text(df_italy[-29:].index[0],np.max(df_italy[-28:]['nuovi_positivi'])-5000,
         'Todays new cases:{}'.format(df_italy['nuovi_positivi'].iloc[-1]),
         fontsize=20,
         bbox=dict(facecolor='white', alpha=1))
plt.grid(visible=None, axis='x')
plt.show()
"""
## Holidays
"""
"""
In the following, we will define a 'holiday' column to include boolean values to check if a day is a holiday or not. We decided to include this column since, after a first analysis, it looks like that on days after holidays the number of new cases decreases, since on holidays usually less PCR tests are performed. By specifing this information to the prediction models, we can obtain a more accurate forecast.
"""
"""
We create a list compregension to extract the holidays from the holidays library, and add bolean values to a list is_holiday depending if the date is a holiday (value=1) or not (value=0).
"""
is_holiday = [1 if x==True else 0 for x in [day in holidays.Italy() for day in df_italy.index]]
"""
Then we assign these values to a new column 'holiday' of the dataframe.
"""
df_italy['holiday'] = is_holiday
"""
# Weekly case distribution analysis
"""
"""
Next we will analyze the weekly distribution to check how the mean, median and standard deviation of new cases changed during the weeks.<br>
First, we create a list of the days for each date.
"""
day = [calendar.day_name[day.weekday()] for day in df_italy.index]
"""
Then we create a new column in the dataframe to host this list.
"""
df_italy['day'] = day
"""
Now we need to create a function that select only the last 4 weeks in the dataframe. We decided to create a new dataframe 'df_italy_small' and assign it the last 34 rows of the  dataframe 'df_italy'. Then we check if the current day, and in case it is a Sunday then we can copy 28 rows (4 weeks) of data and assign to a new dataframe 'df_4weeks'
"""
idx=0 #index to move along the rows
df_4weeks = pd.DataFrame() #empty dataframe
df_italy_small = df_italy.iloc[-34:].iloc[::-1] #consider 34 rows (worst case)

while(True):
    if df_italy_small.iloc[idx:].day[0] == 'Sunday':
        df_4weeks=df_italy_small.iloc[idx:idx+28]
        break
    else:
        idx+=1
"""
Then we extract the 4 values for the 4 weeks, and add a column to indicate the week.
"""
df_week1=df_4weeks.iloc[:7]
df_week1['week']='1 week ago'

df_week2=df_4weeks.iloc[7:14]
df_week2['week']='2 weeks ago'

df_week3=df_4weeks.iloc[14:21]
df_week3['week']='3 weeks ago'

df_week4=df_4weeks.iloc[21:28]
df_week4['week']='4 weeks ago'

df_4weeks_2 = pd.concat([df_week1, df_week2, df_week3, df_week4])
"""
We extract also the mean, median and standard deviation of the cases during these weeks.
"""
mean_1 = np.round(df_week1.nuovi_positivi.mean(),0)
median_1 = np.round(df_week1.nuovi_positivi.median(),0)
std_1 = np.round(df_week1.nuovi_positivi.std(),0)

mean_2 = np.round(df_week2.nuovi_positivi.mean(),0)
median_2 = np.round(df_week2.nuovi_positivi.median(),0)
std_2 = np.round(df_week2.nuovi_positivi.std(),0)

mean_3 = np.round(df_week3.nuovi_positivi.mean(),0)
median_3 = np.round(df_week3.nuovi_positivi.median(),0)
std_3 = np.round(df_week3.nuovi_positivi.std(),0)

mean_4 = np.round(df_week4.nuovi_positivi.mean(),0)
median_4 = np.round(df_week4.nuovi_positivi.median(),0)
std_4 = np.round(df_week4.nuovi_positivi.std(),0)
"""
And create ad additional dataframe where we include the week and the three statistics.
"""
df_stats = pd.DataFrame({'week':['1 week ago','2 weeks ago','3 weeks ago','4 weeks ago'],
                         'mean':[mean_1,mean_2,mean_3,mean_4],
                         'median':[median_1,median_2,median_3,median_4],
                         'std':[std_1,std_2,std_3,std_4]})
df_stats
_, ax = plt.subplots(2,2,figsize=(14,9))
plt.suptitle('Analysis of weekly COVID19 cases over last 4 weeks', fontsize=32)

##PLOT 1

sns.kdeplot(x='nuovi_positivi', data=df_week4, label='4 weeks ago', shade=1,ax=ax[0,0])
sns.kdeplot(x='nuovi_positivi', data=df_week3, label='3 weeks ago', shade=1,ax=ax[0,0])
sns.kdeplot(x='nuovi_positivi', data=df_week2, label='2 weeks ago', shade=1,ax=ax[0,0])
sns.kdeplot(x='nuovi_positivi', data=df_week1, label='1 week ago', shade=1,ax=ax[0,0])

ax[0,0].axvline(mean_1, linewidth=1.5, color='red', linestyle='--')
ax[0,0].axvline(mean_2, linewidth=1.5, color='green', linestyle='--')
ax[0,0].axvline(mean_3, linewidth=1.5, color='orange', linestyle='--')
ax[0,0].axvline(mean_4, linewidth=1.5, color='blue', linestyle='--')

ax[0,0].legend(fontsize=13,fancybox=True, shadow=True, frameon=True, loc="upper left")
ax[0,0].set_xlabel('New Cases', fontsize=15)
ax[0,0].set_title('New cases Distribution ', fontsize=20)

##PLOT 2
g = sns.barplot(y='mean', x='week', data=df_stats.iloc[::-1] ,ax=ax[0,1], ci=False)
g.bar_label(g.containers[0], padding=2, fontsize=16, color='black')
sns.lineplot(y='mean', x='week', data=df_stats.iloc[::-1], ax=ax[0,1], ci=False, color='black',linewidth=1, linestyle='--')
ax[0,1].set_ylabel('New Cases', fontsize=15)
ax[0,1].set_xlabel(' ')
ax[0,1].set_title('Mean cases', fontsize=20)

##PLOT 3
g = sns.barplot(y='median', x='week', data=df_stats.iloc[::-1] ,ax=ax[1,0], ci=False)
g.bar_label(g.containers[0], padding=2, fontsize=16, color='black')
sns.lineplot(y='median', x='week', data=df_stats.iloc[::-1], ax=ax[1,0], ci=False, color='black',linewidth=1, linestyle='--')
ax[1,0].set_ylabel('New Cases', fontsize=15)
ax[1,0].set_xlabel(' ')
ax[1,0].set_title('Median cases', fontsize=20)

##PLOT 4
g = sns.barplot(y='std', x='week', data=df_stats.iloc[::-1] ,ax=ax[1,1], ci=False)
g.bar_label(g.containers[0], padding=2, fontsize=16, color='black')
sns.lineplot(y='std', x='week', data=df_stats.iloc[::-1], ax=ax[1,1], ci=False, color='black',linewidth=1, linestyle='--')
ax[1,1].set_ylabel('New Cases', fontsize=16)
ax[1,1].set_xlabel(' ')
ax[1,1].set_title('Weekly Standard Deviation', fontsize=20)


plt.show()
"""
**We can clearly see that the number of weekly cases is getting more stable and lower in terms of Mean, Median and Std. Deviation.**
"""
"""
# Time series modeling and Forecasting 
"""
"""
Now we consider just 2 columns for the time series forecasting: 'new_cases' and 'holiday'.
"""
df = df_italy.copy()
df = df[['nuovi_positivi','holiday']]
df.head()
"""
Moreover, we also define some time windows which will be useful during the forecasting analysis and plotting.
"""
prediction_window = 28 #testing window (4 weeks of data)
forecast_window = 7 # forecasting window (1 week)
window = prediction_window + forecast_window #prediction + forecasting window
"""
# Prophet 
"""
"""
**The first model we use is Prophet by Facebook Research.**<br>
The model requires the dataset to have just 2 columns: 'ds' which host the dates and 'y' which hosts the time series values: for this reason we define a new dataframe 'df_p' which will present just these two columns. The holiday column will be specified by calling a method on the model.
"""
df_p = df['nuovi_positivi'].reset_index().copy()
df_p = df_p.rename(columns={'data': 'ds', 'nuovi_positivi': 'y'})
"""
Now we can define the Prophet model.
"""
prophet_model = Prophet(n_changepoints=50, # hyperparameter
                        seasonality_mode='multiplicative',
                       changepoint_prior_scale=10) # hyperparameter
"""
We add both weekly and yearly seasonality.
"""
prophet_model.add_seasonality('weekly', period = 7, fourier_order = 5)
prophet_model.add_seasonality('yearly', period = 365, fourier_order = 25)
"""
Since we cannot add the holiday column as we defined previously, we add the holidays using the 'add_country_holidays()' method of the prophet model class.
"""
prophet_model.add_country_holidays(country_name='Italy')
prophet_model.fit(df_p)
"""
# Prophet Forecasting
"""
"""
To perform the forecasting, we need to first create a forecast dataframe to include future days. As stated before, we will forecast the time series values of the next week.
"""
future = prophet_model.make_future_dataframe(periods=forecast_window)
"""
Then we call the 'predict()' method of the prophet model, where we specify the dataframe with future days 'future'
"""
forecast = prophet_model.predict(future)
prophet_model.plot(forecast);
plt.title("COVID19 new cases in Italy with forecasting by Prophet", fontsize=23)
plt.ylabel("New cases", fontsize=15)
plt.xlabel('')
plt.show()
"""
**At a first glance, we can see that the predicted time series (in blue) fits well the original data (black dots).**
"""
"""
Moreover, it could be interesting to see the components (the trend for example) of the predicted time series by calling the 'plot_components()' method of the prophet model.
"""
fig = prophet_model.plot_components(forecast)
"""
Plots analysis:
- Trend: We can see the trends of the 5 covid waves: the steeper one is the current one (Jan 2022).
- Holidays: We can see the effects of holidays on the number of new cases.
- Weekly seasonality: We can observe the drop of cases on Monday, since on Sunday less tests are carried, and an overall increase over the week, with the highest number of positive cases found around Friday.
- Yearly seasonality: We can observe the drop in cases from May to November, seen both in 2020 and 2021
"""
"""
Next, we want to make a custom plot to check the forecasted days. We first create a dataframe including the dates (ds), predicted values (yhat) and confidence intervals( yhat_lower and yhat_upper).
"""
forecast_df = forecast[['ds', 'yhat', 'yhat_lower', 'yhat_upper']]
forecast_df
plt.figure(figsize=(15,5))
plt.title('COVID19 new cases in Italy with forecasting by Prophet', fontsize=22)
#Actual cases
plt.plot(df[-prediction_window:].index, df[-prediction_window:]['nuovi_positivi'], label='Actual cases', marker='o')

#PROPHET
plt.plot(forecast_df[-window:]['ds'],forecast_df[-window:]['yhat'],color='#006400',label='PROPHET', marker='o')
plt.fill_between(forecast_df[-forecast_window-1:]['ds'], forecast_df[-forecast_window-1:]['yhat_lower'],forecast_df[-forecast_window-1:]['yhat_upper'], color='lightgreen', alpha=0.5)

plt.axvline(today, linewidth=1.5, color='red', linestyle="--")
plt.legend(loc='upper left', fontsize=14, fancybox=True, shadow=True, frameon=True)
plt.ylabel('New cases', fontsize=20)
plt.xticks(forecast_df[-prediction_window-forecast_window:]['ds'], rotation=80, fontsize=14)
plt.yticks(fontsize=15)

plt.grid(visible=None, axis='x')
plt.show()
mape_prophet = mape(forecast_df[-forecast_window-prediction_window:-forecast_window]['yhat'],df_p[-prediction_window:]['y'])
rmse_prophet = mean_squared_error(forecast_df[-forecast_window-prediction_window:-forecast_window]['yhat'],df_p[-prediction_window:]['y'], squared=False)

print('PROPHET RMSE: {:.0f} Cases'.format(rmse_prophet))
print('PROPHET MAPE: {:.1f} %'.format(mape_prophet))
"""
**Prophet achives a good MAPE during the last 4 weeks of data, but fails to predict weekly extreme values, such as the very low values of cases found on Sundays.**
"""
"""
# Neural Prophet
"""
"""
**Next we will use a model called Neural Prophet, built using PyTorch neural networks on top of the original Prophet model by facebook**.<br>
The procedure to make a model by using this algorithm will be very similar to the one seen for Prophet.
"""
m = NeuralProphet(
    n_changepoints=50,
    yearly_seasonality=False,
    weekly_seasonality=False,
    daily_seasonality=False,
    seasonality_mode="multiplicative",
    n_forecasts=forecast_window,
    n_lags=forecast_window,
    learning_rate=1.0,
)
m.add_seasonality('weekly_custom', period = 7, fourier_order = 5)
m.add_seasonality('yearly_custom', period = 365, fourier_order = 20)
m.add_country_holidays(country_name='Italy')
metrics = m.fit(df_p, freq="D")
"""
# Neural Prophet Forecasting
"""
future = m.make_future_dataframe(df_p, periods=forecast_window, n_historic_predictions=len(df_p)-forecast_window)
forecast_df_nn = m.predict(future)
forecast_plot = m.plot(forecast_df_nn)
plt.title("COVID19 new cases in Italy with forecasting by Neural Prophet", fontsize=26)
plt.xlabel("")
plt.ylabel("New cases", fontsize=15)
plt.show()
fig2 = m.plot_components(forecast_df_nn)
"""
The components are comparable to those obtained using Prophet.
"""
forecast_plot = m.plot(forecast_df_nn.iloc[-prediction_window:])
plt.title("New COVID cases in Italy by Neural Prophet (last 2 weeks)", fontsize=20)
plt.xlabel("Date")
plt.ylabel("New cases")
plt.show()
"""
**We can see that there are 7 different yhat prediction columns.**
"""
for i in range(7):
    col = 'yhat' + str(i+1)
    mape_test = mape(forecast_df_nn.iloc[-window:-forecast_window][col],df_p.iloc[-prediction_window:]['y'])
    rmse_test = mean_squared_error(forecast_df_nn.iloc[-window:-forecast_window][col],df_p.iloc[-prediction_window:]['y'], squared=False)
    print('PROPHET RMSE {}: {:.0f} Cases'.format(i+1,rmse_test))
    print('PROPHET MAPE {}: {:.1f} %'.format(i+1,mape_test))
"""
**We will create a new 'yhat_avg' column to host an average prediction of the 7 'yhat' values.**
"""
forecast_df_nn['yhat_avg'] = forecast_df_nn[['yhat1', 'yhat2', 'yhat3', 'yhat4', 'yhat5', 'yhat6', 'yhat7']].mean(axis=1)
plt.figure(figsize=(15,6))
plt.title('COVID19 new cases in Italy with forecasting by Neural Prophet', fontsize=20)

#Actual cases
plt.plot(df[-prediction_window:].index, df[-prediction_window:]['nuovi_positivi'], label='Actual cases', marker='o')

#Neural Prophet predicted cases
plt.plot(forecast_df_nn[-window:]['ds'],forecast_df_nn[-window:]['yhat_avg'],color='purple',label='PROPHET-Forecast', marker='o')

plt.axvline(today, linewidth=1.5, color='red', linestyle="--")
plt.legend(loc='upper left', fontsize=14, fancybox=True, shadow=True, frameon=True)
plt.ylabel('New cases', fontsize=20)
plt.xticks(forecast_df[-prediction_window-forecast_window:]['ds'], rotation=80, fontsize=14)
plt.yticks(fontsize=15)

plt.grid(visible=None, axis='x')
plt.show()
mape_prophet_nn = mape(forecast_df_nn.iloc[-window:-forecast_window]['yhat_avg'],df_p.iloc[-prediction_window:]['y'])
rmse_prophet_nn = mean_squared_error(forecast_df_nn.iloc[-window:-forecast_window]['yhat_avg'],df_p.iloc[-prediction_window:]['y'], squared=False)
print('PROPHET RMSE : {:.0f} Cases'.format(rmse_prophet_nn))
print('PROPHET MAPE : {:.1f} %:'.format(mape_prophet_nn))
"""
**Neural Prophet seems to fit the data very well and make reasonable predictions for the next days.**
"""
"""
# SARIMAX
"""
"""
**The third model we will develop is a SARIMAX model.<br>**
Before starting the modeling, we first check the time series' stationarity performing the Augmented Dickey Fuller test (ADF), also to understand the best value for the D parameter of the model (integrative term).
"""
result=adfuller(df['nuovi_positivi'].dropna())
print(f'ADF Statistics:{result[0]}')
print(f'p-value:{result[1]}')
"""
The p-value is higher than 0.05. This means that the time serie is non stationary with a confidence of 95%. Next we will check if with a one step differentiation, the time serie become stationary (in terms of a trendless time series).
"""
result=adfuller(df['nuovi_positivi'].diff().dropna())
print(f'ADF Statistics:{result[0]}')
print(f'p-value:{result[1]}')
"""
After a 1-order difference the p-value is lower than 0.05.
"""
"""
## ACF AND PACF
"""
"""
To have a better idea of possible the autoregressive parameter (p) and moving average parameter (q) of the SARIMAX model, we can check the auto-correlation function (ACF) and partial auto-correlation function (PACF).
"""
fig, (ax1, ax2)=plt.subplots(2,1,figsize=(8,8))

plot_acf(df['nuovi_positivi'],lags=30, zero=False, ax=ax1)
plot_pacf(df['nuovi_positivi'],lags=30, zero=False, ax=ax2)
plt.show()
"""
The series looks indeed non stationary from these plots, and we cannot easily identify good values of p and q. <br>
For this reason we will use the convenient auto *arima module* to find good parameters for the sarimax model.
"""
"""
## AUTO ARIMA
"""
results=pm.auto_arima(df['nuovi_positivi'], start_p=0, d=None, start_q=0, max_p=3, max_q=3,
                      seasonal=True, m=7, D=None, test='adf', start_P=0, start_Q=0, max_P=3, max_Q=3,
                      information_criterion='aic', trace=True, error_action='ignore',
                      trend=None, exog=df['holiday'],with_intercept=True, stepwise=True)
model=SARIMAX(df['nuovi_positivi'], order=(0,1,1), seasonal_order=(2,0,0,7), exog = df['holiday'])
results=model.fit()
results.summary()
results.plot_diagnostics(figsize=(8,8))
plt.show()
"""
These plots indicate a good but improvable model. This is probably due to the high variability of the time series among the different waves.<br>
"""
"""
# SARIMAX Forecasting
"""
"""
## SARIMAX Prediction
"""
"""
Our goal now is to create a dataframe which will host the model test predictions (last 4 weeks of data) and the forecast values (future 7 days).<br>
We start by creating the prediction dataset.
"""
prediction = results.get_prediction(start=-prediction_window, exog = df['holiday'])
mean_prediction = prediction.predicted_mean
"""
Moreover, we also get the confidence intervals from the sarimax prediction
"""
confi_int_p = prediction.conf_int()
lower_limits_p = confi_int_p.iloc[:,0]
upper_limits_p = confi_int_p.iloc[:,1]
"""
Then we create a new dataframe which will include 3 columns: predicted value 'yhat' and the upper and lower values for yhat (confidence interval)
"""
lower_today = np.full([1, prediction_window], np.nan).flatten() #empty list with length = 28 (4 weeks)
upper_today = np.full([1, prediction_window], np.nan).flatten() #empty list with length = 28 (4 weeks)
"""
We also define the confidence interval for the prediction of today's new cases: if we dont do so, the following plot will have a 'gap' for the forecast value.
"""
lower_today[-1] = confi_int_p.iloc[:,0][-1] # lower value for prediction of todays value
upper_today[-1] = confi_int_p.iloc[:,1][-1] # upper value for prediction of todays value
sarimax_prediction = pd.DataFrame({'yhat':mean_prediction, 'y_lower':lower_today,'y_upper': upper_today})
"""
## SARIMAX FORECAST
"""
"""
We will create a forecast dataframe similarly how we created the prediction dataframe.
"""
forecast = results.get_forecast(steps=forecast_window, exog = df['holiday'].iloc[-forecast_window:])
mean_forecast=forecast.predicted_mean
#Confidence Intervals for forecasting
confi_int_f=forecast.conf_int()
lower_limits_f=confi_int_f.iloc[:,0]
upper_limits_f=confi_int_f.iloc[:,1]
sarimax_forecast = pd.DataFrame({'yhat':mean_forecast, 'y_lower':lower_limits_f,'y_upper':upper_limits_f})
"""
Finally we append the forecast dataframe to the prediction dataframe.
"""
sarimax_results = sarimax_prediction.append(sarimax_forecast)
plt.figure(figsize=(16,6))
plt.title('COVID19 new cases in Italy with forecasting by SARIMAX', fontsize=30)


#Actual cases
plt.plot(df[-prediction_window:].index,df[-prediction_window:]['nuovi_positivi'], label='Actual cases', marker='o')

#sarimax
plt.plot(sarimax_results.index, sarimax_results.yhat,color='purple',label='SARIMAX', marker='o')
plt.fill_between(sarimax_results[-forecast_window-1:].index, sarimax_results[-forecast_window-1:].y_lower, sarimax_results[-forecast_window-1:].y_upper, color='purple', alpha=0.1)

#text = 'Today\'s new cases : {:.0f}\nTomorrows new cases : {:.0f}'.format(float(df['nuovi_positivi'][-1:]),mean_forecast[0])
#plt.text(today + pd.DateOffset(days=1), np.min(df[-prediction_window:]['nuovi_positivi']), text, bbox=dict(facecolor='white', alpha=1), fontsize=14)


plt.axvline(today, linewidth=1.5, color='red', linestyle="--")
#plt.text(today, np.max(mean_forecast), 'Forecast->', bbox=dict(facecolor='white', alpha=1),fontsize=17)

plt.legend(loc='upper left', fontsize=14, fancybox=True, shadow=True, frameon=True)
plt.ylabel('New cases', fontsize=20)
plt.xticks(forecast_df[-prediction_window-forecast_window:]['ds'], rotation=80, fontsize=14)
plt.yticks(fontsize=15)
plt.grid(visible=None, axis='x')
plt.show()
mape_sarimax= mape(df[-prediction_window:]['nuovi_positivi'], mean_prediction.values)
rmse_sarimax = mean_squared_error(df[-prediction_window:]['nuovi_positivi'], mean_prediction.values, squared=False)

print('SARIMAX RMSE: {:.0f} Cases'.format(rmse_sarimax))
print('SARIMAX MAPE: {:.1f} %'.format(mape_sarimax))
"""
**SARIMAX seems to have predicted decently last weeks data, with high errors on the previous weeks. Nonetheless, the predicted cases for the next days look reasonable compared to the current week cases.**
"""
"""
# Results Summary
"""
pd.DataFrame({'Model':['Prophet','Neural Prophet','SARIMAX'],'MAPE': [mape_prophet,mape_prophet_nn,mape_sarimax],'RMSE':[rmse_prophet,rmse_prophet_nn,rmse_sarimax]}).set_index('Model')
plt.figure(figsize=(18,8))
plt.title('COVID19 new cases in Italy with Forecasted cases by ML models', fontsize=28)
#Actual cases
plt.bar(df[-prediction_window:].index, df[-prediction_window:]['nuovi_positivi'], label='Actual cases', color='silver', edgecolor="black", linewidth=2.0, alpha=0.6)
#PROPHET
plt.plot(forecast_df[-window:]['ds'],forecast_df[-window:]['yhat'],color='blue',label='Prophet', marker='o',linewidth=3.0)
plt.fill_between(forecast_df[-forecast_window-1:]['ds'], forecast_df[-forecast_window-1:]['yhat_lower'],forecast_df[-forecast_window-1:]['yhat_upper'], color='blue', alpha=0.1)

#NEURAL PROPHET
plt.plot(forecast_df_nn[-window:]['ds'],forecast_df_nn[-window:]['yhat_avg'],color='green',label='Neural Prophet', marker='o',linewidth=3.0)

#sarimax
plt.plot(sarimax_results.index, sarimax_results.yhat,color='red',label='SARIMAX', marker='o',linewidth=3.0)
plt.fill_between(sarimax_results[-forecast_window-1:].index, sarimax_results[-forecast_window-1:].y_lower, sarimax_results[-forecast_window-1:].y_upper, color='red', alpha=0.1)


text = 'Today\'s new cases : {:.0f}\nTomorrow\'s new cases:\nProphet : {:.0f}\nNeural Prophet : {:.0f}\nSARIMAX : {:.0f}'.format(float(df['nuovi_positivi'][-1:]),forecast_df[forecast_df['ds']==str(tomorrow)]['yhat'].values[0],forecast_df_nn[forecast_df_nn['ds']==str(tomorrow)]['yhat_avg'].values[0],mean_forecast[0])
plt.text(today + pd.DateOffset(days=1), 180000, text, bbox=dict(facecolor='white', alpha=1), fontsize=17)

plt.axvline(today, linewidth=4, color='black', linestyle="--")
plt.legend(fontsize=12, bbox_to_anchor=(0.12, 1.0), fancybox=True, shadow=True, frameon=True)
plt.ylabel('New cases', fontsize=20)
plt.xticks(forecast_df[-prediction_window-forecast_window:]['ds'], rotation=80, fontsize=14)
plt.yticks(fontsize=15)
plt.grid(visible=None, axis='x')

plt.show()
print('Today\'s new cases : {:.0f}'.format(float(df['nuovi_positivi'][-1:])))
print('Tomorrows new cases according to SARIMAX : {:.0f}'.format(mean_forecast[0]))
print('Tomorrows new cases according to PROPHET : {:.0f}'.format(forecast_df[forecast_df['ds']==str(tomorrow)]['yhat'].values[0]))
print('Tomorrows new cases according to Neural PROPHET : {:.0f}'.format(forecast_df_nn[forecast_df_nn['ds']==str(tomorrow)]['yhat_avg'].values[0]))
"""
**Overall, we can see that Neural Prophet and SARIMAX predict and forecast similar values, which are higher compared to those predicted and forecasted by Prophet.**<br>
"""
"""
Finally, we will create a final dataframe that will host the average prediction among the three models.
"""
df_final = pd.DataFrame({'date':forecast_df_nn[-window:]['ds'], 'y_p':forecast_df[-window:]['yhat'], 'y_p_nn':forecast_df_nn[-window:]['yhat_avg'], 'y_s':sarimax_results['yhat'].values})
df_final = df_final.set_index('date')
df_final['y_avg'] = df_final.mean(axis=1)
plt.figure(figsize=(16,6))
plt.title('COVID19 new cases in Italy with AVG Forecasted cases', fontsize=30)
#Actual cases
plt.bar(df[-prediction_window:].index, df[-prediction_window:]['nuovi_positivi'], label='Actual cases', color='black',alpha=1)

#AVG Forecast
plt.bar(df_final.index, df_final.y_avg,label='AVG cases by ML models', color='#00FF00',alpha=0.6, edgecolor="black", linewidth=2.0)

text = 'New Cases:\nToday: {:.0f}\nTomorrow {:.0f}'.format(float(df['nuovi_positivi'][-1:]),df_final[-forecast_window:]['y_avg'].values[0])
plt.text(today + pd.DateOffset(days=3), 7000, text, bbox=dict(facecolor='white', alpha=1), fontsize=17)

plt.axvline(today, linewidth=2.5, color='red', linestyle="--")
plt.legend(loc='upper right', fontsize=14,  fancybox=True, shadow=True, frameon=True)
plt.ylabel('New cases', fontsize=20)
plt.xticks(forecast_df[-prediction_window-forecast_window:]['ds'], rotation=80, fontsize=14)
plt.yticks(fontsize=15)
plt.grid(visible=None, axis='x')

plt.show()
"""
**Overall, we can see that the average predicted values fits well the last 4 weeks data. Moreover, we can see the weekly trend and holiday effect on forecasted values.**
"""
"""
**I will update this notebook daily with the new data and add deeper analysis. Thanks for reading the notebook :)**
"""