# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# +
import pyreadr
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import statsmodels.api as sm
from statsmodels.regression.linear_model import OLS
from statsmodels.tools import add_constant
from sklearn.linear_model import LinearRegression as lm
from sklearn.preprocessing import scale
from sklearn.decomposition import PCA
from sklearn.ensemble import RandomForestRegressor

embed = lambda x, d : np.hstack([x.shift(i) for i in range(d)])[d - 1:] # R's embed(x, d)

Y = pyreadr.read_r('rawdata.rda')['dados']
nprev=132


# -

def runrf(window, indice, lag):
    
    pca = PCA()
    SCALE = scale(window, with_mean=True, with_std=False)
    pca.fit(SCALE)
    scores = pca.transform(SCALE)

    WC = pd.concat([window, pd.DataFrame(scores[:, :4], index=window.index)], axis=1)
    aux = embed(WC, lag + 4)
    y = aux[:, indice]
    X = np.delete(aux, range(WC.shape[1] * lag), axis=1)

    if lag == 1:
        out = aux[-1, :X.shape[1]]
    else:
        out = np.delete(aux, range(WC.shape[1] * (lag-1)), axis=1)
        out = out[-1, :X.shape[1]]

    y = y[:len(y)-lag+1]
    X = X[:len(X)-lag+1]

    model = RandomForestRegressor(random_state = 15)
    model.fit(X, y)
    pred = model.predict([out])

    return model, pred


def rf_rolling_window(Y, nprev, indice=1, lag=1, graph=False):
    
    importance = []
    pred = np.empty(nprev)*np.nan

    for i in range(nprev):
        window = Y.iloc[i:len(Y)-(nprev-i)] # The last obs. is not inclued in the last iter.
        model, pred[i] = runrf(window, indice, lag)
        importance.append(model.feature_importances_)
        print(f"iteration {i}")

    comp = Y.iloc[:, indice].to_frame('actual')
    comp['pred'] = np.concatenate((np.empty(len(Y)-nprev)*np.nan, pred))

    if graph:
        fig, ax = plt.subplots(1,1,figsize=(12, 6))
        comp.plot(ax=ax)

    comp = comp.dropna()
    rmse = np.sqrt(np.mean((comp.actual - comp.pred)**2))
    mae = np.mean(np.abs(comp.actual - comp.pred))
    errors = {'rmse':rmse, 'mae':mae}

    return {'pred':comp.pred, 'importance':importance, 'errors':errors}


# +
DF = pd.DataFrame()

for name, index in [('c', 0), ('p', 1)]:
    df = pd.DataFrame()
    for l in range(12):
        re = rf_rolling_window(Y, nprev, index, l+1, graph=False)
        df = pd.concat([df, re['pred'].to_frame(l+1)], axis=1)
    df = pd.concat([df], axis = 1, keys=[name])
    DF = pd.concat([DF, df], axis=1)
# -



StarcoderdataJupyterScriptsDedupFiltered