# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3 (ipykernel)
#     language: python
#     name: python3
# ---

# +
import investpy
# # !pip install yfinance

import yfinance as yf
import pandas as pd
import numpy as np
import datetime
import pandas_datareader as pdr
import investpy
import sklearn
from sklearn import datasets
from sklearn.datasets import make_moons
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import scale
import warnings
# from warnings import filterwarnings

import arviz as az
import matplotlib
import numpy as np
import pymc3 as pm
import scipy as sp
import scipy.sparse
import scipy.sparse.linalg
import seaborn as sn


from matplotlib import pyplot as plt
print(f"Running on PyMC3 v{pm.__version__}")

# +
# TODO: Need DXY (US Dollar) monthly data
# TODO: US10yr Treasury Maturity Rate - Need to replace with monthly data
# TODO: BUND Monthy data 
# XLK: Tech Monthly data - ETF
# XLF: Finance Monthly data - ETF
# BTC: Monthly data - ETF

# Headline CPI - yoy% change monthly and quarterly
# Consumer Price Index for All Urban Consumers: Energy in U.S. City Average (CPIENGSL) MONTHLY: Seasonally Adjusted
# Personal Consumption Expenditures Excluding Food and Energy (Chain-Type Price Index) (PCEPILFE) MONTHLY: Seasonally Adjusted
# Consumer Price Index for All Urban Consumers: Transportation Services in U.S. City Average (CUSR0000SAS4) MONTHLY: Seasonally Adjusted
# Producer Price Index by Commodity: All Commodities (PPIACO) MONTHLY: Not Seasonally Adjusted

# Setting up the Start and End time 
# series_code = ['CURRCIR', 'DGS10']
data_source = 'fred'

start = datetime.datetime (1960, 1, 1)      #  (2005, 5, 1)
end = datetime.datetime (2030, 12, 1)

CPI_Headline = pdr.DataReader('CPIAUCSL', data_source, start, end)
CPI_Headline.index.names = ['Date']
CPI_Headline.columns = ['CPI_Headline']


# TODO: Need DXY (US Dollar) monthly data
# TODO: BUND Monthy data 

copperProducts = pdr.DataReader('WPUSI019011', data_source, start, end)
copperProducts.index.names = ['Date']
copperProducts.columns = ['Copper/Prod']
iron_steel = pdr.DataReader('WPU101', data_source, start, end)
iron_steel.index.names = ['Date']
iron_steel.columns = ['Iron/Steel']
chemical_man = pdr.DataReader('PCU325325', data_source, start, end)
chemical_man.index.names = ['Date']
chemical_man.columns = ['ChemMan']
con_mach_eq = pdr.DataReader('WPS112', data_source, start, end)
con_mach_eq.index.names = ['Date']
con_mach_eq.columns = ['ConMachEq']

fed_fundsRate = pdr.DataReader('FEDFUNDS', data_source, start, end)
fed_fundsRate.index.names = ['Date']
fed_fundsRate.columns = ['FedFundsRate']
PCE = pdr.DataReader('PCEPI', data_source, start, end)
PCE.index.names = ['Date']
PCE.columns = ['PersonalConExp']

# +
"""Need to turn oil, gold , and silver into monthly 
as well as use gold and silver as a ratio"""

gold = investpy.commodities.get_commodity_historical_data('Gold', 
                                                   from_date = '01/01/1980', 
                                                   to_date = '01/01/2030', 
                                                   country=None, 
#                                                    as_json=False, 
                                                   order='ascending', 
                                                   interval='Monthly')
gold.index.names = ['Date']
silver = investpy.commodities.get_commodity_historical_data('Silver', 
                                                   from_date = '01/01/1980', 
                                                   to_date = '01/01/2030', 
                                                   country=None, 
#                                                    as_json=False, 
                                                   order='ascending', 
                                                   interval='Monthly')
silver.index.names = ['Date']
copper = investpy.commodities.get_commodity_historical_data('Copper', 
                                                   from_date = '01/01/1980', 
                                                   to_date = '01/01/2030', 
                                                   country='united states', 
#                                                    as_json=False, 
                                                   order='ascending', 
                                                   interval='Monthly')
lumber = investpy.commodities.get_commodity_historical_data('Lumber', 
                                                   from_date = '01/01/1980', 
                                                   to_date = '01/01/2030', 
                                                   country=None, 
#                                                    as_json=False, 
                                                   order='ascending', 
                                                   interval='Monthly')
lumber.index.names = ['Date']
nat_gas = investpy.commodities.get_commodity_historical_data('Natural Gas', 
                                                   from_date = '01/01/1980', 
                                                   to_date = '01/01/2030', 
                                                   country=None, 
#                                                    as_json=False, 
                                                   order='ascending', 
                                                   interval='Monthly')
nat_gas.index.names = ['Date']
crude = investpy.commodities.get_commodity_historical_data('Crude Oil WTI', 
                                                   from_date = '01/01/1980', 
                                                   to_date = '01/01/2030', 
                                                   country=None, 
#                                                    as_json=False, 
                                                   order='ascending', 
                                                   interval='Monthly')
crude.index.names = ['Date']
corn = investpy.commodities.get_commodity_historical_data('US Corn', 
                                                   from_date = '01/01/1980', 
                                                   to_date = '01/01/2030', 
                                                   country=None, 
#                                                    as_json=False, 
                                                   order='ascending', 
                                                   interval='Monthly')
corn.index.names = ['Date']
wheat = investpy.commodities.get_commodity_historical_data('US Wheat', 
                                                   from_date = '01/01/1980', 
                                                   to_date = '01/01/2030', 
                                                   country=None, 
#                                                    as_json=False, 
                                                   order='ascending', 
                                                   interval='Monthly')
wheat.index.names = ['Date']
soy = investpy.commodities.get_commodity_historical_data('US Soybeans', 
                                                   from_date = '01/01/1980', 
                                                   to_date = '01/01/2030', 
                                                   country=None, 
#                                                    as_json=False, 
                                                   order='ascending', 
                                                   interval='Monthly')
soy.index.names = ['Date']
sugar = investpy.commodities.get_commodity_historical_data('US Sugar #11', 
                                                   from_date = '01/01/1980', 
                                                   to_date = '01/01/2030', 
                                                   country=None, 
#                                                    as_json=False, 
                                                   order='ascending', 
                                                   interval='Monthly')
sugar.index.names = ['Date']
coffee = investpy.commodities.get_commodity_historical_data('US Coffee C', 
                                                   from_date = '01/01/1980', 
                                                   to_date = '01/01/2030', 
                                                   country=None, 
#                                                    as_json=False, 
                                                   order='ascending', 
                                                   interval='Monthly')
coffee.index.names = ['Date']

US10yr = investpy.bonds.get_bond_historical_data(bond='U.S. 10Y', 
                                        from_date = '01/01/1980', 
                                        to_date = '01/01/2030', 
#                                         country=united states, 
                                        order='ascending', 
                                        interval='Monthly')
US10yr.index.names = ['Date']

DXY = investpy.indices.get_index_historical_data('US Dollar Index', 
                                        from_date = '01/01/1980', 
                                        to_date = '01/01/2030', 
                                        country='united states', 
                                        order='ascending', 
                                        interval='Monthly')
DXY.index.names = ['Date']

# TRCCRB
CRB = investpy.indices.get_index_historical_data('TR/CC CRB', 
                                                 from_date = '01/01/1980', 
                                                 to_date = '01/01/2030', 
                                                 country='world', 
                                                 order='ascending', 
                                                 interval='Monthly')
CRB.index.names = ['Date']                                        
# -

CRB.tail(1)

# +
# investpy.indices.get_indices(country='united states', symbol='TRCCRB')
# investpy.indices.search_indices(by='symbol', value='TRCCRB')

# +
# Dropping columns
drop_cols = ['Open', 'High', 'Low', 'Currency']

gold = gold.drop(drop_cols, axis=1)
gold.columns = ["Gold", "GoldVolm"]
silver = silver.drop(drop_cols, axis=1)
silver.columns = ["Silver", "SilverVolm"]

tempMerge = pd.merge(silver, gold, left_index=True, right_index=True)
tempMerge['sil_goldRatio'] = tempMerge["Silver"] / tempMerge["Gold"]
sil_goldRatio = tempMerge.copy()
sil_goldRatio = sil_goldRatio.drop(["Silver", "Gold"], axis=1)
sil_goldRatio['DiffSilGold_Volm'] = sil_goldRatio['GoldVolm'] - sil_goldRatio['SilverVolm'] 

sil_goldRatio.columns = ["SilverVolm", "GoldVolm", "Sil_GoldRatio", "DiffSilGold_Volm"]
copper = copper.drop(drop_cols, axis=1)
copper.columns = ["Copper", "CopperVolm"]
lumber = lumber.drop(drop_cols, axis=1)
lumber.columns = ["Lumber", "LumberVolm"]
nat_gas = nat_gas.drop(drop_cols, axis=1)
nat_gas.columns = ["NatGas", "NatGasVolm"]
crude = crude.drop(drop_cols, axis=1)
crude.columns = ["WTIcrude", "CrudeVolm"]
corn = corn.drop(drop_cols, axis=1)
corn = corn.drop(["Volume"], axis = 1)
corn.columns = ["Corn"]
wheat = wheat.drop(drop_cols, axis=1)
wheat = wheat.drop(["Volume"], axis=1)
wheat.columns = ["Wheat"]
soy = soy.drop(drop_cols, axis=1)
soy = soy.drop(["Volume"], axis=1)                    
soy.columns = ["Soy"]
sugar = sugar.drop(drop_cols, axis=1)
sugar.columns = ["Sugar", "SugarVolm"]
coffee = coffee.drop(drop_cols, axis=1)
coffee.columns = ["Coffee", "CoffeeVolm"]

# US10yr
US10yr = US10yr.drop(['Open', 'High', 'Low'], axis=1)
US10yr.columns = ["US10yr"]

DXY = DXY.drop(['Open', 'High', 'Low', 'Volume', 'Currency'], axis=1)
DXY.columns = ["DXY"]

CRB = CRB.drop(['Open', 'High', 'Low', 'Volume', 'Currency'], axis=1)
CRB.columns = ['CRB']
# coffee.columns = ["Coffee", "CoffeeVolm"]
# -

CRB.tail()

sil_goldRatio.tail()


# +
def roc(df, n):
    """Calulates the 2nd Derivative (Rate of Change/Acceleration)"""    
    df1 = df.copy()
    df1 = (np.log(df1).diff(n))
    df1 = df1 - (df1.shift(1))
    df1 = df1 * 100
    return df1


def percentChg(df, n):
    df1 = df.copy()
    df1 = df1.pct_change(n)
    return df1


# def convert_to_quarterly(df):
#     """Aggregates 3 months of data to a quarterly value."""
#     df1 = df.copy()
#     df1.date = pd.to_datetime(df1['Date'])
# #     df = df.set_index("DATE")
#     agg_columns = ['data_1', 'data_2']
#     extra_cols = [x for x in df.columns if x not in agg_columns]
#     df_out = df.resample("QS-JUN")[agg_columns].sum()
#     df_out.index = df_out.index + MonthEnd(1)
#     df_out[extra_cols] = df[extra_cols]  
#     return df_out

# data = convert_to_quarterly(data)
# -

PCE_roc = roc(PCE, 12)
print("Max:", PCE_roc.max())
PCE_roc.tail(1)

sil_gold_roc = roc(sil_goldRatio, 12)
print("Max:", sil_gold_roc['Sil_GoldRatio'].max())
sil_gold_roc.tail(1)

copper_roc = roc(copper, 12)
print("Max:", copper_roc.max())
copper_roc.tail(1)

FedFunds_roc = roc(fed_fundsRate, 12)
print("Max:", FedFunds_roc.max())
FedFunds_roc.tail(1)

lumber_roc = roc(lumber, 12)
print("Max:", lumber_roc.max())
lumber_roc.tail(1)

CPI_Head_roc = roc(CPI_Headline, 12)
print("Max:", CPI_Head_roc.max())
CPI_Head_roc.tail(1)

corn_roc = roc(corn, 12)
print("Max:", corn_roc.max())
corn_roc.tail(1)

wheat_roc = roc(wheat, 12)
print("Max:", wheat_roc.max())
wheat_roc.tail(1)

soy_roc = roc(soy, 12)
print("Max:", soy_roc.max())
soy_roc.tail(1)

sugar_roc = roc(sugar, 12)
print("Max:", sugar_roc.max())
sugar_roc.tail(1)

coffee_roc = roc(coffee, 12)
print("Max:", coffee_roc.max())
coffee_roc.tail(1)

nat_gas_roc = roc(nat_gas, 12)
print("Max:", nat_gas_roc.max())
nat_gas_roc.tail(1)

DXY_roc = roc(DXY, 12)
print("Max:", DXY_roc.max())
DXY_roc.tail(1)

US10yr_roc = roc(US10yr, 12)
print("Max:", US10yr_roc.max())
US10yr_roc.tail(1)

CRB_roc = roc(CRB, 12)
print("Max:", CRB_roc.max())
CRB.tail(1)

# ## These seem to be leading indicators of Headline CPI 

con_mach_eq_roc = roc(con_mach_eq, 12)
print("Max:", con_mach_eq_roc.max())
con_mach_eq_roc.tail(1)

chemicalManuf_roc = roc(chemical_man, 12)
print("Max:", chemicalManuf_roc.max())
chemicalManuf_roc.tail(1)

crude_roc = roc(crude, 12)
print("Max:", crude_roc['WTIcrude'].max())
crude_roc.tail(1)

ironSteel_roc = roc(iron_steel, 12)
print("Max:", ironSteel_roc.max())
ironSteel_roc.tail(1)

sil_gold_roc = roc(sil_goldRatio, 12)
print("Max:", sil_gold_roc['Sil_GoldRatio'].max())
sil_gold_roc.drop(['SilverVolm', 'GoldVolm', 'DiffSilGold_Volm'], axis=1, inplace=True)
sil_gold_roc.tail(1)

# +
# copperProd_roc = roc(copperProducts, 12)
# print("Max:", copperProd_roc.max())
# copperProd_roc.tail(15)

# +

inflation_roc = pd.merge(DXY_roc, US10yr_roc, left_index=True, right_index=True)
inflation_roc= pd.merge(inflation_roc, CRB_roc, left_index=True, right_index=True)
inflation_roc = pd.merge(inflation_roc, sil_gold_roc, left_index=True, right_index=True)

# inflation_roc['baseEffect_roc'] = inflation_roc['CPI_Headline'].shift(24)

inflation_roc = pd.merge(inflation_roc, crude_roc, left_index=True, right_index=True)
inflation_roc = pd.merge(inflation_roc, lumber_roc, left_index=True, right_index=True)
inflation_roc = pd.merge(inflation_roc, nat_gas_roc, left_index=True, right_index=True)
# inflation_roc = pd.merge(inflation_roc, crude_roc, left_index=True, right_index=True)
inflation_roc = pd.merge(inflation_roc, corn_roc, left_index=True, right_index=True)
inflation_roc = pd.merge(inflation_roc, wheat_roc, left_index=True, right_index=True)
inflation_roc = pd.merge(inflation_roc, soy_roc, left_index=True, right_index=True)
inflation_roc = pd.merge(inflation_roc, sugar_roc, left_index=True, right_index=True)
inflation_roc = pd.merge(inflation_roc, coffee_roc, left_index=True, right_index=True)
inflation_roc = pd.merge(inflation_roc, copper_roc, left_index=True, right_index=True)

# inflation_roc = pd.merge(inflation_roc, PCE_roc, left_index=True, right_index=True)
# inflation_roc = pd.merge(inflation_roc, ironSteel_roc, left_index=True, right_index=True)
# inflation_roc = pd.merge(inflation_roc, chemicalManuf_roc, left_index=True, right_index=True)
# inflation_roc = pd.merge(inflation_roc, copperProd_roc, left_index=True, right_index=True)
# inflation_roc = pd.merge(inflation_roc, con_mach_eq_roc, left_index=True, right_index=True)
inflation_roc.drop(["CrudeVolm", "LumberVolm", "NatGasVolm", "SugarVolm", "CoffeeVolm", "CopperVolm"], axis=1, inplace=True)
print(inflation_roc.shape)
inflation_roc.tail(5)
# -

inflation_roc.corr()

# +
# Merging Inflation 'Possible' Factors - Commodities

inflation_factors = pd.merge(DXY, US10yr, left_index=True,right_index=True)
inflation_factors = pd.merge(inflation_factors, CRB, left_index=True, right_index=True)
inflation_factors = pd.merge(inflation_factors, sil_goldRatio, left_index=True, right_index=True)
inflation_factors.drop(['SilverVolm', 'GoldVolm', 'DiffSilGold_Volm'], axis=1, inplace=True)

# inflation_factors['baseEffect'] = inflation_factors['CPI_Headline'].shift(24)
# inflation_factors['baseEffect_2yr'] = (inflation_factors['CPI_Headline'] - inflation_factors['baseEffect']) / inflation_factors['baseEffect'] * 100
# inflation_factors.drop(['baseEffect'], axis=1, inplace=True)

# inflation_factors['Inflation_yoy%'] = percentChg(inflation_factors['CPI_Headline'], 12)
# inflation_factors.drop(['CPI_Headline'], axis=1, inplace=True)

inflation_factors = pd.merge(inflation_factors, copper, left_index=True, right_index=True)
inflation_factors = pd.merge(inflation_factors, lumber, left_index=True, right_index=True)
inflation_factors = pd.merge(inflation_factors, nat_gas, left_index=True, right_index=True)
inflation_factors = pd.merge(inflation_factors, crude, left_index=True, right_index=True)
inflation_factors = pd.merge(inflation_factors, corn, left_index=True, right_index=True)
inflation_factors = pd.merge(inflation_factors, wheat, left_index=True, right_index=True)
inflation_factors = pd.merge(inflation_factors, soy, left_index=True, right_index=True)
inflation_factors = pd.merge(inflation_factors, sugar, left_index=True, right_index=True)
inflation_factors = pd.merge(inflation_factors, coffee, left_index=True, right_index=True)

inflation_factors.drop(["CrudeVolm", "LumberVolm", "NatGasVolm", "SugarVolm", "CoffeeVolm", "CopperVolm"], axis=1, inplace=True)


# inflation_factors = pd.merge(inflation_factors, PCE, left_index=True, right_index=True)

print(inflation_factors.shape)
inflation_factors.tail(15)
# -

inflation_factors.corr()

inflation_factors.columns

inflation_factors.to_csv('inflation_factors.csv')

inflation_roc.to_csv('inflation_roc.csv')

data = inflation_factors.copy()
data.head()

CPI_Headline['CPI_yoy%'] = percentChg(CPI_Headline, 12)

data = pd.merge(CPI_Headline, data, left_index=True, right_index=True)
data.head()

CPI = data[['CPI_Headline']]
CPI.head()


# +
def roc(df, n):
    """Calulates the First Difference of the 2nd Derivative (Rate of Change/Acceleration)"""    
    df1 = df.copy()
    df1 = (np.log(df1).diff(n))
    df1 = df1 - (df1.shift(1))
    df1 = df1 * 100
    return df1

def ROC(df, n):
    """Calulates the 2nd Derivative (Rate of Change/Acceleration)"""    
    df1 = df.copy()
    df1 = np.log(df1 / df1.shift(n))
    df1 = df1 * 100
    return df1
# -

# CPI_Headline['CPI_Headline'].diff(12).tail(13)


# data.head()
CPI_Headline = pdr.DataReader('CPIAUCSL', data_source, start, end)
CPI_Headline.index.names = ['Date']
CPI_Headline.columns = ['CPI_Headline']
CPI_Headline.dropna(inplace=True)
CPI_Headline['CPI_yoy%'] = percentChg(CPI_Headline, 12)
CPI_Headline.tail(13)

# CPI_Headline['CPI_yoy_roc'] = ROC(CPI_Headline['CPI_Headline'], 12)
CPI_Headline['CPI_yoyFirstDiffroc'] = roc(CPI_Headline['CPI_Headline'], 12)
# CPI_Headline['mean_yoy2'] = CPI_Headline['CPI_yoy%'].rolling(2).mean()
CPI_Headline.tail(40)

# +
plt.style.use('fivethirtyeight')
fig, ax = plt.subplots(figsize=(30,10))  # chart 3:1

ax.set_title("CPI", 
             fontsize=22, fontweight='bold', x=.242, y=1.07)
# plt.plot(CPI_Headline.index, CPI_Headline['mean_yoy2'], color='red', linewidth=1)
plt.plot(CPI_Headline.index, CPI_Headline['CPI_yoy%'], color='black', linewidth=1)
plt.plot(CPI_Headline.index, CPI_Headline['CPI_yoyFirstDiffroc'], color='green', linewidth=3)
# plt.plot(vix['Close'].rolling(90).mean(), color='r', linewidth=1)

# Turn off the spines
ax.spines['left'].set_visible(False)
ax.spines['right'].set_visible(False)
ax.spines['top'].set_visible(False)
ax.spines['bottom'].set_visible(False)

plt.show()
# -

# ## Variational Inference: Bayesian Neural Network





# ## Factor Analysis

# +
# CRB_d = investpy.indices.get_index_historical_data('TR/CC CRB', 
#                                                  from_date = '01/01/1980', 
#                                                  to_date = '01/01/2030', 
#                                                  country='world', 
#                                                  order='ascending', 
#                                                  interval='weekly')
# CRB_d.tail(20)
# -







StarcoderdataJupyterScriptsDedupFiltered