"""
This is a simple example of how to extract finance data, specifically forex exchange data, and to test some simple prediction algoritms. It was intended for my education. Any suggestion/comment is more than welcomed!.  
"""
"""
## Installing packages and loading general libraries
"""
### forex_python page https://forex-python.readthedocs.io/en/latest/usage.html
! pip install forex_python
### yfinance page https://pypi.org/project/yfinance/
! pip install yfinance
import numpy as np # linear algebra
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)
import matplotlib.pyplot as plt
import seaborn as sns
sns.set()

from datetime import datetime, date
from forex_python.converter import get_rate
import yfinance as yf
"""
## Basic forex analysis using `forex_python`
"""
"""
Here, I started using `forex_python` to access forex exchange info. It was very useful since I could access many exchange rates at once for one currency. However, it takes some time to download the data. 
Since later on I found about `yfinance`, which is faster and includes more information, the next cells are commented. I keep them for bookeeping purposes.
"""
# df = {}
# df['date'] = []
# df['USDCHF'] = []
# df['USDGBP'] = []
# df['CHFGBP'] = []

# start_date = date(2022, 1, 1)
# for idate in pd.date_range(start_date, date.today(), freq='5B'):
#     df['date'].append( pd.to_datetime(idate) )
#     df['USDCHF'].append(get_rate("USD", "CHF", idate))
#     df['USDGBP'].append(get_rate("USD", "GBP", idate))    
#     df['CHFGBP'].append(get_rate("CHF", "GBP", idate))        
    
# df = pd.DataFrame(df)
# df.to_csv('forex_simple.csv')
# df.head()
# sns.lineplot(data=df) #, kind='line') #, x='date', y=['USDCHF', 'USDGBP'])
# from forex_python.converter import CurrencyRates

# def downloadRates( currency='USD', start_date=date(2022, 1, 1), frequency='5B' ):
#     listDf = []
#     for idate in pd.date_range(start_date, date.today(), freq=frequency):
#         tmp = pd.DataFrame( [ CurrencyRates().get_rates(currency, idate)] )
#         tmp['date'] = pd.to_datetime( idate )
#         listDf.append(tmp)
#     finalDf = pd.concat(listDf)
    
#     finalDf.to_csv(f'{currency}rates_{start_date}_{frequency}.csv')
#     return finalDf
# downloadRates( currency='USD', start_date=date(2015, 1, 1), frequency='5B' )
# downloadRates( currency='CHF', start_date=date(2015, 1, 1), frequency='5B' )
"""
Here, the cvs dataset was collected from the previous steps but since I didn't want to run again and again the same cells, I include those files as kaggle datasets.
"""
tmp = pd.read_csv('../input/currency-exchange-from-2015/USDrates_2015-01-01_5B.csv')
tmp.drop(columns=['Unnamed: 0', 'LTL', 'ISK', 'ILS'], inplace=True)
tmp['date'] = pd.to_datetime( tmp['date'] )
tmp['USD'] = 1.0
tmp.head()
tmp.info()
# tmp['ILS'].unique()
plt.figure(figsize=(20,10))
sns.lineplot(x='date', y='value', hue='variable', data=pd.melt(tmp, ['date'])).set(yscale='log')
plt.figure(figsize=(20,10))
tmp1 = tmp.set_index('date')
tmp1 = tmp1[tmp1<2]
tmp1.reset_index(inplace=True)
tmp1.dropna(axis='columns',inplace=True)
# tmp1
# tmp1 = tmp[(tmp['date']>'2020-1-1' & tmp>1000)]
sns.lineplot(x='date', y='value', hue='variable', data=pd.melt(tmp1, ['date'])) #.set(yscale='log')
"""
### Simple Linear regressions 
"""
from sklearn.model_selection import train_test_split
from sklearn.linear_model import LinearRegression
from sklearn.linear_model import LogisticRegression
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import StandardScaler
from sklearn.preprocessing import Normalizer
from sklearn.metrics import mean_absolute_error
rawdata = tmp.select_dtypes('float64').drop(columns='USD')

normdata = Normalizer().fit_transform(rawdata)
normdata = pd.DataFrame(normdata)
normdata.set_axis(rawdata.columns.to_list(), axis=1, inplace=True)
normdata['date'] = tmp['date']

normdata['return'] = normdata['CHF'] - normdata['CHF'].shift(-1)
normdata['label'] = normdata['return'].shift(-1)
normdata['label'] = normdata['label'].apply(lambda x: 1 if x>0.0 else 0)
normdata.dropna(axis=0, inplace=True)
Y = normdata['label']
X = normdata.drop(columns='label')
normdata.plot(x='date', y='return', figsize=(10,4))
x_train, x_test, y_train, y_test = train_test_split(X.drop(columns='date'), Y, train_size=0.8, test_size=0.2, random_state=1)

model = make_pipeline( Normalizer(), 
                     LinearRegression())
# model = LinearRegression()
model.fit(x_train, y_train)
print(model.score(x_train, y_train))
y_predict = model.predict(x_test)
mean_absolute_error(y_test, y_predict)
"""
## Simple analysis using `yfinance`
"""
"""
The following steps were inspired by [this notebook](https://github.com/hayatoy/ml-forex-prediction/blob/master/Predict%20Future%20Price%20of%20EURUSD%20with%20sklearn%2C%20yahoo%20data.ipynb).
"""
rawdata = yf.download(tickers = 'CHFUSD=X', start='2010-01-01', end=datetime.today(), period ='1d')
rawdata.reset_index(inplace=True)

# Normalize aclose value
# We use this value to train model
rawdata['return'] = rawdata['Adj Close'] - rawdata['Adj Close'].shift(1)
rawdata['return'] = rawdata['return'] / (rawdata['return'].max() - rawdata['return'].min())

# Make label, 1 as rising price, 0 as falling price
rawdata['label'] = rawdata['return'].shift(-1)
rawdata['label'] = rawdata['label'].apply(lambda x: 1 if x>0.0 else 0)
rawdata.drop(columns='Volume', inplace=True)
rawdata.dropna(inplace=True)
rawdata.tail()
rawdata = rawdata[ np.abs(rawdata['return'])<0.2 ]
data_toPred = rawdata[ rawdata['Date']>'2021-1-1' ]
display(data_toPred)
data = rawdata[ rawdata['Date']<'2021-1-1' ]
data.tail()
# data['test'] = data['High'] -data['Low']
# data.plot(x='Date', y='test', figsize=(15,5))
data.plot(x='Date', y='Low', figsize=(15,5))
data.plot(x='Date', y='return', figsize=(15,5))
x_train, x_test, y_train, y_test = train_test_split(data.drop(columns=['label','Date']), data['label'], 
                                                    train_size=0.8, test_size=0.2, random_state=1, shuffle=False)

# model = make_pipeline( Normalizer(), 
#                      LinearRegression())
# model = LinearRegression()
model = LogisticRegression()
model.fit(x_train, y_train)
print(model.score(x_train, y_train))
y_predict = model.predict(x_test)
mean_absolute_error(y_test, y_predict)
from sklearn.ensemble import GradientBoostingClassifier
clf = GradientBoostingClassifier(random_state=0, learning_rate=0.01, n_estimators=10000)

clf.fit(x_train, y_train)
print(clf.score(x_test, y_test))
y_predict = clf.predict(x_test)
mean_absolute_error(y_test, y_predict)
def calc_profit(row):
    if row['won']:
        return abs(row['return'])*contracts - commission
    else:
        return -abs(row['return'])*contracts - commission
# Calculate equity..

contracts  = 1000.0
commission = 0.0


df_trade = data_toPred.drop(columns=['Date','label'])
df_trade['pred']   = clf.predict(df_trade)
df_trade['label']  = data_toPred['label']
df_trade['won']    = df_trade['label'] == df_trade['pred']
df_trade['return'] = df_trade['return'].shift(-1) * (rawdata['return'].max() - rawdata['return'].min())
# df_trade.drop(df_trade.index[len(df_trade)-1], inplace=True)
df_trade['pnl'] = df_trade.apply(lambda row: calc_profit(row), axis=1)
df_trade['equity'] = df_trade['pnl'].cumsum()
df_trade['Date'] = data_toPred['Date']

display(df_trade.tail())
# df_trade.plot( y='equity', figsize=(15,5), title=f'Backtest with ${contracts} initial capital')
df_trade.plot(x='Date', y='equity', figsize=(15,5), title=f'Backtest with ${contracts} initial capital')
plt.xlabel('Trades')
plt.ylabel('Equity (USD)')
# for r in df_trade.iterrows():
#     if r[1]['won']:
#         plt.axvline(x=r[0], linewidth=0.5, alpha=0.8, color='g')
#     else:
#         plt.axvline(x=r[0], linewidth=0.5, alpha=0.8, color='r')
df_trade