# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python (traders_nlp)
#     language: python
#     name: traders_nlp
# ---

# ## Import packages

# +
import numpy as np
import pandas as pd
import datetime as dt

from matplotlib import pyplot as plt
from pandas.tseries.holiday import USFederalHolidayCalendar as calendar
# -

# ## Read csv

# +
df = pd.read_csv("../ib-data/nyse/DDD.csv")

# Parse strings to datetime object
df['date']  = pd.to_datetime(df['date'], format='%Y-%m-%d %H:%M:%S')
# -

df['date'].dt.date.isin(calendar())

 cal = calendar()
holidays = cal.holidays(start=df['date'].min() - dt.timedelta(days=1), end=df['date'].max() + dt.timedelta(days=1))


# ## Datetime features

def generate_timestamp_features(df):
    
    ### This set values should uniquely identify a record
    df['year'] = df['date'].dt.year
    df['monthOfYear'] = df['date'].dt.month
    df['dayOfMonth'] = df['date'].dt.day
    df['hourOfDay'] = df['date'].dt.hour
    df['minuteOfHour'] = df['date'].dt.minute
    
    ###################################################################################################################
    
    ### Day of week
    df['dayOfWeek'] = df['date'].dt.dayofweek
    ### Day of year
    df['dayOfYear'] = df['date'].dt.dayofyear
    ### Week of year
    df['weekOfYear'] = df['date'].dt.week
    
    ###################################################################################################################
    
    cal = calendar()
    holidays = cal.holidays(start=df['date'].min() - dt.timedelta(days=1), end=df['date'].max() + dt.timedelta(days=1))

    ### Holiday
    df['isHoliday'] = df['date'].isin(holidays)
    df['prevDayIsHoliday'] = (df['date'] - dt.timedelta(days=1)).dt.normalize().isin(holidays)
    df['nextDayIsHoliday'] = (df['date'] + dt.timedelta(days=1)).dt.normalize().isin(holidays)

    ### Convert to int
    df['isHoliday'] = df['isHoliday'].astype(int)
    df['prevDayIsHoliday'] = df['prevDayIsHoliday'].astype(int)
    df['nextDayIsHoliday'] = df['nextDayIsHoliday'].astype(int)
    
    ###################################################################################################################
    
    df = df[['date', 
             'year', 'monthOfYear', 'dayOfMonth', 'hourOfDay', 'minuteOfHour',
             'dayOfWeek', 'dayOfYear', 'weekOfYear', 'isHoliday', 'prevDayIsHoliday', 'nextDayIsHoliday', 
             'open', 'high', 'low', 'close', 'volume','barCount', 'average']]
    
    return df



# +
df = generate_timestamp_features(df)

df.columns
# -

df

# ## Set Datetime Index (Optional)

# +
# Set datetime column as index
# df = df.set_index('date')
# -

df.index

df.values

# ## Plotting

plt.plot(df['close'][0:100])
plt.show()

# ## Generate New Dataset

import glob
import os

nyse_csv_paths = sorted(glob.glob("../ib-data/nyse-daily/*.csv"))
nasdaq_csv_paths = sorted(glob.glob("../ib-data/iex/*.csv"))

# #### NYSE

for i in range(len(nyse_csv_paths)):
    path = nyse_csv_paths[i]
    
    # read csv
    df = pd.read_csv(path)

    # Parse strings to datetime object
    df['date']  = pd.to_datetime(df['date'], format='%Y-%m-%d %H:%M:%S')
    
    # generate features
    df = generate_timestamp_features(df)
    
    # save to new file
    PATH_NAME = "../ib-data/nyse-daily-features/" + os.path.basename(path)
    df.to_csv(path_or_buf=PATH_NAME, index=False)
    
    # print message
    print( str(i + 1) + "/" + str(len(nyse_csv_paths)) + " completed.")

# #### NASDAQ

for i in range(len(nasdaq_csv_paths)):
    path = nasdaq_csv_paths[i]
    
    # read csv
    df = pd.read_csv(path)

    # Parse strings to datetime object
    df['date']  = pd.to_datetime(df['date'], format='%Y-%m-%d %H:%M:%S')
    
    # generate features
    df = generate_timestamp_features(df)
    
    # save to new file
    PATH_NAME = "../ib-data/iex-features/" + os.path.basename(path)
    df.to_csv(path_or_buf=PATH_NAME, index=False)
    
    # print message
    print( str(i + 1) + "/" + str(len(nasdaq_csv_paths)) + " completed.")



StarcoderdataJupyterScriptsDedupFiltered