from __future__ import print_function
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
%matplotlib inline

"""
Explanation: Exploratory Analysis
End of explanation
"""


df = pd.read_csv('./../data/data.csv')

df.shape

df.info()

print('no txs: {}, no blocks: {}'.format(df.shape[0], np.unique(df['block_id'].values).shape[0]))

"""
Explanation: Load data
End of explanation
"""


df.loc[:,'time_t'] = pd.to_datetime(df.time_t, yearfirst=True)

df.loc[:,'time_b'] = pd.to_datetime(df.time_b, yearfirst=True)

df['time_t'].head()

df.drop('Unnamed: 0', axis=1, inplace=True)

"""
Explanation: Convert dates to datetime
End of explanation
"""


int(filter(str.isdigit, df['difficulty'][0]))

df['difficulty'] = df['difficulty'].apply(lambda x: int(filter(str.isdigit, x)))

df['difficulty'].head()

df['reward'] = df['reward'].apply(lambda x: int(filter(str.isdigit, x)))

df['reward'].head()

df['totalFee'] = df['totalFee'].apply(lambda x: int(filter(str.isdigit, x)))

df['totalFee'].head()

"""
Explanation: Convert long integer columns

difficulty
reward
totalFee

Filter each string extracting the digits as integers
End of explanation
"""


def float_to_int(col_list):
    for col in col_list:
        df[col] = df[col].apply(lambda x: np.rint(x))
        df[col] = df[col].values.astype(int)

float_to_int(['amount', 'price', 'gasLimit_b', 'gasUsed_b'])

df['isContractTx'].isnull().sum()

"""
Explanation: Convert floats back to ints

when collecting data from the api, some of the ints were longer than 8 bytes causing mongo to throw up errors 
to avoid this we temporarily converted those to floats
End of explanation
"""


df.drop('isContractTx', axis=1, inplace=True)

df['txIndex'].isnull().sum()

df['txIndex'].value_counts()

df['type'].value_counts()

df['newContract'].value_counts()

"""
Explanation: The column "isContractTx" is empty so drop it
End of explanation
"""


df['amount_binary'] = df['amount'].map(lambda x: 1 if x > 0 else 0)

"""
Explanation: Binarize the amount column (1 if > 0 and 0 if 0)
End of explanation
"""


for col in df.columns:
    print(col, df[col].isnull().sum())

"""
Explanation: Check for NaN values
End of explanation
"""


df.info()

"""
Explanation: Columns decriptions
End of explanation
"""


df['price_gwei'] = df['price'].values / 1000000000

plt.hist(df['price_gwei'], bins=3000, color='b', alpha=0.7)
plt.xlim(0,100)
plt.xlabel('Price (GWei)')
plt.show()

np.mean(df['price_gwei'].values)

np.max(df['price_gwei'].values)

np.min(df['price_gwei'].values)

df['amount_binary'].value_counts()

df['type'].value_counts()

grouped_type = df.groupby('type')

df.groupby('type')['price_gwei'].mean()

df.groupby('type')['price_gwei'].describe().T

"""
Explanation: Decription:
Transaction level

hash_t : a unique identifier for each transaction
accountNonce : the number of transactions the sender account has made on the network (mechanism to ensure the same tranaction isn't submitted twice)
amount : for monetary transactions, the amount in Ether that is exchanged
block_id : a unique identifier for each block
gasLimit_t : the maximum amount of gas that is allowed to be spent on gas for a given transaction
gasUsed_t : the actual amount of gas that was consumed on the given transaction 
newContract : whether it is a new contract or not
price : the price per unit gas (Wei)
txIndex : an integer from 1 to 20 (or NaN)
amount_binary : binarized amount column (1 if amount is non-zero)
type : the kind of transaction
tx
call
create
suicide



Block level

blockTime : the total amount of time spent on the block
difficulty : 
gasLimit_b :
gasUsed_b :
reward :
size : 
time_b :
totalFee :
tx_count : the total number of txs in a given block
uncle_count : 

Categorical variables:

txIndex
type
amount_binary

Explore features
New column with converted price from Wei to Gwei
End of explanation
"""


tx = df[df['type'] == 'tx']
call = df[df['type'] == 'call']
create = df[df['type'] == 'create']
suicide = df[df['type'] == 'suicide']

plt.hist(tx['price_gwei'], bins=4000, color='b', alpha=0.7)
plt.xlim(-10,100)
plt.title('tx', fontsize=18)
plt.xlabel('price', fontsize=18)
plt.show()

plt.hist(call['price_gwei'], bins=4000, color='b', alpha=0.7)
plt.xlim(-10,100)
plt.title('call', fontsize=18)
plt.xlabel('price', fontsize=18)
plt.show()

plt.hist(create['price_gwei'], bins=50, color='b', alpha=0.7)
plt.xlim(-10,100)
plt.title('create', fontsize=18)
plt.xlabel('price', fontsize=18)
plt.show()

plt.hist(suicide['price_gwei'], bins=10, color='b', alpha=0.7)
plt.xlim(-10,100)
plt.title('suicide', fontsize=18)
plt.xlabel('price', fontsize=18)
plt.show()

"""
Explanation: Call has the highest mean price, while suicide has the lowest
Plot price for different types
End of explanation
"""


df['day'] = pd.DatetimeIndex(df['time_t']).day

# How many transactions each day?
df['day'].value_counts()

df['hour'] = pd.DatetimeIndex(df['time_t']).hour

df['minute'] = pd.DatetimeIndex(df['time_t']).minute

df['second'] = pd.DatetimeIndex(df['time_t']).second

df['dayofweek'] = pd.DatetimeIndex(df['time_t']).dayofweek

# How many transaction occur on a given day of the week? 
df['dayofweek'].value_counts()

df.columns

# Mean price per day
df['price_gwei'].groupby(df.day).mean()

# How many blocks are in each day (24 hrs)?
df['block_id'].groupby(df.day).nunique()

df[df['amount'] > 0].hist('amount', bins=20)

"""
Explanation: There seems to be some signal among the different transaction types 
Clean up time data
New column for separate day, time
End of explanation
"""


df['amount_eth'] = df['amount'].map(lambda x: float(x) / 1000000000000000000)

df['amount_eth'].head()

df[['amount_eth', 'type', 'newContract', 'price_gwei', 'block_id']].head()

df['price_gwei'].describe()

"""
Explanation: Conversions
Conversion from Gwei to ETH
1 Gwei = 0.000000001 ETH
Amount

amount is ~ to 10^-18 ETH

Convert amount to ETH
End of explanation
"""


from sklearn import preprocessing
le = preprocessing.LabelEncoder()
df['type_enc'] = le.fit_transform(df['type'])

df['type_enc'].value_counts()

"""
Explanation: Encode the type column
End of explanation
"""


df_filtered = df[(df['amount_eth'] > 10**(-6)) & (df['price_gwei'] > 0)]

df_filtered = df[(df['amount_eth'] > 10**(-6)) & (df['price_gwei'] > 0)]

print('deleted {} rows having 0 amounts'.format(df[df['amount_eth'] == 0].values.shape[0]))

print('deleted {} rows having 0 gas price'.format(df[df['price_gwei'] == 0].values.shape[0]))

df_filtered['amount_eth'].apply(np.log10).hist(bins=100)
plt.xlabel('Amount')

df_filtered['amount_eth'].hist()
plt.xlabel('Amount')

df_filtered['price_gwei'].hist(bins=2000)
plt.xlim(0,100)
plt.xlabel('Gas Price')

gas_log = np.log(df_filtered['price_gwei'].values)

plt.scatter(df_filtered['amount_eth'], df_filtered['price_gwei'])
plt.xlabel('Amount')
plt.ylabel('Gas Price')

df['dayofweek'].describe()

df_filtered['dayofweek'].hist(bins=7)
plt.xlabel("Day of Week (Mon-Sun)")

df_filtered['hour'].describe()

df_filtered['hour'].hist(bins=24)
plt.xlim(0,23)
plt.xlabel('Hour')

df_filtered.groupby('type')['gasUsed_t'].mean()

"""
Explanation: Filter data

only monetary transactions (amount > 0)
only gas price > 0
End of explanation
"""


df_filtered['txcnt_second'] = df_filtered['tx_count'].values / df_filtered['blockTime'].values

df_filtered['avg_gasUsed_t_perblock'] = df_filtered.groupby('block_id')['gasUsed_t'].transform('mean')

df_filtered['avg_price_perblock'] = df_filtered.groupby('block_id')['price_gwei'].transform('mean')

df_filtered.to_csv('./../data/data_filtered.csv')

"""
Explanation: Obtain averages over previous blocks for each block id to generate new features

difficulty 
blocktime
transactions/second
uncle rate
gasused by transaction
gasused by block
gasprice

Add column with:

txs per second for each block
avg gasUsed for each block
avg price per transaction for each block
End of explanation
"""


def rolling_avg(window_size):
    
    price = df_filtered[['block_id', 'avg_price_perblock']].drop_duplicates().sort_values(
                        'block_id', ascending=True)
    
    gasUsed_t = df_filtered[['block_id', 'avg_gasUsed_t_perblock']].drop_duplicates().sort_values(
                            'block_id', ascending=True)
    
    txcnt_second = df_filtered[['block_id', 'txcnt_second']].drop_duplicates().sort_values(
                                'block_id', ascending=True)
    
    tx_count = df_filtered[['block_id', 'tx_count']].drop_duplicates().sort_values(
                            'block_id', ascending=True)
    
    gasUsed_b = df_filtered[['block_id', 'gasUsed_b']].drop_duplicates().sort_values(
                            'block_id', ascending=True)
    
    uncle_count = df_filtered[['block_id', 'uncle_count']].drop_duplicates().sort_values(
                                'block_id', ascending=True)
    
    difficulty = df_filtered[['block_id', 'difficulty']].drop_duplicates().sort_values(
                                'block_id', ascending=True)
    
    blocktime = df_filtered[['block_id', 'blockTime']].drop_duplicates().sort_values(
                             'block_id', ascending=True)
    
    # create new pandas dataframe with average values
    rolling_avg = pd.DataFrame()
    
    # calculate rolling averages 
    rolling_avg['avg_blocktime'] = blocktime['blockTime'].rolling(window=window_size).mean()
    rolling_avg['avg_gasUsed_b'] = gasUsed_b['gasUsed_b'].rolling(window=window_size).mean()
    rolling_avg['avg_tx_count'] = tx_count['tx_count'].rolling(window=window_size).mean()
    rolling_avg['avg_uncle_count'] = uncle_count['uncle_count'].rolling(window=window_size).mean()
    rolling_avg['avg_difficulty'] = difficulty['difficulty'].rolling(window=window_size).mean()
    rolling_avg['avg_txcnt_second'] = txcnt_second['txcnt_second'].rolling(window=window_size).mean()
    rolling_avg['avg_gasUsed_t'] = gasUsed_t['avg_gasUsed_t_perblock'].rolling(window=window_size).mean()
    rolling_avg['avg_price'] = price['avg_price_perblock'].rolling(window=window_size).mean()
    
    # insert blockids to merge on
    rolling_avg['blockids'] = df_filtered['block_id'].drop_duplicates().sort_values(ascending=True)
    
    return rolling_avg

"""
Explanation: Calculate the rolling average over given number of windows

This is very messy! Will clean up later
End of explanation
"""


num_blocks = [5, 10, 25, 50, 100, 1000, 2000, 3000, 4000]
for num in num_blocks:
    df_rolling_avg = rolling_avg(num)
    df_rolling_avg.to_csv('./../data/block_avg_{}.csv'.format(num))

"""
Explanation: There are between 600 and 3500 blocks in a given day, so try 5 through 4000 previous blocks
Create csv files for each regime
End of explanation
"""


