from __future__ import print_function
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
%matplotlib inline

"""
Explanation: Exploratory Analysis
End of explanation
"""


df = pd.read_csv('./../data/data.csv')

df.shape

df.info()

print('no txs: {}, no blocks: {}'.format(df.shape[0], np.unique(df['block_id'].values).shape[0]))

"""
Explanation: Load data
more info in the topic of gas costs here:
http://ethdocs.org/en/latest/contracts-and-transactions/account-types-gas-and-transactions.html
End of explanation
"""


df.loc[:,'time_t'] = pd.to_datetime(df.time_t, yearfirst=True)

df.loc[:,'time_b'] = pd.to_datetime(df.time_b, yearfirst=True)

df['time_t'].head()

df.drop('Unnamed: 0', axis=1, inplace=True)

"""
Explanation: Convert dates to datetime
End of explanation
"""


int(filter(str.isdigit, df['difficulty'][0]))

df['difficulty'] = df['difficulty'].apply(lambda x: int(filter(str.isdigit, x)))

df['difficulty'].head()

df['reward'] = df['reward'].apply(lambda x: int(filter(str.isdigit, x)))

df['reward'].head()

df['totalFee'] = df['totalFee'].apply(lambda x: int(filter(str.isdigit, x)))

df['totalFee'].head()

"""
Explanation: Convert long integer columns

difficulty
reward
totalFee

Filter each string extracting the digits as integers
End of explanation
"""


def float_to_int(col_list):
    for col in col_list:
        df[col] = df[col].apply(lambda x: np.rint(x))
        df[col] = df[col].values.astype(int)

float_to_int(['amount', 'price', 'gasLimit_b', 'gasUsed_b'])

df['isContractTx'].isnull().sum()

"""
Explanation: Convert floats back to ints

when collecting data from the api, some of the ints were longer than 8 bytes causing mongo to throw up errors 
to avoid this we temporarily converted those to floats
End of explanation
"""


df.drop('isContractTx', axis=1, inplace=True)

"""
Explanation: The column "isContractTx" is empty so drop it
End of explanation
"""


df['amount_binary'] = df['amount'].map(lambda x: 1 if x > 0 else 0)

"""
Explanation: Binarize the amount column (1 if > 0 and 0 if 0)
End of explanation
"""


for c in df.columns:
    print(c)

df['price_gwei'] = df['price'] / 1000000000.0
df['amount_gwei'] = df['amount'] / 1000000000.0

df[['price_gwei', 'gasUsed_t','gasUsed_b','difficulty' ]].describe()

df['gasShare']  = df.gasUsed_t/df.gasUsed_b
df['gweiPaid'] = df.gasUsed_t*df.price_gwei

gweiDict = df[['gweiPaid','block_id']].groupby('block_id').sum().T.to_dict()

df['gweiPaid_b'] = df.block_id.apply(lambda b: gweiDict[b]['gweiPaid'])

df['gweiShare']  = df.gweiPaid/df.gweiPaid_b

df['free_t'] = (df.gasUsed_t ==0).apply(int)

df[['block_id', 'free_t']].groupby('block_id').mean().hist(bins=50)

df[['type', 'free_t']].groupby('type').mean().plot(kind='bar', color='g')
plt.xlabel('')
plt.title('Free Transactions')
plt.tight_layout()
plt.savefig('./../images/free_transactions.png', dpi=300)

"""
Explanation: Decription:
Transaction level

hash_t : a unique identifier for each transaction
accountNonce : the number of transactions the sender account has made on the network (mechanism to ensure the same tranaction isn't submitted twice)
amount : for monetary transactions, the amount in Ether that is exchanged
block_id : a unique identifier for each block
gasLimit_t : the maximum amount of gas that is allowed to be spent on gas for a given transaction
gasUsed_t : the actual amount of gas that was consumed on the given transaction 
newContract : whether it is a new contract or not
price : the price per unit gas (Wei)
txIndex : an integer from 1 to 20 (or NaN)
amount_binary : binarized amount column (1 if amount is non-zero)
type : the kind of transaction
tx
call
create
suicide



Block level

blockTime : the total amount of time spent on the block
difficulty : 
gasLimit_b :
gasUsed_b :
reward :
size : 
time_b :
totalFee :
tx_count : the total number of txs in a given block
uncle_count : 

Categorical variables:

txIndex
type
amount_binary

Explore features
New column with converted price from Wei to Gwei
End of explanation
"""


#shares by type of events
df[['hash_t', 'type']].groupby('type').count().hash_t.plot(kind='pie')

#fees paid by type
df[['hash_t', 'type','gweiPaid']].groupby('type').sum().gweiPaid.plot(kind='pie')
plt.title('Total Fee Paid')
plt.ylabel('')
plt.tight_layout()
plt.savefig('./../images/pie_gweipaid.png', dpi=300)

#gas used by type
df[['hash_t', 'type','gasUsed_t']].groupby('type').sum().gasUsed_t.plot(kind='pie')
plt.title('Gas Used')
plt.ylabel('')
plt.tight_layout()
plt.savefig('./../images/pie_gasused.png', dpi=300)

#quick look at transactions
txdf = df[(df['type']=='tx')&(df.amount_gwei>0)].copy()
txdf['logGweiAmount'] = txdf.amount_gwei.apply(np.log10)
txdf['logGweiPrice'] = txdf.price_gwei.apply(np.log10)
txdf['logGasUsed'] = txdf.gasUsed_t.apply(np.log10)

txdf[['amount_gwei','price_gwei','gasUsed_t']].describe()

txdf[['logGweiAmount','logGweiPrice','logGasUsed']].describe()

#sns.pairplot(txdf[['logGweiAmount','logGweiPrice','logGasUsed']])

"""
Explanation: Suicide it seems is always free. Create and tx always have a cost. About 20% of call events are free but non of the create events are. more details below.
End of explanation
"""


df[df.gasUsed_t>0].gasUsed_t.apply(np.log10).hist(bins=40)

df[(df.gasUsed_t>0)&(df['type']=='tx')].gasUsed_t.apply(np.log10).hist(bins=10,alpha=.5)
df[(df.gasUsed_t>0)&(df['type']=='call')].gasUsed_t.apply(np.log10).hist(bins=10,alpha=.5)
df[(df.gasUsed_t>0)&(df['type']=='create')].gasUsed_t.apply(np.log10).hist(bins=10,alpha=.5)

plt.title('Gas Used')
plt.legend(['tx','call','create'])

df[(df.gasUsed_t>0)&(df['type']=='tx')].gweiPaid.apply(np.log10).hist(bins=10,alpha=.5)
df[(df.gasUsed_t>0)&(df['type']=='call')].gweiPaid.apply(np.log10).hist(bins=10,alpha=.5)
df[(df.gasUsed_t>0)&(df['type']=='create')].gweiPaid.apply(np.log10).hist(bins=10,alpha=.5)

plt.title('Gwei Paid')
plt.legend(['tx','call','create'])

#compute the fraction of transactions with zero gasUsed by block
df[['block_id', 'free_t']].groupby('block_id').mean().apply(np.log10).plot()
plt.title('log of fraction of transactions with 0 gasUsed')
ax = plt.gca()
ax.set_yticklabels([round(10**y,3) for y in ax.get_yticks()])

signals = df[['block_id', 'free_t','type']].groupby(['type','block_id']).mean().reset_index()
signals.head()

signals.groupby('type').describe()

df[df.gweiPaid>0].gweiPaid.apply(np.log10).hist()

df.gasShare.describe()

df[df.gasShare>0].gasShare.apply(np.log10).hist(bins=25)

#reduced dataframe
rdf = df[['gasUsed_t','gasShare','gweiPaid','gweiShare', 'price_gwei', 'type']].copy()

rdf.head()

#sns.pairplot(rdf[df.gweiPaid>0], hue="type")

#log10 values of reduced dataframe
ldf = rdf[df.gweiPaid>0].copy()
for c in ['gasUsed_t','gasShare','gweiPaid','gweiShare', 'price_gwei']:
    ldf[c] = ldf[c].apply(np.log10)

ldf.describe()

#sns.pairplot(ldf, hue="type")

sns.set(style='whitegrid', rc={"grid.linewidth": 0.1})
sns.set_context("notebook", font_scale=4.0)
sns.lmplot(x="gasUsed_t", y="price_gwei", hue="type",truncate=True, size=18, data=ldf, 
           scatter_kws={"s": 15,"alpha": .15})
plt.ylim(-1, 3)
plt.ylabel('Gas Price', fontsize=40)
plt.xlabel('Gas Used', fontsize=40)
plt.tight_layout()
plt.savefig('./../images/regmodel.png', dpi=300)

g = sns.PairGrid(ldf[ldf['type']=='create'], diag_sharey=False)
g.map_lower(sns.kdeplot, cmap="Blues_d")
g.map_upper(plt.scatter)
g.map_diag(sns.kdeplot, lw=3)
plt.savefig('./../images/pair_kde.png')

"""
Explanation: tx type is the only ones for which amount is non-zero. Moving forward we aren't looking at the ammount focusing on the gas amount used and the price paid for that gas.
End of explanation
"""


from scipy.stats import norm

sns.set(style='whitegrid', rc={"grid.linewidth": 0.1})
sns.set_context("notebook", font_scale=2.0)
x = np.linspace(-3, 3, num=250)

fig, ax = plt.subplots(1, 1, figsize=(10, 6))

ax.plot(x, norm.pdf(x), linewidth=4)
ax.set_xlim(-3, 3)
ax.set_ylim(-0.1, 0.5)
ax.fill_between(x, norm.pdf(x), alpha=0.6)
ax.axvline(x=-1, color='r', linestyle='--')
ax.set_title("")
plt.tight_layout()
plt.savefig('./../images/hyp_dist.png', dpi=300)

"""
Explanation: Toy model figure
End of explanation
"""


