import re
import numpy as np
import pandas as pd
import email
#Plotting stuff
%matplotlib inline
import matplotlib.pyplot as plt
import seaborn as sns
sns.set_context('poster')
import hypergraph

"""
Explanation: The main page containing the enron data can be found at: https://www.cs.cmu.edu/~./enron/
End of explanation
"""


import os
#for root, user, file in os.walk('/Users/jchealy/Downloads/maildir/'):
root = '/Users/jchealy/Downloads/maildir/'
count =0 
edges = []
file_index = []
for user in os.listdir(root):
    #print(user)
    location = root+user+'/_sent_mail'
    if(count>2):
        break   
    if(os.path.isdir(location)):
        #print(location)
        for fname in os.listdir(location):
            file= location+'/'+fname
            #print(file)
            with open(file) as f:
                message = email.message_from_file(f)
                edge = [message['from']]
                if 'To' in message:
                    edge = edge+re.split(r'\s*,\s*', message['to'])
                if 'Cc' in message:
                    edge = edge + re.split(r'\s*,\s*', message['cc'])
                if 'Bcc' in message:
                    edge = edge + re.split(r'\s*,\s*', message['bcc'])
                edges.append(edge)
                file_index.append(file)

"""
Explanation: Read the data into hyperedges.  We preserve order only in so far as the first element in each array is the sender.  Email addresses may appear multiple times if they were included multiple times in the header.  For example their exist cases where a given address was included in both the cc and the bcc lines of the same message.
to, cc and bcc addresses where treated the same and simply merged into a single list.
End of explanation
"""


l = pd.Series([len(x) for x in edges])
d = l.value_counts().sort_index().reset_index()
d.columns = ['index', 'freq']
d['freq']=np.log10(d['freq'])
ax = d.plot(x='index', y='freq', kind='scatter', alpha=0.6, xlim=[-10,max(d['index'])+10])
ax.set_ylabel('Frequency (log10)')
ax.set_xlabel('Hyperedge Size')

"""
Explanation: Descriptive Statistics
Edge size distribution
End of explanation
"""


len(edges)

"""
Explanation: Number of hyperedges in email hypergraph
End of explanation
"""


flat = [item for sublist in edges for item in sublist]
len(flat)
nodes = np.unique(flat)
len(nodes)

"""
Explanation: Number of nodes in email hypergraph
End of explanation
"""


edges[1:10]

"""
Explanation: Next read the edges into Lelands library
Then write various output functions for these structures depending on folks want to ingest
End of explanation
"""


hg = hypergraph.Hypergraph()

for i, edge in enumerate(edges):
    hg.add_edge(i,edge)
    if(i%10000 == 0):
        print(i)

"""
Explanation: Pawel has requested that these addresses be remapped to one up integers and output in one array per line.  To maximize code re-use I'll write that as function in the hypergraph library.
Ingest our array of arrays into a hypergraph object
End of explanation
"""


edge_size = pd.Series([hg.edge[x].size for x in hg.edge])
d = edge_size.value_counts().sort_index().reset_index()
d.columns = ['index', 'freq']
d['freq']=np.log10(d['freq'])
ax = d.plot(x='index', y='freq', kind='scatter', alpha=0.6, xlim=[-10,max(d['index'])+10])
ax.set_ylabel('Frequency (log10)')
ax.set_xlabel('Hyperedge Size')

"""
Explanation: Recompute the edge size distribution
This is a quick example of how to access the hypergraph object.
End of explanation
"""


node_size = pd.Series([hg.node[x].size for x in hg.node])
d = node_size.value_counts().sort_index().reset_index()
d.columns = ['index', 'freq']
d['freq']=np.log10(d['freq'])
d['index']=np.log10(d['index'])
ax = d.plot(x='index', y='freq', kind='scatter', alpha=0.6, xlim=[-1,max(d['index'])+1])
ax.set_ylabel('Frequency (log10)')
ax.set_xlabel('Vertex Size (log10)')

"""
Explanation: Now let's look at node size, which might be refered to as degree by some.
End of explanation
"""


sender = pd.Series([hg.edge[x].labels[0] for x in hg.edge])
sender.head()

sender = pd.Series([hg.edge[x].labels[0] for x in hg.edge])
d = sender.value_counts().plot(kind='line', title='number of email sent by each user', logx=False, logy=True)
plt.xticks([])
plt.ylabel('outgoing hyperedges')
plt.xlabel('sender (decreasing order)')

"""
Explanation: We can look at sender degree (number of emails sent by an individual) since the sender is currently coded as the first vertex in every hyperedge
End of explanation
"""


pd.set_option('precision',2)

edge_size = pd.Series([hg.edge[x].size for x in hg.edge])
edge_card = pd.Series([hg.edge[x].cardinality for x in hg.edge])
#hg.edge[[edge_size!=edge_card]]
edge_difference_index = np.where(edge_size!=edge_card)[0]
print(str(len(edge_difference_index))+" out of " + str(len(edge_size)) + " ("+
      "{:.2f}".format(len(edge_difference_index)/len(edge_size)*100)+
      "%) edges differed between cardinality and size")

[hg.edge[x].labels for x in edge_difference_index][:5]

"""
Explanation: Might be worth inducing the 2 section and overlaying the two degree distributions on each other.
The cardinality of an edge is the number of unique nodes in the edge.  An interesting question might be which email have a cardinality which differs from their size.
End of explanation
"""


keys = hg.nodes
values = range(len(hg.nodes))
vertex_map = dict(zip(keys,values))

filename = "enronNumericHypergraphEdgelist.txt"
file = open(filename,'w')
@np.vectorize
def vertex_mapper(x):
    return vertex_map[x]

for e in hg.edge:
    tup = vertex_mapper(hg.edge[e].labels)
    file.write(" ".join(str(elem) for elem in tup)+"\n")
    #print(" ".join(str(elem) for elem in [vertex_map[x] for x in tup]))
file.close()        

hg.node['tim.belden@enron.com'].labels

"""
Explanation: Output in basic integer tuple format
I'll convert this into a utility function directly in the library next
End of explanation
"""


