url = "https://lists.wikimedia.org/pipermail/analytics/"
arx = Archive(url,archive_dir="../archives")

"""
Explanation: First, collect data from a public email archive.
End of explanation
"""


#threads = arx.get_threads()
len(arx.get_threads())

"""
Explanation: We can count the number of threads in the archive easily. The first time you run Archive.get_thread it may take some time to compute, but the result is cached in the Archive object.
End of explanation
"""


y = [t.get_num_messages() for t in arx.get_threads()]

plt.hist(y, bins=30)
plt.xlabel('number of messages in a thread')
plt.show()

"""
Explanation: We can plot a histogram of the number of messages in each thread. In most cases this will be a power law distribution.
End of explanation
"""


n = [t.get_num_people() for t in arx.get_threads()]

plt.hist(n, bins = 20)
plt.xlabel('number of email-address in a thread')
plt.show()

"""
Explanation: We can also plot the number of people participating in each thread. Here, the participants are differentiated by the From: header on the emails they've sent.
End of explanation
"""


y = [t.get_duration().days for t in arx.get_threads()]

plt.hist(y, bins = (10))
plt.xlabel('duration of a thread(days)')
plt.show()

y = [t.get_duration().seconds for t in arx.get_threads()]

plt.hist(y, bins = (10))
plt.xlabel('duration of a thread(seconds)')
plt.show()

"""
Explanation: The duration of a thread is the amount of elapsed time between its first and last message.
End of explanation
"""


print(arx.get_threads()[0].get_duration())

threads = arx.get_threads

content = arx.get_threads()[0].get_root().data['Body']
content

len(content.split())

"""
Explanation: You can examine the properties of a single thread.
End of explanation
"""


short_threads = []
long_threads = []
for t in arx.get_threads():
    if(t.get_num_messages() < 6): short_threads.append(t)
    else: long_threads.append(t)

print(len(short_threads))
print(len(long_threads))

len(long_threads[0].get_content())

dist_short = []
dist_long = []
for t in short_threads:
    avg_short = sum([len(i.split()) for i in t.get_content()]) / len(t.get_content())
    dist_short.append(avg_short)
for t in long_threads:
    avg_long = sum([len(i.split()) for i in t.get_content()]) / len(t.get_content())
    dist_long.append(avg_long)

plt.hist(dist_short, bins = (15))
plt.show()

plt.hist(dist_long, bins = (15))
plt.show()

print(sum(dist_short)/ len(dist_short))
print(sum(dist_long)/ len(dist_long))

%reload_ext autoreload
s_leaves = []
s_notleaves = []
for t in threads:
    for node in t.get_leaves():
        s_leaves.append(len(node.data['Body'].split()))
    for node in t.get_not_leaves():
        s_notleaves.append(len(node.data['Body'].split()))

plt.hist(s_leaves, bins = (15))
plt.show()

plt.hist(s_notleaves, bins = (15))
plt.show()

print(sum(s_leaves)/len(s_leaves))
print(sum(s_notleaves)/len(s_notleaves))

import re
mess = threads[85].get_leaves()[0].data['Body']

print(mess)

mess.split('\n')
message = list()
for l in mess.split('\n'):
    n = len(l)
    if(len(l)!=0 and l[0] != '>' and l[n-6:n] != 'wrote:'):
        message.append(l)
new = str()
for l in message:
    new = new + l + '\n'


print(new)

print(EmailReplyParser.parse_reply(mess))

print(mess)

threads[85].get_leaves()[0].data

"""
Explanation: Suppose we want to know whether or not longer threads (that contain more distinct messages) have more words.
End of explanation
"""


