from bigbang.archive import Archive
from config.config import CONFIG
import bigbang.datasets.domains as domains
import bigbang.analysis.utils as utils
import numpy as np
import os
import pandas as pd
import re
import seaborn as sns

import powerlaw


"""
Explanation: Classifying Email Domains
This notebook explores the possibility of classifying email domains based on the characteristics of their activity distribution.
End of explanation
"""


dc = domains.load_data()

"""
Explanation: Domain categories
This is data about known domain categories.

Generic hosts. Generic providers of email addresses. See https://github.com/edwin-zvs/email-providers
Personal. Email domains set up for use by an individual or a very small company.
Companies. An email domain used by the employees of a large corporation.
Internet Governance. Email domains associated with an Internet Governance body.
Academia. Email domains associated with an university, research lab, or professional research organization.
End of explanation
"""


number = r"^[0-9]"

arxs = {}

for file in os.listdir(CONFIG.mail_path):
    if file.endswith(".csv") and not re.match(number, file):
        name = file[:-4]
        try:
            arxs[name] = Archive(name)
        except Exception as e:
            print(name + ": " + str(e))

print("Total number of Archives: " + str(len(arxs)))

"""
Explanation: Working with the data
Load IETF archive data
We start by loading the IETF archive data.
End of explanation
"""


def senders(arx):
    emails = arx.data['From'].apply(utils.extract_email)
    domains = arx.data['From'].apply(utils.extract_domain)
    froms = pd.concat([arx.data['From'], emails, domains], axis=1, keys=['From','email','domain'])
    
    return froms

wg_senders = {wg : senders(arxs[wg]) for wg in arxs}

"""
Explanation: Combine email data
Collect information about the senders from each mailing list.
End of explanation
"""


def add_wg_row(df, wg):
    df2 = df.copy()
    df2['wg'] = wg
    
    return df2

all_senders = pd.concat([add_wg_row(wg_senders[wg], wg) for wg in wg_senders])

"""
Explanation: Explore the data
We are going to start with using statistics from the global corpus of data. To combine the data from each working group into one DataFrame.
End of explanation
"""


all_senders.groupby('domain').size().sort_values(ascending=False).plot(loglog = True)

data = all_senders.groupby('domain').size().values # data can be list or numpy array
results = powerlaw.Fit(data)
print(results.power_law.alpha)
print(results.power_law.xmin)

R, p = results.distribution_compare('power_law', 'lognormal', normalized_ratio=True)
print(R, p)

all_senders.groupby('wg').size().sort_values(ascending=False).plot(loglog = True)

data = all_senders.groupby('wg').size().values # data can be list or numpy array
results = powerlaw.Fit(data)
print(results.power_law.alpha)
print(results.power_law.xmin)

R, p = results.distribution_compare('power_law', 'lognormal', normalized_ratio=True)
print(R, p)

x = all_senders.groupby('domain')['email'].unique().apply(len)
y = all_senders.groupby('domain')['wg'].unique().apply(len)

import matplotlib.pyplot as plt

fig = plt.figure()
ax = plt.gca()
ax.scatter(x, y, c='blue', alpha=0.05, edgecolors='none')
ax.set_xlabel('unique emails')
ax.set_xlabel('unique wgs')
ax.set_yscale('log')
ax.set_xscale('log')

"""
Explanation: Each email domain participates in some number of working groups. It also contains some number of unique email addresses. Just to get a sense of the variety, we will look at and compare these two distributions.
End of explanation
"""


domain_unique_emails = all_senders.groupby('domain')['email'].apply(lambda g : np.log1p(g.unique().size))
domain_unique_wgs = all_senders.groupby('domain')['wg'].apply(lambda g : np.log1p(g.unique().size))
domain_unique_wgs

domain_means = all_senders.groupby('domain')['email'].apply(lambda g : g.value_counts().mean())
domain_stds = all_senders.groupby('domain')['email'].apply(lambda g : g.value_counts().std())
domain_means_log = np.log1p(domain_means)
domain_medians = all_senders.groupby('domain')['email'].apply(lambda g : g.value_counts().median())
domain_medians_log = np.log1p(domain_medians)
domain_max = all_senders.groupby('domain')['email'].apply(lambda g : g.value_counts().max())
domain_max_log = np.log1p(domain_max)

df = pd.DataFrame(data = {
    'n_e' : domain_unique_emails, 
    'n_w' : domain_unique_wgs,
    'log_mean' : domain_means_log,
    'log_median' : domain_medians_log, 
    'log_max' : domain_max_log,
    'nstd' : domain_stds.fillna(0) / domain_means
})

dc['category'].at['gmail.com']

"""
Explanation: Compute more domain statistics
We will now compute more summary statistics for each domain.
End of explanation
"""


def categorize_domain(d):
    try:
        return dc['category'].at['gmail.com']
    except Exception as e:
        return None

labels = pd.Series(data = df.index, index = df.index).apply(categorize_domain)

df['labels'] = labels

"""
Explanation: And add the category labels if known.
End of explanation
"""


df = df[df['log_mean'] > 0.0]

df.head()

fig = plt.figure(figsize=(8,8))
ax = plt.gca()
ax.plot(df['log_median'], df['nstd'], 'o', c='yellow', alpha=0.02)

colors = ['black', 'red', 'green', 'cyan', 'magenta']

for i, group in enumerate(dc['category'].unique()):
    print(group)
    group_idx = set(df.index).intersection(set(dc[dc['category'] == group].index))
    gdf = df.loc[group_idx, :]
    ax.plot(gdf['log_median'], gdf['nstd'], '+', c=colors[i], alpha=0.5, label = group)

ax.set_xlabel('Log Median')
ax.set_ylabel('Normalized Standard Deviation')
ax.set_yscale('log')
ax.set_xscale('log')

ax.set_title('Statistics of Email Domain Distributions')

ax.set_ylim([0.1,10])

ax.legend()



"""
Explanation: A little data cleaning....
End of explanation
"""


