import bigbang.datasets.domains as domains
import bigbang.analysis.utils as utils
import bigbang.analysis.attendance as attendance

from ietfdata.datatracker     import *
from ietfdata.datatracker_ext import *
import pandas as pd
import matplotlib.pyplot as plt
import dataclasses

import bigbang.datasets.organizations as organizations
org_cats = organizations.load_data()

"""
Explanation: IETF Affiliations from Attendance Records
End of explanation
"""


datatracker = DataTracker()

meetings = datatracker.meetings(meeting_type = datatracker.meeting_type(MeetingTypeURI('/api/v1/name/meetingtypename/ietf/')))
full_ietf_meetings = list(meetings)

ietf_meetings = []
for meeting in full_ietf_meetings:
    meetingd = dataclasses.asdict(meeting)
    meetingd['meeting_obj'] = meeting
    meetingd['num'] = int(meeting.number)
    ietf_meetings.append(meetingd)    

meetings_df = pd.DataFrame.from_records(ietf_meetings)

"""
Explanation: Getting attendance records from datatracker
When attendees register for a meeting, the report their name, email address, and affiliation.
While this is noisy data (any human-entered data is!), we will use this information to associate domains with affilations. E.g. the email domain apple.com is associated with the company Apple.
We will also use this data to enrich our understanding of individual affiliations over time.
End of explanation
"""


dt = DataTrackerExt() # initialize, for all meeting registration downloads

"""
Explanation: Individual Affiliations
End of explanation
"""


ietf_meetings[110]['date']

meeting_attendees_df = pd.DataFrame()
for meeting in ietf_meetings:
    if meeting['num'] in [104,105,106,107,108,109]: # can filter here by the meetings to analyze
        registrations = dt.meeting_registrations(meeting=meeting['meeting_obj'])
        df = pd.DataFrame.from_records([dataclasses.asdict(x) for x in list(registrations)])
        df['num'] = meeting['num']
        df['date'] = meeting['date']
        df['domain'] = df['email'].apply(utils.extract_domain)
        full_name = df['first_name'] + " " + df['last_name']
        df['full_name'] = full_name
        meeting_attendees_df = meeting_attendees_df.append(df)

"""
Explanation: This will construct a dataframe of every attendee's registration at every specified meeting. (Downloading this data takes a while!)
End of explanation
"""


ind_affiliation = meeting_attendees_df[['full_name', 'affiliation', 'email', 'domain','date']]

"""
Explanation: Filter by those who actually attended the meeting (checked in, didn't just register).
End of explanation
"""


ind_affiliation[:10]

ind_affiliation['affiliation'].dropna().value_counts()

"""
Explanation: This format of data -- with name, email, affiliation, and a timestamp -- can also be extracted from other IETF data, such as the RFC submission metadata. Later, we will use data of this form to infer duration of affilation for IETF attendees.
End of explanation
"""


affil_domain = ind_affiliation[['affiliation', 'domain', 'email']].pivot_table(
    index='affiliation',columns='domain', values='email', aggfunc = 'count')

"""
Explanation: Matching affiliations with domains
End of explanation
"""


ddf = domains.load_data()

generics = ddf[ddf['category'] == 'generic'].index
personals = ddf[ddf['category'] == 'personal'].index

generic_email_domains = set(affil_domain.columns).intersection(generics)
affil_domain.drop(generic_email_domains, axis = 1, inplace = True)

personal_email_domains = set(affil_domain.columns).intersection(personals)
affil_domain.drop(personal_email_domains, axis = 1, inplace = True)

ad_max = affil_domain.apply(lambda row: row.max(), axis=1)
ad_mean = affil_domain.apply(lambda row: row.dropna().mean(), axis=1)
ad_count = affil_domain.apply(lambda row: row.dropna().count(), axis=1)
ad_sum = affil_domain.apply(lambda row: row.dropna().sum(), axis=1)

ad_max_domain = affil_domain.apply(lambda row: row.idxmax(), axis=1)

## Add the columns *after* computing the statistics!
affil_domain['max'] = ad_max
affil_domain['mean'] = ad_mean
affil_domain['count'] = ad_count
affil_domain['sum'] = ad_sum
affil_domain['max_domain'] = ad_max_domain

ad_stats = affil_domain[['max_domain','max','count','mean','sum']].sort_values('max', ascending=False)

ad_stats[:100]

ad_stats[:100].to_csv("affiliation_domain_stats.csv")

ad_stats['sum']

"""
Explanation: Drop both known generic and known personal email domains.
End of explanation
"""


affil_dates = ind_affiliation.pivot_table(
    index="date",
    columns="full_name",
    values="affiliation",
    aggfunc="first"
).fillna(method='ffill').fillna(method='bfill')

top_attendees = ind_affiliation.groupby('full_name')['date'].count().sort_values(ascending=False)[:40].index

top_attendees

affil_dates[top_attendees]

affil_dates[top_attendees].to_csv("inferred_affiliation_dates.csv")

"""
Explanation: Duration of affiliation
The current data we have for individual affiliations is "point" data, reflecting the affiliation of an individual on a particular date.
For many kinds of analysis, we may want to understand the full duration for which an individual has been associated with an organization. This requires an inference from the available data points to dates that are not explicitly represented in the data.
For now, we will use a rather simple form of inference: filling in any missing data from the last (temporally) known data point. And then if there's still missing data, infer backwards.
End of explanation
"""


import bigbang.analysis.process as process

# drop subsidiary organizations
org_cats = org_cats[org_cats['subsidiary of / alias of'].isna()]

org_cats

"""
Explanation: Linking to Organization lists
End of explanation
"""


org_names = ad_stats['sum']
org_names = org_names.append(
    pd.Series(index = org_cats['name'], data = 1)
)
org_names = org_names.sort_values(ascending = False)
org_names = org_names[~org_names.index.duplicated(keep="first")]

ents = process.resolve_entities(
    org_names,
    process.containment_distance,
    threshold=.15
)

replacements = {}
for r in [{name: ent for name in ents[ent]} for ent in ents]:
    replacements.update(r)

ad_stats['norm_org'] = ad_stats.apply(lambda x : replacements[x.name], axis = 1)
org_cats['norm_org'] = org_cats.apply(lambda x : replacements[x['name']], axis = 1)

org_cats_plus = org_cats.join(ad_stats[['max_domain', 'norm_org']], on = 'norm_org', rsuffix="_ietf")

org_cats_plus_match = org_cats_plus[(~org_cats_plus['max_domain'].isna())].drop('norm_org_ietf',axis=1).rename({'max_domain' : 'max_domain_ietf'}, axis = 1)

org_cats_plus_match.to_csv("org_categories_matched_with_ietf_attendence_domains.csv")

org_cats_plus_match[:20]

"""
Explanation: Normalize/resolve the names from the IETF attedence records.
End of explanation
"""


meeting_range = [106,107,108]

a, b, c = attendance.name_email_affil_relations_from_IETF_attendance(meeting_range, threshold = 0.17)

a

b

b['affiliation'].value_counts()['cisco']

c

"""
Explanation: Export the graph of relations
Getting the affiliation data relations extracted from the attendance tables.
Final form: Three tables:
 - Name - Email, earliest and latest date
 - Name - Affiliation, earliest and latest date
 - Email - Affiliation, earliest and latest date
These can be combined into a tripartite graph, which should have a component for each affiliation entity.
End of explanation
"""


from bigbang.archive import Archive
arx = Archive("httpbisa")

"""
Explanation: Match to a mailing list
End of explanation
"""


arx.add_affiliation(b)

arx.data[['From','Date','affiliation']].dropna()

"""
Explanation: From the archive data: From -> email address, Date
Match with table B: email,. min_date, max_date, to get Affiliation
Add Affiliation to the archive data.
End of explanation
"""


