# imports
import pandas as pd
import codecs
import graphistry
import warnings
import networkx as nx
from networkx.algorithms import assortativity as assort
from networkx.algorithms import centrality as central
import matplotlib.pyplot as plt
from collections import OrderedDict
from operator import itemgetter
from collections import Counter
from itertools import islice
import numpy as np
import igraph as ig
# setup
warnings.filterwarnings('ignore')
api_key = open('API_key.txt').read()
graphistry.register(key=api_key)
%matplotlib inline

def load_num_doc(file_path):
    """
    Reading num - doc dictonary for mapping document name and id
    """
    f = codecs.open(file_path, 'r', 'utf8')
    lines = f.readlines()
    f.close()
    num_doc_mapper = {}
    clean_lines = [line[:-2] for line in lines if line.endswith('\r\n')]
    clean_lines.append(lines[-1])
    for line in clean_lines:
        number = int(line.split(',')[0])
        text = line[len(str(number))+1:]
        num_doc_mapper[number] = text
    return num_doc_mapper

def sort_dictionary_by_value_asc(input_dict):
    output_dict = OrderedDict(sorted(input_dict.items(), key=itemgetter(1)))
    return output_dict

def sort_dictionary_by_value_desc(input_dict):
    output_dict = OrderedDict(sorted(input_dict.items(), key=itemgetter(1)))
    return output_dict

# reading graph
edges = pd.read_csv('dataset/new_data/all_text_lines.txt', sep='\t\t\t', names=['src', 'dest'])
print(edges.head())
# reading igraph from file
this_igraph = ig.Graph.Read_Ncol('dataset/new_data/all_num_lines.txt', directed=True)

# visualization using graphistry
graphistry.bind(source='src', destination='dest').plot(edges)

"""
Explanation: Serbian Legislation Network
Describing Serbian legislation system as a complex network.
Data
The latest version of each current legal document is available at <a href="http://www.pravno-informacioni-sistem.rs/SlGlasnikPortal/reg/advancedSearch">Serbian Legal Information System website</a>. <a href="https://github.com/vdragan1993/serbian-document-network/blob/master/src/crawler.py">Crawler</a> and <a href="https://github.com/vdragan1993/serbian-document-network/blob/master/src/scraper.py">Scraper</a> were developed in order to collect all republican legislations with their ID card and list of related regulations. Only legislations with ID card were scraped. Original collected data can be found in <a href="https://github.com/vdragan1993/serbian-document-network/tree/master/dataset/original_data">dataset/original_data/</a>.
Network
In order to create legislation network, nodes and their links needs to be created. 
Nodes
Every collected document is a node. List of document names extracted from their ID cards with custom made id number can be found in <a href="https://github.com/vdragan1993/serbian-document-network/blob/master/dataset/new_data/new_num_doc_sorted.txt">dataset/new_data/</a>.
Links
Links between nodes are full-explicit references. References extraction process for every document:

Converting Serbian Cyrillic to Latin, and replacing special characters with their ASCII pairs (č -> c, š -> s...)
Tokenization
Stemming using <a href="https://github.com/vdragan1993/serbian-stemmer">Serbian Stemmer</a>
Joining stemmed words into space-separated text
Searching for collected document names in joined text
Saving detected reference in <b>this_document_name\t\t\tfound_document_name</b> format

After this, all detected references were aggregated into one document. Also, another document was created by replacing document names with their custom made id numbers.
Result of this process are references by <a href="https://github.com/vdragan1993/serbian-document-network/blob/master/dataset/new_data/all_text_lines.txt">document name</a> and <a href="https://github.com/vdragan1993/serbian-document-network/blob/master/dataset/new_data/all_num_lines.txt">document id</a>, and this documents are our legislation network. Also, references detected in every document can be found in <a href="https://github.com/vdragan1993/serbian-document-network/tree/master/dataset/new_data/graph">dataset/new_data/graph/</a>.
In order to evaluate process accuracy, full-explicit references were manually detected in 10 randomly selected documents. After validation, obtained accuracy of references extraction process was: %.
End of explanation
"""


# reading num - name dictionary
num_doc_mapper = load_num_doc('dataset/new_data/new_num_doc_sorted.txt')

# reading and creating network using networkx
graph = nx.read_edgelist('dataset/new_data/all_num_lines.txt', create_using=nx.DiGraph(), nodetype=int)
print(nx.info(graph))

# highest degrees
print("Nodes with highest degrees: (in + out)\n")
degrees_high = sort_dictionary_by_value_desc(graph.degree())
degrees_high_count = Counter(degrees_high)
for k, v in degrees_high_count.most_common(5):
    print('%s: %i (%i + %i)\n' % (num_doc_mapper[k], v, graph.in_degree(k), graph.out_degree(k)))

# lowest degrees
print("Nodes with lowest degrees: (in + out)\n")
deegrees_low = sort_dictionary_by_value_asc(graph.degree())
deegrees_low_count = islice(deegrees_low.items(), 0, 5)
for k, v in deegrees_low_count:
    print('%s: %i (%i + %i)\n' % (num_doc_mapper[k], v, graph.in_degree(k), graph.out_degree(k)))

# highest in_degrees
print("Nodes with highest in degrees:\n")
in_degrees_high = sort_dictionary_by_value_desc(graph.in_degree())
in_degrees_high_count = Counter(in_degrees_high)
for k, v in in_degrees_high_count.most_common(5):
    print('%s: %i\n' % (num_doc_mapper[k], v))

# highest out_degrees
print("Nodes with highest out degrees:\n")
out_degrees_high = sort_dictionary_by_value_desc(graph.out_degree())
out_degrees_high_count = Counter(out_degrees_high)
for k, v in out_degrees_high_count.most_common(5):
    print('%s: %i\n' % (num_doc_mapper[k], v))

"""
Explanation: Network Science Measures
End of explanation
"""


print("Graph diameter: {0}".format(this_igraph.diameter()))
print("Average path length: {0}".format(this_igraph.average_path_length()))
print("Graph density: {0}".format(this_igraph.density()))

"""
Explanation: The <b>diameter</b> of a graph is the maximum eccentricity of any vertex in the graph. That is, d is the greatest distance between any pair of vertices.<br/>
For directed graphs, the <b>graph density</b> is defined as D = |E| / (|V| (|V| - 1)) where E is the number of edges and V is the number of vertices in the graph.
End of explanation
"""


print("Cycles: \n")
cycles = nx.algorithms.find_cycle(graph)
for cycle in cycles:
    print('%s --> %s\n' % (num_doc_mapper[cycle[0]], num_doc_mapper[cycle[1]]))

"""
Explanation: Cycles
Cycle found via a directed, depth-first traversal.
End of explanation
"""


print("Nodes with highest average degree connectivity: \n")
adc_high = sort_dictionary_by_value_desc(assort.average_degree_connectivity(graph))
adc_high_count = Counter(adc_high)
for k, v in adc_high_count.most_common(5):
    print('k=%i: %f\n' % (k, v))

"""
Explanation: Assortativity
The average degree connectivity is the average nearest neighbor degree of nodes with degree k.
End of explanation
"""


print("Nodes with highest degree centrality: \n")
dc_high = sort_dictionary_by_value_desc(central.degree_centrality(graph))
dc_high_count = Counter(dc_high)
for k, v in dc_high_count.most_common(5):
    print('%s: %f\n' % (num_doc_mapper[k], v))

print("Nodes with highest in-degree centrality: \n")
idc_high = sort_dictionary_by_value_desc(central.in_degree_centrality(graph))
idc_high_count = Counter(idc_high)
for k, v in idc_high_count.most_common(5):
    print('%s: %f\n' % (num_doc_mapper[k], v))
    
print("\n\nNodes with highest out-degree centrality: \n")
odc_high = sort_dictionary_by_value_desc(central.out_degree_centrality(graph))
odc_high_count = Counter(odc_high)
for k, v in odc_high_count.most_common(5):
    print('%s: %f\n' % (num_doc_mapper[k], v))

"""
Explanation: Centrality
In graph theory and network analysis, indicators of centrality identify the most important vertices within a graph.
Degree
The <b>degree centrality</b> for a node v is the fraction of nodes it is connected to.<br/>
The <b>in-degree centrality</b> for a node v is the fraction of nodes its incoming edges are connected to.<br/>
The <b>out-degree centrality</b> for a node v is the fraction of nodes its outgoing edges are connected to.
End of explanation
"""


print("Nodes with highest closeness centrality: \n")
cc_high = sort_dictionary_by_value_desc(central.closeness_centrality(graph))
cc_high_count = Counter(cc_high)
for k, v in cc_high_count.most_common(5):
    print('%s: %f\n' % (num_doc_mapper[k], v))

"""
Explanation: Closeness
Closenes centrality of a node u is the reciprocal of the sum of the shortest path distances form u to all n-1 other nodes. Thus the more central a node is, the closer it is to all other nodes.
End of explanation
"""


print("Nodes with lowest betweenness centrality: \n")
bc_low = sort_dictionary_by_value_asc(central.betweenness_centrality(graph))
bc_low_count = islice(bc_low.items(), 0, 5)
for k, v in bc_low_count:
    print('%s: %f\n' % (num_doc_mapper[k], v))

"""
Explanation: Betweenness
Computing the shortest-path betweenness centrality for nodes. Betweenness centrality of a node v is the sum of the fraction of all-pairs shortest paths that pass through v. <br/>
Betweenness centrality quantifies the number of times a node actts as a bridge along the shortest path between two other nodes.
End of explanation
"""


# highest PR
print("Nodes with highest PageRank values: \n")
pr_high = sort_dictionary_by_value_desc(nx.algorithms.pagerank(graph))
pr_high_count = Counter(pr_high)
for k, v in pr_high_count.most_common(10):
    print('%s: %f\n' % (num_doc_mapper[k], v))

# lowest PR
print("Nodes with lowest PageRank values: \n")
pr_low = sort_dictionary_by_value_asc(nx.algorithms.pagerank(graph))
pr_low_count = islice(pr_low.items(), 0, 10)
for k,  v in pr_low_count:
    print('%s: %f\n' % (num_doc_mapper[k], v))

"""
Explanation: Page Rank
PageRank computes a ranking of the nodes in the graph based on the structure of the incoming links.
End of explanation
"""


# highest K
print("Nodes with highest Katz centrality: \n")
k_high = sort_dictionary_by_value_desc(nx.katz_centrality_numpy(graph))
k_high_count = Counter(k_high)
for k, v in k_high_count.most_common(10):
    print ('%s: %f\n' % (num_doc_mapper[k], v))

# lowest Katz
print("Nodes with lowest Katz centrality: \n")
k_low = sort_dictionary_by_value_asc(nx.katz_centrality_numpy(graph))
k_low_count = islice(k_low.items(), 0, 10)
for k, v in k_low_count:
    print('%s: %f\n' % (num_doc_mapper[k], v))

"""
Explanation: Katz
Katz centrality computes the centrality for a node based on the centrality of its neigbors. It is a generalization of the eigenvector centrality.
End of explanation
"""


# highest ev
print("Nodes with highest Eigenvector centrality: \n")
ev_high = sort_dictionary_by_value_desc(nx.eigenvector_centrality(graph))
ev_high_count = Counter(ev_high)
for k, v in ev_high_count.most_common(10):
    print('%s: %f\n' % (num_doc_mapper[k], v))

# lowest ev
print("Nodes with lowest Eigenvector centrality: \n")
ev_low = sort_dictionary_by_value_asc(nx.eigenvector_centrality(graph))
ev_low_count = islice(ev_low.items(),0, 10)
for k, v in ev_low_count:
    print ('%s: %f\n' % (num_doc_mapper[k], v))

"""
Explanation: Eigenvector
Eigenvector centrality computes the centrality for a node based on the centrality of its neighbors. It assigns relative scores to all nodes in the network based on the concept that connections to high-scoring nodes contribute mode to the score of the node in question that equal connections to low-scoring nodes.
End of explanation
"""


print("Global Clustering Coefficient: {0}".format(nx.transitivity(graph)))

"""
Explanation: Clustering
In graph theory, a <b>clustering coefficient</b> is a measure of the degree to which nodes in a graph tend to cluster together. The global version of this measure was designed to give an overall indication of the clustering in the network.<br/>
The <b>global clustering coefficient</b> is based on triplets of nodes. A triplet consists of three connected nodes. A triangle therefore includes three closed triplets, one centered on each of  the nodes. The global clustering coefficient is the number of closed triplets (or 3 x triangles) over the total number of triplets (both open and closed):<br/>
<b>C = 3 x number of triangles / number of connected triplets of vertices = number of closed triplets / number of connected triplets of vertices</b>.
End of explanation
"""


dendogram = this_igraph.community_walktrap(steps=10)
clusters = dendogram.as_clustering()
membership = clusters.membership
memberships_dict = {}
for name, membership in zip(this_igraph.vs["name"], membership):
    if membership not in memberships_dict:
        memberships_dict[membership] = []
    memberships_dict[membership].append(num_doc_mapper[int(name)])
print("Total number of communities: {0}".format(len(memberships_dict)))
# number of members 
memership_nums = {}
for k in memberships_dict:
    memership_nums[k] = len(memberships_dict[k])

# top communities
print("Communities with most members: \n")
mem_num_high = sort_dictionary_by_value_desc(memership_nums)
mem_num_high_count = Counter(mem_num_high)
for k, v in mem_num_high_count.most_common(5):
    print('\nCommunity %i with %i members:' % (k, v))
    #print(memberships_dict[k])

# bottom communities
print("Communities with least members: \n")
mem_num_low = sort_dictionary_by_value_asc(memership_nums)
mem_num_low_count = islice(mem_num_low.items(), 0, 5)
for k, v in mem_num_low_count:
    print('\nCommunity %i with %i members:' % (k, v))
    print(memberships_dict[k])

"""
Explanation: Community Detection
<a href="https://www-complexnetworks.lip6.fr/~latapy/Publis/communities.pdf">Latapy and Pons algorithm</a> for community detection base on random walks. This algorithm (Walktrap) can be adatped to directed edges, weights, and can alter resolution by walk length.
<b>Basic idea:</b> Simulate many short random walks on the network and compute pairwise similarity measures bassed on these walks. Use these similarity values to aggreggate vertices into communities. 
<b>Time Complexity:</b> depends on walk length, O(|V|^2 log |V|) typically.
End of explanation
"""


