lastnode = 5000

datafile = open('/var/datasets/wdc/small-pld-arc')

G = nx.DiGraph()

for line in datafile:
    ijstr = line.split('\t')
    
    i=int(ijstr[0])
    j=int(ijstr[1])
    
    if i>lastnode:
        break
    if j>lastnode:
        continue
    G.add_edge(i,j)
    
datafile.close()
Gorig = G.copy()

indexfile = open('/var/datasets/wdc/small-pld-index')
index = {}

for line in indexfile:
    namei = line.split('\t')
    
    name=namei[0]
    i=int(namei[1])
    
    if i>lastnode:
        break

    index[i]=name
    
indexfile.close()

def cleanupgraph(G):
    comp = nx.weakly_connected_components(G.copy())
    for c in comp:
        if len(c)<4:
            G.remove_nodes_from(c)

def graphcleanup(G):
    for (node, deg) in G.degree_iter():
            if deg==0:
                G.remove_node(node)
            elif deg==1:
                if G.degree((G.predecessors(node) + G.successors(node))[0]) == 1:
                    G.remove_node(node)
            elif deg==2 and G.in_degree(node)==1:
                if (G.predecessors(node) == G.successors(node)) and G.degree((G.predecessors(node) + G.successors(node))[0]) == 2:
                    G.remove_node(node)

cleanupgraph(G)

G.size()

Gorig.number_of_nodes()

Gorig.size()

"""
Explanation: Load data
End of explanation
"""


#from IPython.core.display import display_javascript
import json
from networkx.readwrite import json_graph

d = json_graph.node_link_data(G)
for node in d['nodes']:
    node['name']=node['id']
    node['value']=G.degree(node['id'])
    if True:
        node['group'] = node['id'] % 4
    else:
        if node['id']<10:
            node['group']=0#node['id'] % 4
        else:
            node['group']=1#node['id'] % 4
        
d['adjacency'] = json_graph.adjacency_data(G)['adjacency']
json.dump(d, open('rwgraph.json','w'))

%%html
<div id="d3-example"></div>
<style>
.node {stroke: #fff; stroke-width: 1.5px;}
.link {stroke: #999; stroke-opacity: .3;}
</style>
<script src="randomwalk.js"></script>

"""
Explanation: Convert to Javascript for interactivity
Adapted from:
http://nbviewer.ipython.org/github/ipython-books/cookbook-code/blob/master/notebooks/chapter06_viz/04_d3.ipynb
From:
http://networkx.github.io/documentation/latest/examples/javascript/force.html
End of explanation
"""


Javascript(filename='force.js')

L = nx.linalg.laplacianmatrix.directed_laplacian_matrix(G)
Linv = np.linalg.inv(L)

L.shape

n = L.shape[0]
Reff = np.zeros((n,n))

Gsparse = G.copy()

graphcleanup(Gsparse)

nodelookup={Gsparse.nodes()[idx]:idx for idx in range(len(Gsparse.nodes()))}

edge = np.zeros((n,1))
for (i,j) in Gsparse.edges_iter():
    edge[nodelookup[i]] = 1
    edge[nodelookup[j]] = -1
    Reff[nodelookup[i],nodelookup[j]] = edge.T.dot(Linv.dot(edge))
    edge[[nodelookup[i]]] = 0
    edge[[nodelookup[j]]] = 0

ReffAbs=np.abs(Reff)+np.abs(Reff.T)

"""
Explanation: Uses:
https://github.com/mbostock/d3/wiki/Force-Layout
http://bl.ocks.org/mbostock/4062045
End of explanation
"""


res = ReffAbs.reshape(n**2)
argp = np.argpartition(res,n**2-n)

mask = (ReffAbs < res[argp[-int(0.5*Gsparse.number_of_nodes())]]) & (ReffAbs >0)
for (i,j) in Gsparse.edges():
    if mask[nodelookup[i],nodelookup[j]]:
        Gsparse.remove_edge(i,j)

cleanupgraph(Gsparse)

d = json_graph.node_link_data(Gsparse)
for node in d['nodes']:
    node['name']=index[node['id']]
    node['value']=Gsparse.degree(node['id'])
    node['group']=index[node['id']][-3:]

json.dump(d, open('graph.json','w'))

Gorig.number_of_edges()

Gsparse.number_of_edges()

Gsparse.number_of_nodes()

GsparseAdj = nx.linalg.adjacency_matrix(Gorig).toarray()
GsparseAdj = nx.to_numpy_matrix(Gorig)
GsparseAdj[ReffAbs < res[argp[-300]]] = 0
Gsparse = nx.from_numpy_matrix?
Gsparse = nx.from_numpy_matrix
Gsparse = nx.from_numpy_matrix(GsparseAdj, create_using=nx.DiGraph())

Gsparse = nx.from_numpy_matrix

edge = np.zeros((n,1))
for i in range(n):
    if i % int(math.ceil((float(10)/100)*n)) == 0: 
        print int(math.floor(100*float(i)/n)), '%'
        
    edge[i] = 1

    for j in range(i+1, n):
        edge[j] = -1
        Reff[i,j] = edge.T.dot(Linv.dot(edge))
        edge[j] = 0
        
    edge[i] = 0

"""
Explanation: If you call
arr.argsort()[:3]
It will give you the indices of the 3 smallest elements.
array([0, 2, 1], dtype=int64)
So, for n, you should call
arr.argsort()[:n]
End of explanation
"""


