# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 2
#     language: python
#     name: python2
# ---

# ## Web and Social Project Code
#
# using Python 2.7

# %matplotlib inline
import pandas as pd
from collections import Counter
import matplotlib.pyplot as plt
import numpy as np
import networkx as nx

data  = open("Data/msnbc990928.seq")

lines = data.readlines()

lines[:10]

# ## Data

data2 = [lin.strip().split() for lin in lines]
data2[:5]    

len_line=[len(lin) for lin in data2]
print len_line[:5]
print sum(len_line)/len(len_line)

pd.DataFrame(len_line).describe().to_csv("Orig_metrics.csv")

# ## Most Common Landing Page

first_pages  =  [lin[0] for lin in data2]
first_pages[:10]

Counter(first_pages).most_common()

first_pages_dict = Counter(first_pages)

# ## Scatter plot to show frequency of page when it is a landing page

plt.scatter(x = list(first_pages_dict.keys()), y=list(first_pages_dict.values()))
plt.show()

# ## Scatter plot to show frequency of page when it is an exit page

last_pages  =  [lin[-1] for lin in data2]
last_pages[:10]

Counter(last_pages).most_common()

last_pages_dict = Counter(last_pages)

plt.scatter(x = list(last_pages_dict.keys()), y=list(last_pages_dict.values()))
plt.show()

da = np.array(data2)

# ## Max and Min No of Page Views

cnt = Counter(np.concatenate(da))

cnt.most_common()

sorted(Counter(np.concatenate(da)).keys())

np.concatenate(da).shape

pages = ['1','2', '3', '4', '5', '6','7', '8', '9', '10', 
         '11', '12', '13', '14', '15', '16', '17']

# ## Exit Rate

exit_rate={}
for page in pages:
    # Number of times a page appears in the sequence
    page_true = len([lin for lin in data2 if page in lin ])
    # Number of times a page is an exit page
    exit_pages_no = len([lin for lin in data2 if page == lin[-1] ])
    exit_rate[page] = exit_pages_no / (page_true*1.0)
exit_rate

tab = pd.DataFrame()

tab['pages'] = sorted(exit_rate.keys())
tab

tab['exit_rate']=[exit_rate[i] for i in sorted(exit_rate.keys())]
tab.T

# ## Bounce Rate

bounce_rate={}
for page in pages:
    # Number of times a page is bounced
    bounce_pages_no = len([lin for lin in data2 if len(lin)==1 and lin[0] == page])
    # Number of times a page is a potential bounce
    page_b_true = len([lin for lin in data2 if lin[0]==page])
    bounce_rate[page] = bounce_pages_no / (page_b_true*1.0)
bounce_rate

tab['bounce_rate']=[bounce_rate[i] for i in sorted(bounce_rate.keys())]

tab['page_counts']=[cnt[i] for i in sorted(cnt.keys())]

tab.set_index('pages').T.describe()

tab.to_csv("metrics.csv")
tab.set_index('pages').T.describe().to_csv("metrics_desc.csv")

tab.set_index('pages').T

tab.T

# ## Incidence Matrix Code

incidence_matrix= np.zeros((len(pages), len(pages)))

# ## Pages Name

pages_name = "frontpage news tech local opinion on-air misc weather msn-news health living business msn-sports sports summary bbs travel".split()
pages_name, len(pages_name)

# ## Mapping Pages and Page Names

mapping = dict(zip(pages, range(len(pages))))
mapping

 map_df = pd.DataFrame([i for i in sorted(mapping.values())])

mapping_original = dict((zip(range(len(pages_name)), pages_name)))
mapping_original
map_df['pages']=[mapping_original[i] for i in sorted(mapping_original.keys())]
map_df.to_csv("mappings.csv")

# ## Create an incidence matrix based on co-occurrence

# +
incidence_matrix= np.zeros((len(pages), len(pages)))

for line in data2:  # For Each seq in the file
    length = len(line)
    if length>=2:
        #print line
        for i in range(length-1):  # for each element in the file 
            tmp = (line[i], line[i+1])
            incidence_matrix[mapping[tmp[0]], mapping[tmp[1]]]+=1
                        #print k, j , tmp

print(incidence_matrix.shape)
# -

incidence_matrix.mean(), incidence_matrix.min(), incidence_matrix.max()

incidence_matrix.shape

matrix2 = incidence_matrix >= incidence_matrix.mean() 
matrix2.shape

G = nx.from_numpy_matrix(incidence_matrix)

nx.info(G)

G = nx.relabel_nodes(G, mapping_original)

type(G)

G.is_directed()

G.degree()

nx.degree_centrality(G)

nx.betweenness_centrality(G)

nx.eigenvector_centrality(G)

G.edges(data=True)[:10]

nx.connectivity.average_node_connectivity(G)

nx.closeness_centrality(G)

list(nx.clique.find_cliques(G))

nx.pagerank(G, alpha=1)

# ## Graph based on Incidence matrix with threshod

thres = incidence_matrix.mean()
thres

G = nx.from_numpy_matrix(incidence_matrix>=thres)

nx.info(G)

G = nx.relabel_nodes(G, mapping_original)

type(G)

G.is_directed()

deg = G.degree()

dc = nx.degree_centrality(G)

bc = nx.betweenness_centrality(G)

ec=nx.eigenvector_centrality(G)

cc=nx.closeness_centrality(G)

G.edges(data=True)[:10]

list(nx.clique.find_cliques(G))

pr = nx.pagerank(G, alpha=1, max_iter=1000)

nx.density(G)

tabc = pd.DataFrame()

tabc['pages'] = sorted(G.nodes())
tabc = tabc.set_index('pages')
tabc.T

tabc['degree_centrality']= [dc[i] for i in sorted(dc.keys())]
tabc

tabc['degree']= [deg[i] for i in sorted(deg.keys())]
tabc

tabc['betweeness_centrality']= [bc[i] for i in sorted(bc.keys())]
tabc

tabc['eigenVector_centrality']= [ec[i] for i in sorted(ec.keys())]
tabc

tabc['closeness_centrality']= [cc[i] for i in sorted(cc.keys())]
tabc

tabc['page_rank']= [pr[i] for i in sorted(pr.keys())]
tabc

tabc.to_csv("Graph_metrics.csv")
tabc.describe().to_csv("Graph_metrics_desc.csv")

tabc.T

G.nodes()


# ## For Finding the suggestion Triads

def get_recommendation(pair, data):
    '''Find the most common triad for the given pair of pages, return as a list of values'''
    most_occured_page=[]
    occured_page=[]
    for line in data:
        if len(line)>=3:
            #print line
            length= len(line)
            for i in range(length):
                if i < length-2:
                    tmp = (line[i], line[i+1], line[i+2])
                    #print 'tmp', tmp, pair, set(tmp).intersection(set(pair))
                    if pair[0]==tmp[0] and pair[1]==tmp[1]:
                        #print pair, '-', tmp
                        most_occured_page.append(tmp[-1])
                        occured_page.append(tmp)    
    rec = Counter(occured_page)
    return rec.most_common()


pair =('6', '5')
get_recommendation(pair, data2)

pair = ("2", "5")
get_recommendation(pair, data2)

pd.DataFrame(get_recommendation(("2", "5"), data2)).to_csv("Recommend.csv")

StarcoderdataJupyterScriptsDedupFiltered