# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# # Project

import math
import itertools
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import networkx as nx
from scipy import sparse
import scipy.sparse.linalg
from pyunlocbox import functions, solvers

# ## <NAME>

# +
credits = pd.read_csv('../../data/tmdb_5000_credits.csv')
credits = credits[credits.cast != '[]']


movies = pd.read_csv('../../data/tmdb_5000_movies.csv')
movies.drop(['homepage', 'keywords','original_language','overview','release_date','spoken_languages', \
             'status','title','tagline','vote_count'\
            ], \
            axis=1, \
            inplace=True \
           )
# -

credits.drop(['title', 'crew'], axis=1, inplace=True)
credits['cast_id'] = credits['cast'].apply(lambda row: list(set(pd.read_json(row)['id'])))
#credits['cast_name'] = credits['cast'].apply(lambda row: list(set(pd.read_json(row)['name'])))
#credits['gender'] = credits['cast'].apply(lambda row: list(set(pd.read_json(row)['gender'])))

# +
frames = pd.DataFrame()
new_df = pd.DataFrame()

for idx, film in credits.iterrows():
    cast_df = pd.DataFrame(eval(credits['cast'][idx]))
    cast_df['credits'] = idx
    cast_df = cast_df.drop(['character','order', 'credit_id', 'cast_id'],axis = 1)  
    
    frames = [new_df, cast_df]
    new_df = pd.concat(frames, join = 'outer', ignore_index=True)
# -

discount_old = credits['cast_id'].apply(pd.Series).stack().value_counts()
discount_old = list(discount_old[discount_old > 4].index.astype(int))
#discount_old[:10]

nodes_df = new_df['credits'].groupby([new_df.gender, new_df.id, new_df.name]).apply(list).reset_index()
nodes_df = nodes_df[nodes_df['gender'].isin(['1','2'])]
discount_1 = nodes_df['id'].tolist()
discount = [x for x in discount_old if x in discount_1]
#nodes_df = nodes_df[nodes_df.id.isin(discount)]
#nodes_df.drop(columns=['credits'], inplace=True)
#nodes_df = nodes_df[nodes_df['gender'].isin(['1','2'])]

print('Old Values of the Discount')
print(discount_old[:10])
print(len(discount_old))
print('New Values of the Discount')
print(discount[:10])
print(len(discount))

# +
credits['cast_id'] = credits['cast_id'].apply(lambda x: [y for y in x if y in discount])
credits['edges'] = credits['cast_id'].apply(lambda x: list(itertools.combinations(x, 2)))
edges = list(credits['edges'].apply(pd.Series).stack())
edges[0:5]

edges_df = pd.DataFrame(edges)
# -

#Normally the number of edges was:
print('Normally the number of edges was:')
print(edges_df)

edges_df = edges_df.merge(nodes_df, left_on = 0, right_on='id', how='inner').drop(columns=['name','credits'])
edges_df = edges_df.merge(nodes_df, left_on = 1, right_on='id', how='inner').drop(columns=['name','credits'])

edges_df.head()

edges_df['same_gender']=0
for i in range(len(edges_df)):
    if edges_df['gender_x'][i]==edges_df['gender_y'][i]:
        edges_df['same_gender'][i]=1

edges_df = edges_df.drop(columns=['gender_x','id_x','gender_y','id_y'])
edges_df =edges_df[edges_df['same_gender'] == 1]
edges_df = edges_df.drop(columns=['same_gender'])
edges_df = edges_df.reset_index(drop=True)

len(edges_df)

edges_df.head()

# +
discarded_movies = set()

for idx, movie in credits.iterrows():
    if len(movie['edges']) == 0:
        discarded_movies.add(movie['movie_id'])

print(len(discarded_movies)) 
# -

credits = credits[~credits['movie_id'].isin(discarded_movies)]
credits.head()

movies['profit'] = movies['revenue']-movies['budget']
movies_credits = movies.merge(credits, left_on='id', right_on='movie_id', how='inner').drop(columns=['movie_id'])

movies_credits = movies_credits[movies_credits.genres != '[]']
movies_credits['genre_id'] = movies_credits['genres'].apply(lambda row: list(pd.read_json(row)['id']))
movies_credits['genre_name'] = movies_credits['genres'].apply(lambda row: list(pd.read_json(row)['name']))

genre = movies_credits[['cast_id', 'genre_id', 'genre_name']]
genre.loc[:, 'genre_id_disc'] = genre['genre_id'].apply(lambda x: x[0])
genre.loc[:, 'genre_name_disc'] = genre['genre_name'].apply(lambda x: x[0])

genre_df = pd.DataFrame(genre.cast_id.tolist(), index=genre.genre_name_disc).stack().reset_index(name='cast_id')[['cast_id','genre_name_disc']]
most_freq_genre = genre_df.groupby(['cast_id']).agg(lambda x:x.value_counts().index[0])

profit_df = pd.DataFrame(movies_credits.cast_id.tolist(), index=movies_credits.profit).stack().reset_index(name='cast_id')[['cast_id','profit']]
profit_df['cast_id'] = profit_df.cast_id.astype(int)
profit_df = profit_df.groupby('cast_id', as_index=False).mean()
profit_df.set_index('cast_id', inplace=True)
profit_df.head()


profit_df = ((profit_df['profit']/(10**7)).round(0))*(10**7)
profit_df = profit_df.to_frame()

ranking_df = pd.DataFrame(movies_credits.cast_id.tolist(), index=movies_credits.vote_average).stack().reset_index(name='cast_id')[['cast_id','vote_average']]
ranking_df['cast_id'] = ranking_df.cast_id.astype(int)
ranking_df = ranking_df.groupby('cast_id', as_index=False).mean()
ranking_df.set_index('cast_id', inplace=True)
ranking_df.head()
     

ranking_df = round(ranking_df['vote_average'] * 2) / 2
ranking_df = ranking_df.to_frame()

actors = ranking_df.merge(most_freq_genre, on='cast_id', how='inner')
actors = actors.merge(profit_df, on='cast_id', how='inner')

actors = actors.reset_index()
actors.head()

#nodes_df = new_df['credits'].groupby([new_df.gender, new_df.id, new_df.name]).apply(list).reset_index()
nodes_df = nodes_df[nodes_df.id.isin(discount)]
nodes_df.drop(columns=['credits'], inplace=True)
#nodes_df = nodes_df[nodes_df['gender'].isin(['1','2'])]

actors = actors.merge(nodes_df, left_on = 'cast_id', right_on='id', how='inner').drop(columns=['cast_id'])

actors[actors['name']=='<NAME>']

actors.sort_values(by='profit', ascending=False)

# +
#features = nodes_df.set_index('id').drop('name', axis=1)
#features.head()
# -

discount_df = pd.DataFrame(discount)
features = discount_df.merge(actors, left_on = 0, right_on='id', how='inner').drop(columns=[0])
features.head()

# ## Doing the Adjacency again
# Cause we took out some genders and our size went from 3766 to 3500

edges = edges_df.values.tolist()
len(edges)

# +
adj = pd.DataFrame(np.zeros(shape=(len(discount),len(discount))), columns=discount, index=discount)
for e1, e2 in edges:
    if e1 in discount and e2 in discount:
        adj.at[e1, e2] += 1
        adj.at[e2, e1] += 1
    else:
        edges.remove((e1,e2))

adj.head()

# +
#One outlier, its <NAME>, ID=90596, index number 3415

# +
adjacency = adj.values
adj_max = adjacency.max()
adjacency = np.vectorize(lambda x: x/adj_max)(adjacency)

adjacency = pd.DataFrame(adjacency)
# -

adjacency.head()

#IF WE NEED NON WEIGHTED ADJACENCY
adjacency_non_weighted = np.copy(adjacency)
adjacency_non_weighted[adjacency_non_weighted > 0] = 1
adjacency_non_weighted = np.asmatrix(adjacency_non_weighted)

graph = nx.from_numpy_array(adjacency_non_weighted)

node_props = features.to_dict()

for key in node_props:
    nx.set_node_attributes(graph, node_props[key], key)

graph.node[0]

nx.draw_spring(graph)

nx.write_gexf(graph, 'CoAppGenderAdjGephiFile.gexf')

adjacency.to_csv("CoAppGenderAdjacency.csv")

features.to_csv("CoAppGenderFeatures.csv")

edges_df.to_csv("CoAppGenderEdges.csv")



StarcoderdataJupyterScriptsDedupFiltered