<reponame>jsheng112/Rec-System-Reselection
# -*- coding: utf-8 -*-
"""personalized_reranking.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/1oZprviDSd9H3dkN93hz3Cj9Yuu1bW8x_
"""

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import os

import random
random.seed(415)

# read in baseline results
test_results = pd.read_csv('test_results.csv')
test_results = pd.DataFrame(test_results, columns=['userId', 'movieId', 'actual', 'cf_predictions'])
print(test_results.head())

# read in training data and testing data 
train_data = pd.read_csv('data_train_set.csv')
test_data = pd.read_csv('data_test_set.csv')

# combine training and testing data 
train_test_data = [train_data, test_data]
ratings = pd.concat(train_test_data)
ratings.reset_index(drop=True, inplace=True)

# separate movies into short head and long tail
df_movies_count_ratings = pd.DataFrame(ratings.groupby('movieId').size(), columns=['count']).reset_index()
total_num = len(df_movies_count_ratings)
short_head = int(0.2 * total_num)
df_short_head = df_movies_count_ratings.nlargest(short_head, 'count')['movieId'].unique()
df_long_tail = df_movies_count_ratings.nsmallest(total_num-short_head, 'count')['movieId'].unique()

# classify movies into short-head and long-tail
def movie_cat(df_long_tail, df_short_head, movieId):
  if (movieId in df_long_tail):
    return "long_tail"
  elif (movieId in df_short_head):
    return "short_head"
  else:
    return "other"

# create a dataframe containing category for each movie
unique_movies = sorted(ratings.movieId.unique())
categories = []
for movieId in unique_movies:
  categories.append(movie_cat(df_long_tail, df_short_head, movieId))

movie_category_data = {'category': categories}
df_movie_category = pd.DataFrame(data=movie_category_data, index = unique_movies)

# calculate user short-head preference and long-tail preference
def preference(df_movie_cat, ratings, user_id):
  user_ratings = ratings.loc[(ratings['userId'] == user_id) & (ratings['rating'] >=3)]['movieId'].unique();
  total = len(user_ratings)
  num_long_tail = [x for x in user_ratings if df_movie_cat.loc[x].category == 'long_tail']
  num_short_head = [x for x in user_ratings if df_movie_cat.loc[x].category == 'short_head']
  if (total > 0):
    a = len(num_short_head)/total
    b = len(num_long_tail)/total
    return a, b
  else:
    return 0, 0

# calculate user short-head preference and long-tail preference for each user
user_id = test_results.userId.unique();
long_tail_prefs = [];
short_head_prefs = [];
for u in user_id:
  short_head, long_tail = preference(df_movie_category, ratings, u)
  short_head_prefs.append(short_head)
  long_tail_prefs.append(long_tail)

# save user preference into data frame
df_user_prefs = {'userId': user_id, 'short_head_prefs': short_head_prefs, 'long_tail_prefs': long_tail_prefs}
df_user_prefs = pd.DataFrame(data=df_user_prefs)
df_user_prefs.to_csv("user_preferences.csv")

# get user predictions according to predicted ratings sorted in descending order
def get_users_predictions(user_id, formatted_results):
    recommended = pd.DataFrame(formatted_results.loc[user_id])
    recommended.columns = ["predicted_rating"]
    recommended = recommended.sort_values('predicted_rating', ascending=False)    
    return recommended.index.tolist()

# collect the movies for a user into a list
def get_movie_list(user_id, test_results):
  return test_results.loc[test_results['userId'] == user_id].sort_values('actual', ascending=False)['movieId'].unique().tolist()


# format test data
user_id = test_results.userId.unique().tolist()
new_df = {}
new_df['userId'] = user_id
new_list = []
for user in user_id:
  movies_temp = get_movie_list(user, test_results)
  new_list.append(movies_temp)
new_df['actual'] = new_list
test_results_with_recs = pd.DataFrame(data=new_df)

# get the predicted recommendations for all users
formatted_results = test_results.pivot_table(index='userId', columns='movieId', values='cf_predictions').fillna(0)
cf_recommendations = []
for user in user_id:
    cf_predictions = get_users_predictions(user, formatted_results)
    cf_recommendations.append(cf_predictions)
        
test_results_with_recs['cf_predictions'] = cf_recommendations
test_results_with_recs.head()

# get the top n user predictions
def get_users_predictions_n(user_id, n, cf_recommendations_top_10):
  top_n_recommendations = cf_recommendations_top_10.loc[cf_recommendations_top_10['userId'] == user_id]['cf_predictions'].item()
  top_n_recommendations = top_n_recommendations[0:n]
  return top_n_recommendations

# For each user, create prediction recommendation list based on top 10 ranking items
cf_recommendations_top_10 = []
for user in user_id:
    cf_predictions = get_users_predictions_n(user, 10, test_results_with_recs)
    cf_recommendations_top_10.append(cf_predictions)

test_results_with_recs['cf_predictions_10'] = cf_recommendations_top_10
test_results_with_recs.head()

# perform Personalized Reranking calculation
def rerank(df_user_prefs, cf_model, mid, uid, df_movie_cat, current_list, user_preferences):
  # For each user, construct a candidate set and long-tail preference:
  # Loop through each row to construct a reranked recommendation for each user:
  recsize = 10
  longtail_weight = 0.8
  base_score = cf_model.loc[uid][mid]
  category = df_movie_cat.loc[mid]['category'];
  if category == 'long_tail':
    category_pref = user_preferences['long_tail_prefs'].item();
    category_sum = 0
    for x in current_list:
      if df_movie_cat.loc[x].category == 'long_tail':
        category_sum +=1;
    if len(current_list) == 0:
      category_ratio = 0
    else:
      category_ratio = category_sum/len(current_list)
  elif category == 'short_head':
    category_pref = user_preferences['short_head_prefs'].item();
    category_sum = 0
    for x in current_list:
      if df_movie_cat.loc[x].category == 'short_head':
        category_sum +=1;
    if len(current_list) == 0:
      category_ratio = 0
    else:
      category_ratio = category_sum/len(current_list)
  else:
    return base_score
  weighted_base_score = (1- longtail_weight) * base_score
  weighted_category_booster = longtail_weight * category_pref * (1-category_ratio) ** recsize
  new_score = weighted_base_score + weighted_category_booster
  return new_score


# for each user get top 100 movies and perform reranking on those top 100 movies
def reranking_list(df_user_prefs, test_results_with_recs, uid, df_movie_category, formatted_results):
  user_preferences = df_user_prefs.loc[df_user_prefs['userId'] == uid];
  starting_recs = get_users_predictions_n(uid, 100, test_results_with_recs)
  current_list = []
  for i in range(10):
    largest = rerank(df_user_prefs, formatted_results, starting_recs[0], uid, df_movie_category, current_list, user_preferences);
    largest_movie = starting_recs[0];
    for mid in starting_recs:
      score = rerank(df_user_prefs, formatted_results, mid, uid, df_movie_category, current_list, user_preferences);
      if (score > largest):
        largest_movie = mid
        largest = score
    current_list.append(largest_movie)
    starting_recs.remove(largest_movie)
  return current_list

# For each user, create a list of recommendations according to Personalized Reranking
personalized_reranking_recs = []
count = 0
for u in user_id:
  new_recs = reranking_list(df_user_prefs, test_results_with_recs, u, df_movie_category, formatted_results)
  personalized_reranking_recs.append(new_recs)
  count = count + 1

test_results_with_recs['personalized_reranking_predictions'] = personalized_reranking_recs
test_results_with_recs.head()

# create a data frame to hold Personalized Reranking results
personalized_reranking_results = {"userId": user_id, 'personalized_reranking': personalized_reranking_recs}
df_personalized_reranking = pd.DataFrame(data = personalized_reranking_results)

# save Personalized Reranking results and all results
df_personalized_reranking.to_csv("personalized_reranking_results.csv")
test_results_with_recs.to_csv("test_results_with_personalized_reranking_results_added.csv")