# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     name: python3
# ---

# + [markdown] id="BdK0MebeROxc"
# ANN

# + id="DCJvmJMEkNZL"
# !pip install surprise

# + id="mbKbHYVNSDiC"
# !pip install annoy

# + id="wcoF4uS2RSDQ"
from surprise import Dataset, SVD, Reader,NMF
from surprise.model_selection import cross_validate,train_test_split
from surprise.model_selection import GridSearchCV,RandomizedSearchCV
from surprise import accuracy
from hyperopt import tpe, fmin, hp, Trials, STATUS_OK,space_eval
import sys
import seaborn as sns
from timeit import Timer
from datetime import datetime
from surprise import Dataset, SVD, Reader,NMF
from surprise.model_selection import cross_validate,train_test_split
from surprise.model_selection import GridSearchCV,RandomizedSearchCV
from surprise import accuracy
from hyperopt import tpe, fmin, hp, Trials, STATUS_OK,space_eval
from collections import defaultdict
import random
from sklearn.metrics import mean_squared_error
from numpy.linalg import inv
import pandas as pd
import numpy as np
import math
from numpy.linalg import inv
from numpy.linalg import multi_dot
import matplotlib.pyplot as plt
import itertools
import warnings
warnings.filterwarnings("ignore")

# + id="G6pHnhtHRWhG"
data = pd.read_csv('/data/sample_data.csv')

# + colab={"base_uri": "https://localhost:8080/", "height": 424} id="95qj-DqdViDB" outputId="841de942-9a39-43ab-db68-3805e05fccce"
data.drop(columns=['Unnamed: 0','timestamp','date','year'],inplace=True)
data

# + [markdown] id="TA_Hdqgqyr_e"
# ## Splitting Main dataset to a train and a test dataset

# + id="NCHFQrfCU4wg"
user_freq=data.groupby(['userId']).size().reset_index(name='counts')
users_lt3=user_freq[user_freq['counts']<3][['userId']]
users_ge3=user_freq[user_freq['counts']>=3][['userId']]
train1 = pd.merge(data, users_lt3, on=['userId'],how='inner')
data1 = pd.merge(data, users_ge3, on=['userId'],how='inner')
random.seed(2)
test=data1.groupby('userId').sample(frac=.3, random_state=2)
test_idx = data1.index.isin(test.index.to_list())
train = train1.append(data1[~test_idx])


train_df = train
test_df = test

# + [markdown] id="VZat979Ayw16"
# ## Splitting Train data to a train and validation dataset for Hyperparameter Optimization of Annoy (ANN)

# + id="cs7MoPHtCf8L"
user_freq1 = data.groupby(['userId']).size().reset_index(name='counts')

users_lt2=user_freq1[user_freq1['counts']<2][['userId']]
users_ge2=user_freq1[user_freq1['counts']>=2][['userId']]
train2 = pd.merge(train, users_lt2, on=['userId'],how='inner')
data2 = pd.merge(train, users_ge2, on=['userId'],how='inner')
random.seed(2)

validation = data2.groupby('userId').sample(frac=0.2, random_state=5)
validation_idx = data2.index.isin(validation.index.to_list())
train_data = train2.append(data2[~validation_idx])

# + [markdown] id="ETpZp05do-r2"
# ##  Before Applying ANN, it is a prerequisite that we have our latent matrix i.e. U and V matrices ready.
# Therefore, I am reusing code from the SVD Collaborative Filtering model to get the latent matrix.

# + id="uRotfcuHVozJ"
reader = Reader(rating_scale=(1,5))
train = Dataset.load_from_df(train[['userId', 'movieId', 'rating']], reader=reader)
test = Dataset.load_from_df(test[['userId', 'movieId', 'rating']], reader=reader)
raw_ratings = test.raw_ratings
threshold = int(1 * len(raw_ratings))
A_raw_ratings = raw_ratings[:threshold]

test = test.construct_testset(A_raw_ratings)

# + id="AEEu8REZVqBr"
raw_ratings1 = train.raw_ratings
threshold = int(1 * len(raw_ratings1))
B_raw_ratings = raw_ratings1[:threshold]
train_test = train.construct_testset(B_raw_ratings)

# + [markdown] id="pzcLvKa6UziX"
# The hyperparameters are taken from the hyperparameter optimization performed in homework 2

# + colab={"base_uri": "https://localhost:8080/"} id="mud4Ib_JkqS3" outputId="aa9b2620-163f-45f8-95c7-b0b37919fd3c"
model = SVD(n_epochs=50,n_factors=15,reg_all=0.1,lr_all=0.02)
trainset = train.build_full_trainset()
model.fit(trainset)

# + colab={"base_uri": "https://localhost:8080/"} id="ReMhPFsZELHY" outputId="89ccd98f-b9a2-4c5b-c432-b8a5b45916cc"
# Retrieiving inner ids, as used by surprise package during model training
user_inner_ids = [x for x in trainset.all_users()]
item_inner_ids = [i for i in trainset.all_items()]

# All ids mapped back to values in the actual train set 
user_raw_ids = [trainset.to_raw_uid(x) for x in user_inner_ids]
item_raw_ids = [trainset.to_raw_iid(x) for x in item_inner_ids]

print(user_raw_ids[:5])
print(item_raw_ids[:5])

# + colab={"base_uri": "https://localhost:8080/"} id="q85th3jvlt_n" outputId="64fdf5f6-0a35-4b40-a780-546517d77dfb"
U = model.pu 
V = model.qi
print(U[0])
print(V[0])

# + colab={"base_uri": "https://localhost:8080/"} id="dlLaKcuDlg30" outputId="9b32a0dc-451a-4a0a-e6a0-f7c0a3b61e0e"
print(model.pu.shape) # (n_user, n_factors)
print(model.qi.shape) # (n_items, n_factors)

# + [markdown] id="t8U0hqU8x0x-"
# # Exploratory Data Analysis

# + [markdown] id="wC4jyFKHzAV8"
# ### Movie frequency per user for all users in our training dataset

# + id="_usOl4W7x20l"
user_freq = train_df.groupby('userId', as_index=False).agg(num_movies=('movieId','count'))

# + colab={"base_uri": "https://localhost:8080/", "height": 334} id="ZZz8S43W3Hfq" outputId="3105848b-a835-4c1c-e42a-706f9c6b0848"
plt.figure(figsize=(15,5))
plt.hist(user_freq['num_movies'], bins=100, edgecolor='b', log=True)
plt.xlim(0,)
plt.xticks(np.arange(0,1000,50))
plt.xlabel('Number of ratings')
plt.ylabel('Number of users')
plt.show()


# + [markdown] id="nGhwOGOTAUDg"
# We have a lot of users who have rated more than 50 ratings.
# We will be looking at using nearest neighbours to reduce the size of the the user-movie pairs in the train dataset.

# + colab={"base_uri": "https://localhost:8080/", "height": 334} id="-EcePdmn3Mpe" outputId="3c92b83f-8d9b-46c8-e2d8-66dbc4bb8c82"
plt.figure(figsize=(15,5))
plt.hist(user_freq['num_movies'], bins=100, edgecolor='b', log=True)
plt.xlim(0,200)
plt.xticks(np.arange(0,200,50))
plt.xlabel('Number of ratings')
plt.ylabel('Number of users')
plt.show()

# + [markdown] id="qz7GedxTBTOP"
# The latent matrix is a numpy array of shape (n_items, n_factors. The first raw corresponds to the first raw item id (see this: http://surprise.readthedocs.io/en/latest/FAQ.html#what-are-raw-and-inner-ids), the second correspond to the second raw item id, etc...

# + [markdown] id="qv_VReVyfiqO"
# # Annoy package 
#
# Annoy is a C++ library with Python bindings that builds random projection trees. An index is built with a forest of k trees, where k is a tunable parameter that trades off between precision and performance. It also creates large read-only, file-based data structures that are mapped into memory so that many processes can share the data.

# + id="udvhe0zyT3ih"
from annoy import AnnoyIndex

# + [markdown] id="ifxD8aljLO2O"
# HYPERPARAMETER OPTIMIZATION FOR ANN (FINDING THE OPTIMUM NUMBER OF TREES)

# + id="q3avO7qplzI6"
from google.colab import drive
drive.mount('/content/drive')


# + id="DQzM79oIQXdu"
def filter_nn_user(nn_sim, movie_id,movie_to_users): 
    if nn_sim[0] in movie_to_users[movie_id]:
      return True
    return False


# + id="Q6ZlFoUCPI75"
def calc_raw_uid(inner_id):
  return trainset.to_raw_uid(inner_id)


# + id="Yzmamj9CPJA4"
def calc_cosine(dist):
  return 1 - (dist/2)


# + id="WHW3nbh6awCX"
def Ann_predictions(train, test, n):
  train_users = train['userId'].unique()
  train_df_pivoted = train.pivot(index='userId', columns='movieId', values='rating')

  # Finding the average rating value per user
  avg_value = train_df_pivoted.mean(axis=1)
  user_mean_df = pd.DataFrame(avg_value,columns=['mean'])

  # Subtracting the average rating value from all the ratings of a user
  train_df_centered = train_df_pivoted.sub(train_df_pivoted.mean(axis=1), axis=0)

  movie_to_users = defaultdict(list)
  movie_to_users = train.groupby('movieId')['userId'].apply(list).to_dict()

  # Building Annoy Trees

  f = 15 # n_factors

  model_user_nn = AnnoyIndex(f, 'angular')  # Length of item vector that will be indexed
  for i,u in zip(user_inner_ids,U):
      model_user_nn.add_item(i, u)

  model_user_nn.build(n) # n trees 
  # model_user_nn.save('test_user.ann')

  # Finding the 100 nearest neighbors to a user
  def find_nn(user_id):
    return model_user_nn.get_nns_by_item(trainset.to_inner_uid(user_id), 101, include_distances=True)
  
  l = list(map(find_nn,train_users))

  matrix_similarity_with_nn = {}


  # Filling in similar users and their distances to all users 
  for user_id,nn_dist in zip(train_users,l):
    matrix_similarity_with_nn[user_id] = [(nn,sim) for nn,sim in zip(list(map(calc_raw_uid,nn_dist[0]))[1:],list(map(calc_cosine,nn_dist[1]))[1:])]

  # Prediction function to predict ratings for all user-movie pair in test/validation dataset 
  def predict_nn_rating(user_movie_pair):
    user_id, movie_id = user_movie_pair
    reduced_nn_sim = pd.DataFrame(filter(lambda x: filter_nn_user(x, movie_id, movie_to_users),matrix_similarity_with_nn[user_id]))
    prediction = user_mean_df.loc[user_id].values[0]

    if len(reduced_nn_sim)==0:
      return prediction

    u_list = reduced_nn_sim[0].to_list()
    nr = np.dot(train_df_centered.loc[u_list][movie_id],reduced_nn_sim[1])
    dr = np.sum(np.abs(reduced_nn_sim[1]))
    if(dr!=0):
      prediction += (nr/dr)
    return prediction


  preds = list(map(predict_nn_rating,[(user_id,movie_id) for user_id, movie_id in zip(test['userId'],test['movieId'])]))
  rmse = np.sqrt(mean_squared_error(test['rating'],preds))
  return rmse


# + id="Cyba2PWSbTgP"
rmse_validation = {}
rmse_validation[10] = Ann_predictions(train_data,validation,10)
rmse_validation[15] = Ann_predictions(train_data,validation,15)
rmse_validation[20] = Ann_predictions(train_data,validation,20)

df1 = pd.DataFrame(list(rmse_validation.items()),columns = ['num_of_trees','RMSE'])
df1.to_csv('/data/hyperparameters_rmse.csv')

# + id="ErEzRI-EwjMz"
rmse_validation = {}
rmse_validation[25] = Ann_predictions(train_data,validation,25)
rmse_validation[50] = Ann_predictions(train_data,validation,50)
rmse_validation[80] = Ann_predictions(train_data,validation,80)
df2 = pd.DataFrame(list(rmse_validation.items()),columns = ['num_of_trees','RMSE'])
df2.to_csv('/data/hyperparameters_rmse2.csv')

# + id="tHZuFkc9wnlq"
rmse_validation = {}
rmse_validation[60] = Ann_predictions(train_data,validation,60)
rmse_validation[70] = Ann_predictions(train_data,validation,70)
rmse_validation[90] = Ann_predictions(train_data,validation,90)

df3 = pd.DataFrame(list(rmse_validation.items()),columns = ['num_of_trees','RMSE'])
df3.to_csv('/data/hyperparameters_rmse_3.csv')

# + id="XvCCauj2gqIJ"
df1 = pd.read_csv("/data/hyperparameters_rmse.csv") 
df2 = pd.read_csv("/data/hyperparameters_rmse2.csv") 
df3 = pd.read_csv("/data/hyperparameters_rmse_3.csv") 

# + colab={"base_uri": "https://localhost:8080/", "height": 324} id="AXh6wMhkg-6G" outputId="979d5bcc-430c-4040-88dd-380ffdb6c528"
frames = [df1,df2,df3]
df = pd.concat(frames)
df.drop(columns=['Unnamed: 0'],inplace=True)
df = df.reset_index().drop(columns=['index'])
df = df.sort_values(by='num_of_trees')
df

# + colab={"base_uri": "https://localhost:8080/", "height": 295} id="hbWh08alKWe6" outputId="f1c436bd-b07e-4696-9de6-14fd3b1af6a8"
plt.scatter(df['num_of_trees'],df['RMSE'])
plt.plot(df['num_of_trees'],df['RMSE'])
plt.title("Number of trees  vs RMSE on Validation Dataset")
plt.xlabel("Number of trees build for Annoy ")
plt.ylabel("rmse on validation set")
plt.show()

# + colab={"base_uri": "https://localhost:8080/", "height": 324} id="8NNc2JhdgeeY" outputId="c71a7447-eb59-4e7f-9d36-55ff176f3f91"
df = df.sort_values(by='RMSE')
df

# + [markdown] id="uT-WxIU_gfgZ"
# We find the optimum number of trees to be 50. Hence we now use n = 50 for training our Annoy ANN model to find user-user similarities

# + [markdown] id="HmvsxKTIsJkJ"
# ## Using ANN to Fill the User-Movie Matrix by taking hyperparameter n=50

# + id="2sYKtuYEb6CE"
# Cross Join to Generate User - Movie Matrix
train_movies = train_df['movieId'].unique()
all_movies = pd.DataFrame(train_movies,columns=['movieId'])
all_movies['k'] = 0
all_users = pd.DataFrame(train_users,columns=['userId'])
all_users['k'] = 0
main_data = pd.merge(all_users,all_movies, on='k',how='outer').drop('k', axis=1)


# + id="e5pXeTIkUuSp"
main_data = pd.merge(main_data,train_df, on=['userId','movieId'],how='left')
main_data = main_data[main_data['rating'].isnull()]

# + colab={"base_uri": "https://localhost:8080/"} id="GB5xtPVNWKrP" outputId="8b68582a-0df1-48ea-f130-414f6783a8ed"
len(main_data)

# + id="TqNR12H4fox2"
a, b, c, d = np.split(main_data, 4)

# + id="LklEpgRmW8Wg"
a[['userId','movieId']].to_csv('/data/a.csv',index=False)
b[['userId','movieId']].to_csv('/data/b.csv',index=False)
c[['userId','movieId']].to_csv('/data/c.csv',index=False)
d[['userId','movieId']].to_csv('/data/d.csv',index=False)


# + id="JctHCh97vf07"
def calc_raw_uid(inner_id):
  return trainset.to_raw_uid(inner_id)


# + id="s5mjoUpVvle8"
def calc_cosine(dist):
  return 1 - (dist/2)


# + id="8HQLSOeUx-in"
def filter_nn_user(nn_sim, movie_id,movie_to_users): 
    if nn_sim[0] in movie_to_users[movie_id]:
      return True
    return False


# + id="7gqtEhdHW1WB"
def Ann_predictions_all(train, df, n):
  train_users = train['userId'].unique()
  train_df_pivoted = train.pivot(index='userId', columns='movieId', values='rating')

  avg_value = train_df_pivoted.mean(axis=1)
  user_mean_df = pd.DataFrame(avg_value,columns=['mean'])
  train_df_centered = train_df_pivoted.sub(train_df_pivoted.mean(axis=1), axis=0)

  movie_to_users = defaultdict(list)
  movie_to_users = train.groupby('movieId')['userId'].apply(list).to_dict()

  f = 15 # n_factors
  model_user_nn = AnnoyIndex(f, 'angular')  # Length of item vector that will be indexed
  for i,u in zip(user_inner_ids,U):
      model_user_nn.add_item(i, u)

  model_user_nn.build(n) # n trees 
  # model_user_nn.save('test_user.ann')
  def find_nn(user_id):
    return model_user_nn.get_nns_by_item(trainset.to_inner_uid(user_id), 101, include_distances=True)
  
  l = list(map(find_nn,train_users))

  matrix_similarity_with_nn = {}

  for user_id,nn_dist in zip(train_users,l):
    matrix_similarity_with_nn[user_id] = [(nn,sim) for nn,sim in zip(list(map(calc_raw_uid,nn_dist[0]))[1:],list(map(calc_cosine,nn_dist[1]))[1:])]


  def predict_nn_rating(user_movie_pair):
    user_id, movie_id = user_movie_pair
    reduced_nn_sim = pd.DataFrame(filter(lambda x: filter_nn_user(x, movie_id, movie_to_users),matrix_similarity_with_nn[user_id]))
    prediction = user_mean_df.loc[user_id].values[0]

    if len(reduced_nn_sim)==0:
      return prediction

    u_list = reduced_nn_sim[0].to_list()
    nr = np.dot(train_df_centered.loc[u_list][movie_id],reduced_nn_sim[1])
    dr = np.sum(np.abs(reduced_nn_sim[1]))
    if(dr!=0):
      prediction += (nr/dr)
    return prediction


  preds = list(map(predict_nn_rating,[(user_id,movie_id) for user_id,movie_id in zip(df['userId'],df['movieId'])]))
  #rmse = np.sqrt(mean_squared_error(test['rating'],preds))
  return preds

df = pd.read_csv('/data/b.csv')
file_name = 'b'

t = np.array_split(df, 10)

for i  in range(0,10):
    p=t[i]
    p['predictions']=Ann_predictions_all(train_df, p, 50)
    p.to_csv('/data/'+file_name+'_'+str(i)+'.csv')
    

# + id="TnU_NnMqcsHJ"
t = pd.DataFrame(columns=['userId','movieId','predictions'])
for i in range(0,10):
  df = pd.read_csv('/data/'+'b_'+str(i)+'.csv') 
  t = pd.concat([t,df])

for i in range(0,10):
  df = pd.read_csv('/data/'+'a_'+str(i)+'.csv') 
  t = pd.concat([t,df])


for i in range(0,10):
  df = pd.read_csv('/data/'+'c_'+str(i)+'.csv') 
  t = pd.concat([t,df])

for i in range(0,10):
  df = pd.read_csv('/data/'+'d_'+str(i)+'.csv') 
  t = pd.concat([t,df])

# + id="y9LKQnPRoy9P"
t.drop(columns= ['Unnamed: 0'],inplace=True)

# + id="Tg-DFvMFKQYW"
t.to_csv('/data/complete_predictions_ANN.csv',index=False)

# + colab={"base_uri": "https://localhost:8080/", "height": 415} id="5PxmiHK8oziY" outputId="46207552-51e0-4952-9a89-9ba08751e60e"
rating_preds = pd.merge(test_df,t,on=['userId','movieId'],how='inner')
rating_preds

# + colab={"base_uri": "https://localhost:8080/"} id="6JZW_FVZ-8e6" outputId="4b77917a-f155-4343-aead-044d8cfaf72b"
Rmse_Ann = np.sqrt(mean_squared_error(rating_preds['rating'],rating_preds['predictions']))
print("The RMSE obtained is",Rmse_Ann)

StarcoderdataJupyterScriptsDedupFiltered