# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# ## Movie Recommendation

# + pycharm={"is_executing": false}
import pandas as pd
import matplotlib.pyplot as plt
import numpy as np
from IPython.display import display

# + pycharm={"is_executing": false, "name": "#%%\n"}
movies = pd.read_csv("movies.csv", sep = ",", usecols=range(2))

ratings = pd.read_csv("ratings_train.csv", sep = ",", usecols=range(3))
ratings['rating'] = ratings['rating'].astype(float)
ratings['userId'] = ratings['userId'].astype(int)

# K-Fold Cross Validation
k_fold = 10
knn_k_values = [1, 5, 9, 13, 17]

# Shuffle the data for k-fold cross validation
train = ratings.sample(frac = 1)
number_of_rows = len(train.index)


# + pycharm={"is_executing": false, "name": "#%%   \n"}
# Boolean function to check whether a user has watched a movie or not

def has_watched(user, movie, user_movie):
    try:
        rank = user_movie.at[user,movie]
    except KeyError:
        return False
    if rank == 0: return False # because filled NaNs with zeros
    else: 
        return True


# + pycharm={"is_executing": false, "name": "#%%\n"}
def similarity_cosine(u, v):
    return np.dot(u, v) / (np.linalg.norm(u) * np.linalg.norm(v))


# + pycharm={"is_executing": false, "name": "#%%\n"}
# Find most similar k users to "user", who also watched "movie"

def also_watched_similar_users(user, movie, k, user_movie, rank_table):
    return_list = list()
    similar_users = rank_table.loc[user].values # Get similar users list of the user
    j = 0  # counter for k value
    for i in similar_users:
        if has_watched(i, movie, user_movie):  # If neighbor has also watched movie
            return_list.append(i)
            j = j + 1
            if j == k:
                break
    return return_list


# -

# Below, *get_mean_rating_from_user_list* function takes the user list which returned by previous *also_watched_similar_users*
# function. This function basically calculates the both weighted and unweighted error rate of each user on the list
# and finally calculates the mean error rate of both. 

# + pycharm={"is_executing": false, "name": "#%%\n"}
# Find mean rating of a movie from the users from user_list

def get_mean_rating_from_user_list(user_list, user, movie, user_movie, sorted_similars):
    means = list()
    
    if not user_list:   # If user_list is empty
        return -1
    
    total_rating = 0
    total_rating_weighted = 0
    total_similarity = 0
    
    for n in user_list:
        rate = user_movie.at[n, movie]
        total_rating += rate
        similarity = sorted_similars.at[user, n]
        total_similarity += similarity
        weighted_rate = rate * similarity
        total_rating_weighted += weighted_rate
    
    unweighted_mean = total_rating / len(user_list)
    weighted_mean = total_rating_weighted / total_similarity
    means.extend((unweighted_mean, weighted_mean))
    return means


# -

# Below, *validation_test* function calls the other functions and gets the returned predicted rating values. Afterwards, calculates
# the difference between the real rating and predicted rating for both weighted and unweighted KNN and returns these values.

# + pycharm={"is_executing": false, "name": "#%%\n"}
def validation_test(sorted_similarity, test_data, k, user_movie, rank_table):
    errors = list()
    
    ratings_to_check = test_data["rating"]
    test_data_without_rating = test_data.drop(columns = ["rating"])
    without_rating = test_data_without_rating.to_numpy()
    ratings_to_check.reset_index(drop = True, inplace = True)
    
    total_difference_weighted = 0
    total_difference_unweighted = 0
    
    for m in range(len(without_rating)):
        user, movie, rating = without_rating[m][0], without_rating[m][1], ratings_to_check[m]
        predicted_values = get_mean_rating_from_user_list(
            also_watched_similar_users(user, movie, k, user_movie, rank_table), user, movie, user_movie, sorted_similarity)
        if predicted_values == -1 : continue     # if nobody watched the movie in training dataset
        difference_unweighted = abs(rating - predicted_values[0])
        total_difference_unweighted += difference_unweighted       
        difference_weighted = abs(rating - predicted_values[1])
        total_difference_weighted += difference_weighted
    
    error_unweighted = total_difference_unweighted / len(without_rating)
    error_weighted = total_difference_weighted / len(without_rating)
    errors.extend((error_unweighted, error_weighted))
    
    return errors


# -

# Below for loop is the main operator of the program. This loop splits the data to training and test parts according to
# K-Fold method. After that it trains the data and creates corresponding numpy arrays and pandas data frames. After getting the
# returned mean error rates from *validation_test* function, for each K-fold, calculates the mean error rates for kNN's k values. 
#
# **Tables that are created in training process can be seen in outputs.**

# + pycharm={"is_executing": false, "name": "#%%\n"}
# list to plot K with matplotlib
y1 = list()
y2 = list()
count = 0

for K in knn_k_values:
    total_error_unweighted = 0
    total_error_weighted = 0
    for i in range(k_fold):        
        if i == 0:
            test_of_validation = train.iloc[(number_of_rows // k_fold) * i : (number_of_rows // k_fold) * (i + 1)]
            train_of_validation = train.iloc[(number_of_rows // k_fold) * (i + 1) : (number_of_rows // k_fold) * k_fold] 
        else:
            train_of_validation_first = train.iloc[ : (number_of_rows // k_fold) * i]
            test_of_validation = train.iloc[(number_of_rows // k_fold) * i : (number_of_rows // k_fold) * (i + 1)] 
            train_of_validation_second = train.iloc[(number_of_rows // k_fold) * (i + 1) : (number_of_rows // k_fold) * k_fold]        
            
            # Concatenate the two training parts of validation set
            train_of_validation = pd.concat([train_of_validation_first, train_of_validation_second])
            
        test_of_validation = test_of_validation.drop(test_of_validation[test_of_validation.userId == 53].index)
            
        user_movie_matrix = pd.pivot_table(train_of_validation, values = "rating", index = "userId", columns = "movieId")
        if count == 0 : display(user_movie_matrix.head()) 
           
        # Find each user's mean rating among all movies
        Mean = train_of_validation.groupby(by = "userId",as_index = False)['rating'].mean()
        Mean.head()
        rating_avg = pd.merge(train_of_validation,Mean,on = 'userId')
        rating_avg.rename(columns = {'rating_y':'user_mean', 'rating_x':'rating'}, inplace = True)
        
        # Normalize all ratings according to zero
        rating_avg['normal_rating'] = rating_avg['rating'] - rating_avg['user_mean']
        if count == 0 : display(rating_avg.head()) 
        
        # Fill NaN values with zero 
        user_movie_normal = pd.pivot_table(rating_avg,values = 'normal_rating',index = 'userId',columns = 'movieId')
        user_movie_normal.fillna(0, inplace = True)
        user_movie_normal = user_movie_normal.drop([53])
        user_movie_matrix.fillna(0, inplace = True)
        final_numpy = user_movie_normal.to_numpy()
        if count == 0 : display(user_movie_normal.head()) 
        
        # Calculate cosine similarity between each user and store in user_user_similarity data frame 
        np.seterr(invalid='ignore')
        total = 0
        user_user = np.ones((len(final_numpy), len(final_numpy)))
        
        for i in range(len(final_numpy)):
            for j in range(len(final_numpy)):
                user_user[i][j] = similarity_cosine(final_numpy[i],final_numpy[j]) + 1
            
        # Dataset contains the similarity between each user    
        user_user_similarity = pd.DataFrame(data = user_user,index = user_movie_normal.index, columns = user_movie_normal.index)
        if count == 0 : display(user_user_similarity.head())
        
        # Create new data frame with sorted user-user similarities   
        a = user_user_similarity.values
        a = np.nan_to_num(a)
        c = np.copy(a)
        a.sort(axis=1) 
        a = a[:, ::-1]  # ascending
        user_user_sorted_similarity = pd.DataFrame(a, user_user_similarity.index, user_user_similarity.columns)
        user_user_similarity = pd.DataFrame(c, user_movie_normal.index, user_movie_normal.index)
        user_rank_table = user_user_similarity.apply(
            lambda x: pd.Series(x.sort_values(ascending = False).iloc[1:len(user_user_similarity)].index, index = ['top{}'.format(i) for i in range(2, len(user_user_similarity)+1)]), axis = 1)
        
        if count == 0 : display(user_rank_table.head())
        error_rates = validation_test(user_user_sorted_similarity, test_of_validation, K, user_movie_matrix, user_rank_table)
        error_rate_unweighted = error_rates[0]
        total_error_unweighted += error_rate_unweighted
        error_rate_weighted = error_rates[1]
        total_error_weighted += error_rate_weighted       
        count += 1
    
    print("k Value: " + str(K) + " Unweighted Mean Error Rate: " + str(total_error_unweighted / k_fold) + " Weighted Mean Error Rate: " + str(total_error_weighted / k_fold))
    y1.append(total_error_unweighted / k_fold)
    y2.append(total_error_weighted / k_fold)
    
   

# + pycharm={"is_executing": false, "name": "#%%\n"}
# Plot with matplotlib

line1 = plt.plot(knn_k_values, y1, label='Unweighted')
line2 = plt.plot(knn_k_values, y2, label='Weighted')
plt.xlabel('k Value')
plt.ylabel('Mean Error Rates')
plt.legend()


# + [markdown] pycharm={"name": "#%% md\n"}
# As it seen from the graph, when k is very small(<4), we have a very high error rate according to other k values. When we choose a k value bigger than 5,
# error rate is very acceptable. As k value increases error rate is going down. However, because of the memory and time consumption, bigger k values are
# not ideal. For this dataset example, a k value between 6-10 looks as best choice. 
#
# On the other hand, because we normalized all rating values between 0 - 2, user similarities are very close to each other. As a result of that, weighted k-NN
# gives us very similar results to unweighted k-NN. 
#

StarcoderdataJupyterScriptsDedupFiltered