"""
 Import packages and Data
"""
from surprise import Reader, Dataset, SVD, evaluate
import pandas as pd
import numpy as np
from sklearn.model_selection import train_test_split

r_cols = ['user_id', 'movie_id', 'rating', 'unix_timestamp']
ratings = pd.read_csv('../input/ml-100k/u.data', sep='\t', names=r_cols, index_col='movie_id', encoding='latin-1')

m_cols = ['movie_id', 'title', 'release_date', 'video_release_date', 'imdb_url','unknown', 'Action', 'Adventure',\
          'Animation', 'Children\'s', 'Comedy', 'Crime', 'Documentary', 'Drama', 'Fantasy','Film-Noir', 'Horror',\
          'Musical', 'Mystery', 'Romance', 'Sci-Fi', 'Thriller', 'War', 'Western']
movies = pd.read_csv('../input/ml-100k/u.item', sep='|', names=m_cols, index_col=0, encoding='latin-1')

u_cols = ['user_id', 'age', 'sex', 'occupation', 'zip_code']
users = pd.read_csv('../input/ml-100k/u.user', sep='|', names=u_cols, encoding='latin-1', parse_dates=True)
from datetime import datetime

ratings['unix_timestamp'] = ratings['unix_timestamp'].apply(datetime.fromtimestamp)
ratings.columns = ['user_id', 'rating', 'time']
ratings.head(10)
"""
Here we can see how ratings distributed.
"""
ratings['rating'].hist(bins=9)
"""
So far we will only use the movie title from this DataFrame. We may need the types of the movie later in our model.
"""
movies['release_date'] = pd.to_datetime(movies['release_date'])
movies.head(10)
for i in users['occupation'].unique():
    users[i] = users['occupation'] == i
users.drop('occupation', axis=1, inplace=True)
users.head(10)
"""
For each movie we count how many ratings it got, and what's the mean and standard deviation.
"""
ratings_movie_summary = ratings.groupby('movie_id')['rating'].agg(['count', 'mean', 'std'])
ratings_movie_summary.head(10)
"""
For each user, we count how many ratings he or she gives, and the mean and standard deviation as well.
"""
ratings_user_summary = ratings.groupby('user_id')['rating'].agg(['count', 'mean', 'std'])
ratings_user_summary.head(10)
ratings_movie_summary.sort_values(by='count')['count'].hist(bins=20)
ratings_movie_summary.sort_values(by='mean')['mean'].hist(bins=20)
ratings_user_summary.sort_values(by='count')['count'].hist(bins=20)
ratings_user_summary.sort_values(by='mean')['mean'].hist(bins=20)
"""
We create a pivot table for ratings and store the total mean and standard deviation values.
"""
ratings_p = pd.pivot_table(ratings, values='rating', index='user_id', columns='movie_id')
ratings_p.iloc[:10, :10]
mean = ratings_p.stack().mean()
std = ratings_p.stack().std()
"""
**Notations:**

$\mu_i$ : The mean of all ratings received by movie i.

$\mu_u$ : The mean of all ratings from user u.

$\mu$ : The mean of all ratings.

$\sigma_i$ : The standard deviation of all ratings received by movie i.

$\sigma_u$ : The standard deviation of all ratings from user u.

$r_{ui}$ : User u's rating on movie i.

$\hat{r}_{ui}$ : Prediction about user u's rating on movie i.

$N^k_u(i)$ : k nearest neighbors of movie i, that are rated by user u.
"""
"""
- **Baseline Model**

In the first baseline model, we predict the rating from a specific user on a specific movie, just by the average rating that a movie receives, with adjustment by how this user's average rating compared with the total average.

$$\hat{r}_{ui} = \mu_u + \mu_i - \mu$$
"""
movie_mean = np.ones(ratings_p.shape)
movie_mean = pd.DataFrame(movie_mean * np.array(ratings_movie_summary['mean']).reshape(1,1682))
user_mean = np.ones(ratings_p.T.shape)
user_mean = pd.DataFrame(user_mean * np.array(ratings_user_summary['mean'])).T
pred = movie_mean + user_mean - mean
score = abs(np.array(ratings_p) - pred)
score_2 = score ** 2
print('RMSE: {:.4f}'.format(np.sqrt(score_2.stack().mean())))
print('MAE: {:.4f}'.format(score.stack().mean()))
"""
Since we don't have train-test-split in our prediction, we are actually using the mean of the data to predict every single data. Therefore, the score might be biased because of data leakage. So we do cross-validation on the model.
"""
from sklearn.model_selection import KFold

kfolds = KFold(n_splits = 5, random_state = 13)
rmse = []
mae = []
i = 0
print('Evaluating RMSE, MAE of the Baseline Model. \n')
print('-'*12)
for train_index, test_index in kfolds.split(ratings):
    train = ratings.copy()
    test = ratings.copy()
    train['rating'].iloc[test_index] = np.NaN
    test['rating'].iloc[train_index] = np.NaN
    train_movie_summary = train.groupby('movie_id')['rating'].agg(['count', 'mean', 'std'])
    train_user_summary = train.groupby('user_id')['rating'].agg(['count', 'mean', 'std'])
    test_p = pd.pivot_table(test, values='rating', index='user_id', columns='movie_id', dropna=False)
    movie_mean = np.ones(ratings_p.shape)
    movie_mean = pd.DataFrame(movie_mean * np.array(train_movie_summary['mean']).reshape(1,1682))
    user_mean = np.ones(ratings_p.T.shape)
    user_mean = pd.DataFrame(user_mean * np.array(train_user_summary['mean'])).T
    train_p = movie_mean + user_mean - mean
    score = abs(np.array(test_p) - train_p)
    score_2 = score ** 2
    rmse += [np.sqrt(score_2.stack().mean())]
    mae += [score.stack().mean()]
    i += 1
    print('Fold', i)
    print('RMSE: {:.4f}'.format(np.sqrt(score_2.stack().mean())))
    print('MAE: {:.4f}'.format(score.stack().mean()))
    print('-'*12)
print('-'*12)
print('Mean RMSE: {:.4f}'.format(np.mean(rmse)))
print('Mean MAE: {:.4f}'.format(np.mean(mae)))
print('-'*12)
print('-'*12)
"""
- **Baseline_Plus Model**

In the second model, we want to do it slightly better using z-score.

$$\hat{r}_{ui} = \mu_u + \sigma_u * \frac{(\mu_i - \mu)}{\sigma}$$
"""
movie_mean = np.ones(ratings_p.shape)
movie_mean = pd.DataFrame(movie_mean * np.array(ratings_movie_summary['mean']).reshape(1,1682))
user_mean = np.ones(ratings_p.T.shape)
user_mean = pd.DataFrame(user_mean * np.array(ratings_user_summary['mean'])).T
user_std = np.ones(ratings_p.T.shape)
user_std = pd.DataFrame(user_std * np.array(ratings_user_summary['std'])).T
pred_plus = user_mean + (movie_mean - mean)/std * user_std
score_plus = abs(np.array(ratings_p) - pred_plus)
score_2_plus = score_plus ** 2
print('RMSE: {:.4f}'.format(np.sqrt(score_2_plus.stack().mean())))
print('MAE: {:.4f}'.format(score_plus.stack().mean()))
"""
Use our baseline model to recommend movies for user 196.
"""
user_196 = movies[['title', 'release_date']]
user_196['Estimate_Score'] = np.array(pred_plus.loc[195])
user_196 = user_196.sort_values('Estimate_Score', ascending=False)
print(user_196.head(10))
"""
Here is the cross-validation score for our second model
"""
rmse_plus = []
mae_plus = []
i = 0
print('Evaluating RMSE, MAE of the Baseline_Plus Model. \n')
print('-'*12)
for train_index, test_index in kfolds.split(ratings):
    train = ratings.copy()
    test = ratings.copy()
    train['rating'].iloc[test_index] = np.NaN
    test['rating'].iloc[train_index] = np.NaN
    train_movie_summary = train.groupby('movie_id')['rating'].agg(['count', 'mean', 'std'])
    train_user_summary = train.groupby('user_id')['rating'].agg(['count', 'mean', 'std'])
    test_p = pd.pivot_table(test, values='rating', index='user_id', columns='movie_id', dropna=False)
    movie_mean = np.ones(ratings_p.shape)
    movie_mean = pd.DataFrame(movie_mean * np.array(train_movie_summary['mean']).reshape(1,1682))
    user_mean = np.ones(ratings_p.T.shape)
    user_mean = pd.DataFrame(user_mean * np.array(train_user_summary['mean'])).T
    user_std = np.ones(ratings_p.T.shape)
    user_std = pd.DataFrame(user_std * np.array(train_user_summary['std'])).T
    train_p = user_mean + (movie_mean - mean)/std * user_std
    score = abs(np.array(test_p) - train_p)
    score_2 = score ** 2
    rmse_plus += [np.sqrt(score_2.stack().mean())]
    mae_plus += [score.stack().mean()]
    i += 1
    print('Fold', i)
    print('RMSE: {:.4f}'.format(np.sqrt(score_2.stack().mean())))
    print('MAE: {:.4f}'.format(score.stack().mean()))
    print('-'*12)
print('-'*12)
print('Mean RMSE: {:.4f}'.format(np.mean(rmse_plus)))
print('Mean MAE: {:.4f}'.format(np.mean(mae_plus)))
print('-'*12)
print('-'*12)
"""
- **Baseline Model with SVM/Gradient Boosting**

We can improve this model even more, by applying SVM regressor or Gradient Boosting on each approximation, instead of just using z-score.
"""
"""
- SVM
"""
from sklearn.svm import SVR

movie_mean = np.ones(ratings_p.shape)
movie_mean = pd.DataFrame(movie_mean * np.array(ratings_movie_summary['mean']).reshape(1,1682))
X = np.array(ratings_p*0) + movie_mean
svm = SVR(gamma=1, C=1)
pred_svm = ratings_p.copy()
for i in range(ratings_p.shape[0]):
    svm.fit(np.array(X.iloc[i].dropna()).reshape(-1,1), ratings_p.iloc[i].dropna())
    pred_svm.iloc[i] = svm.predict(np.array(movie_mean.iloc[0]).reshape(-1,1))
score_svm = abs(np.array(ratings_p) - pred_svm)
score_2_svm = score_svm ** 2
print('RMSE: {:.4f}'.format(np.sqrt(score_2_svm.stack().mean())))
print('MAE: {:.4f}'.format(score_svm.stack().mean()))
"""
Use our svm model to recommend movies for user 196.
"""
user_196_svm = movies[['title', 'release_date']]
user_196_svm['Estimate_Score'] = np.array(pred_svm.loc[195])
user_196_svm = user_196_svm.sort_values('Estimate_Score', ascending=False)
print(user_196_svm.head(10))
"""
Cross-Validation
"""
rmse_svm = []
mae_svm = []
fold = 0
movie_mean = pd.DataFrame(np.ones(ratings_p.shape) * np.array(ratings_movie_summary['mean']).reshape(1,1682))
print('Evaluating RMSE, MAE of the Baseline_SVM Model. \n')
print('-'*12)
for train_index, test_index in kfolds.split(ratings):
    train = ratings.copy()
    test = ratings.copy()
    train['rating'].iloc[test_index] = np.NaN
    test['rating'].iloc[train_index] = np.NaN
    train_movie_summary = train.groupby('movie_id')['rating'].agg(['count', 'mean', 'std'])
    train_user_summary = train.groupby('user_id')['rating'].agg(['count', 'mean', 'std'])
    train_p = pd.pivot_table(train, values='rating', index='user_id', columns='movie_id', dropna=False)
    test_p = pd.pivot_table(test, values='rating', index='user_id', columns='movie_id', dropna=False)
    train_mean = pd.DataFrame(np.ones(ratings_p.shape) * np.array(train_movie_summary['mean']).reshape(1,1682))
    X = np.array(train_p*0) + train_mean
    pred = ratings_p.copy()
    for i in range(ratings_p.shape[0]):
        svm.fit(np.array(X.iloc[i].dropna()).reshape(-1,1), train_p.iloc[i].dropna())
        pred.iloc[i] = svm.predict(np.array(movie_mean.iloc[0]).reshape(-1,1))
    score = abs(np.array(test_p) - pred)
    score_2 = score ** 2
    rmse_svm += [np.sqrt(score_2.stack().mean())]
    mae_svm += [score.stack().mean()]
    fold += 1
    print('Fold', fold)
    print('RMSE: {:.4f}'.format(np.sqrt(score_2.stack().mean())))
    print('MAE: {:.4f}'.format(score.stack().mean()))
    print('-'*12)
print('-'*12)
print('Mean RMSE: {:.4f}'.format(np.mean(rmse_svm)))
print('Mean MAE: {:.4f}'.format(np.mean(mae_svm)))
print('-'*12)
print('-'*12)
"""
Gradient Boosting
"""
from xgboost import XGBRegressor

movie_mean = np.ones(ratings_p.shape)
movie_mean = pd.DataFrame(movie_mean * np.array(ratings_movie_summary['mean']).reshape(1,1682))
X = np.array(ratings_p*0) + movie_mean
xgb = XGBRegressor(learning_rate=0.1, max_depth=2, min_child_weight=10, gamma=1)
pred_xgb = ratings_p.copy()
for i in range(ratings_p.shape[0]):
    xgb.fit(np.array(X.iloc[i].dropna()).reshape(-1,1), ratings_p.iloc[i].dropna())
    pred_xgb.iloc[i] = xgb.predict(np.array(movie_mean.iloc[0]).reshape(-1,1))
score_xgb = abs(np.array(ratings_p) - pred_xgb)
score_2_xgb = score_xgb ** 2
print('RMSE: {:.4f}'.format(np.sqrt(score_2_xgb.stack().mean())))
print('MAE: {:.4f}'.format(score_xgb.stack().mean()))
"""
Cross-Validation
"""
rmse_xgb = []
mae_xgb = []
fold = 0
movie_mean = pd.DataFrame(np.ones(ratings_p.shape) * np.array(ratings_movie_summary['mean']).reshape(1,1682))
print('Evaluating RMSE, MAE of the Baseline_XGB Model. \n')
print('-'*12)
for train_index, test_index in kfolds.split(ratings):
    train = ratings.copy()
    test = ratings.copy()
    train['rating'].iloc[test_index] = np.NaN
    test['rating'].iloc[train_index] = np.NaN
    train_movie_summary = train.groupby('movie_id')['rating'].agg(['count', 'mean', 'std'])
    train_user_summary = train.groupby('user_id')['rating'].agg(['count', 'mean', 'std'])
    train_p = pd.pivot_table(train, values='rating', index='user_id', columns='movie_id', dropna=False)
    test_p = pd.pivot_table(test, values='rating', index='user_id', columns='movie_id', dropna=False)
    train_mean = pd.DataFrame(np.ones(ratings_p.shape) * np.array(train_movie_summary['mean']).reshape(1,1682))
    X = np.array(train_p*0) + train_mean
    pred = ratings_p.copy()
    for i in range(ratings_p.shape[0]):
        xgb.fit(np.array(X.iloc[i].dropna()).reshape(-1,1), train_p.iloc[i].dropna())
        pred.iloc[i] = xgb.predict(np.array(movie_mean.iloc[0]).reshape(-1,1))
    score = abs(np.array(test_p) - pred)
    score_2 = score ** 2
    rmse_xgb += [np.sqrt(score_2.stack().mean())]
    mae_xgb += [score.stack().mean()]
    fold += 1
    print('Fold', fold)
    print('RMSE: {:.4f}'.format(np.sqrt(score_2.stack().mean())))
    print('MAE: {:.4f}'.format(score.stack().mean()))
    print('-'*12)
print('-'*12)
print('Mean RMSE: {:.4f}'.format(np.mean(rmse_xgb)))
print('Mean MAE: {:.4f}'.format(np.mean(mae_xgb)))
print('-'*12)
print('-'*12)
"""
- **Pearsons'R Correlation Model**

We might also want to recommend movies just for a specific movie. Like the recommendation list showed on the webpage of a specific movie.

Here we recommend new movies based on the Pearsons'R correlation between movies.
"""
def recommend(movie_title, min_count):
    print("For movie ({})".format(movie_title))
    print("- Top 10 movies recommended based on Pearsons'R correlation - ")
    i = movies[movies['title'] == movie_title].index[0]
    target = ratings_p[i]
    similar_to_target = ratings_p.corrwith(target)
    corr_target = pd.DataFrame(similar_to_target, columns = ['PearsonR'])
    corr_target.dropna(inplace = True)
    corr_target = corr_target.sort_values('PearsonR', ascending = False)
    corr_target.index = corr_target.index.map(int)
    corr_target = corr_target.join(movies).join(ratings_movie_summary)\
                  [['PearsonR', 'title', 'count', 'mean']]
    print (corr_target[corr_target['count']>min_count][:10].to_string(index=False))
recommend('Shawshank Redemption, The (1994)', 10)
"""
- **K-Nearest Neighbor (kNN) Model**

We can treat the **Pearsons' R Correlation** between movies as the distance, and using these distances to build a **K-Nearest Neighbor model**.

Notations:

$r_{ui}$ : User u's rating on movie i.

$\hat{r}_{ui}$ : Prediction about user u's rating on movie i.

$N^k_u(i)$ : k nearest neighbors of movie i, that are rated by user u.

Then we can have our kNN model as:

$$\hat{r}_{ui} = \frac{\sum_{j \in N^k_u(i)} corr(i, j) * r_{uj}}{\sum_{j \in N^k_u(i)} corr(i, j)}$$
"""
sim = ratings_p.corr().abs()
sim.iloc[:10, :10]
knn_pred = ratings_p.copy()
for i in ratings_p.index:
    if i % 10 == 0:
        print(i)
    N = sim.loc[ratings[ratings['user_id'] == i].index]
    for j in ratings_p.columns:
        try:
            N_k = N[j].sort_values(ascending=False, kind='heapsort').drop(j)[:30]
        except:
            N_k = N[j].sort_values(ascending=False, kind='heapsort')[:30]
        weighted_rating = N_k*ratings_p.loc[i, N_k.index]
        knn_pred.loc[i, j] = weighted_rating.sum()/N_k.sum()

knn_pred.iloc[:10, :10]
score_knn = abs(np.array(ratings_p) - knn_pred)
score_2_knn = score_knn ** 2
print('RMSE: {:.4f}'.format(np.sqrt(score_2_knn.stack().mean())))
print('MAE: {:.4f}'.format(score_knn.stack().mean()))
"""
Cross-Validation on kNN
"""
rmse_knn = []
mae_knn = []
fold = 0
print('Evaluating RMSE, MAE of the Baseline_XGB Model. \n')
print('-'*12)
for train_index, test_index in kfolds.split(ratings):
    train = ratings.copy()
    test = ratings.copy()
    train['rating'].iloc[test_index] = np.NaN
    test['rating'].iloc[train_index] = np.NaN
    train_movie_summary = train.groupby('movie_id')['rating'].agg(['count', 'mean', 'std'])
    train_user_summary = train.groupby('user_id')['rating'].agg(['count', 'mean', 'std'])
    train_p = pd.pivot_table(train, values='rating', index='user_id', columns='movie_id', dropna=False)
    test_p = pd.pivot_table(test, values='rating', index='user_id', columns='movie_id', dropna=False)
    knn_pred = ratings_p.copy()
    for i in ratings_p.index:
        N = sim.loc[train[train['user_id'] == i].index]
        for j in ratings_p.columns:
            try:
                N_k = N[j].sort_values(ascending=False, kind='heapsort').drop(j)[:30]
            except:
                N_k = N[j].sort_values(ascending=False, kind='heapsort')[:30]
            weighted_rating = N_k*train_p.loc[i, N_k.index]
            knn_pred.loc[i, j] = weighted_rating.sum()/N_k.sum()
    score = abs(np.array(test_p) - knn_pred)
    score_2 = score ** 2
    rmse_knn += [np.sqrt(score_2.stack().mean())]
    mae_knn += [score.stack().mean()]
    fold += 1
    print('Fold', fold)
    print('RMSE: {:.4f}'.format(np.sqrt(score_2.stack().mean())))
    print('MAE: {:.4f}'.format(score.stack().mean()))
    print('-'*12)
print('-'*12)
print('Mean RMSE: {:.4f}'.format(np.mean(rmse_knn)))
print('Mean MAE: {:.4f}'.format(np.mean(mae_knn)))
print('-'*12)
print('-'*12)
"""
- **kNN_Plus Model**

Now we can improve our KNN model just by the same trick we used on our baseline model: adjust by the z-score.

Notation:

$\mu_i$ : The mean of all ratings received by movie i.

$\sigma_i$ : The standard deviation of all ratings received by movie i.

$$\hat{r}_{ui} = \mu_i + \sigma_i * \frac{\sum_{j \in N^k_u(i)} corr(i, j) *( r_{uj} - \mu_j) / \sigma_j}{\sum_{j \in N^k_u(i)} corr(i, j)}$$
"""
knn_plus_pred = ratings_p.copy()
for i in ratings_p.index:
    N = sim.loc[ratings[ratings['user_id'] == i].index]
    for j in ratings_p.columns:
        try:
            N_k = N[j].sort_values(ascending=False, kind='heapsort').drop(j)[:30]
        except:
            N_k = N[j].sort_values(ascending=False, kind='heapsort')[:30]
        weighted_rating = N_k*(ratings_p.loc[i, N_k.index] - ratings_movie_summary.loc[N_k.index, 'mean'])/ ratings_movie_summary.loc[N_k.index, 'std']
        knn_plus_pred.loc[i, j] = weighted_rating.sum()/N_k.sum() * ratings_movie_summary.loc[j, 'std'] + ratings_movie_summary.loc[j, 'mean']

knn_plus_pred.iloc[:10, :10]
score_knn_plus = abs(np.array(ratings_p) - knn_plus_pred)
score_2_knn_plus = score_knn ** 2
print('RMSE: {:.4f}'.format(np.sqrt(score_2_knn_plus.stack().mean())))
print('MAE: {:.4f}'.format(score_knn_plus.stack().mean()))
"""
Cross-Validation on kNN_Plus
"""
rmse_knn_plus = []
mae_knn_plus = []
fold = 0
print('Evaluating RMSE, MAE of the Baseline_XGB Model. \n')
print('-'*12)
for train_index, test_index in kfolds.split(ratings):
    train = ratings.copy()
    test = ratings.copy()
    train['rating'].iloc[test_index] = np.NaN
    test['rating'].iloc[train_index] = np.NaN
    train_movie_summary = train.groupby('movie_id')['rating'].agg(['count', 'mean', 'std'])
    train_user_summary = train.groupby('user_id')['rating'].agg(['count', 'mean', 'std'])
    train_p = pd.pivot_table(train, values='rating', index='user_id', columns='movie_id', dropna=False)
    test_p = pd.pivot_table(test, values='rating', index='user_id', columns='movie_id', dropna=False)
    knn_plus_pred = ratings_p.copy()
    for i in ratings_p.index:
        if i % 100 == 0:
            print(i)
        N = sim.loc[train[train['user_id'] == i].index]
        for j in ratings_p.columns:
            try:
                N_k = N[j].sort_values(ascending=False).drop(j)[:30]
            except:
                N_k = N[j].sort_values(ascending=False)[:30]
            weighted_rating = N_k*(train_p.loc[i, N_k.index] - train_movie_summary.loc[N_k.index, 'mean'])/ train_movie_summary.loc[N_k.index, 'std']
            knn_plus_pred.loc[i, j] = weighted_rating.sum()/N_k.sum() * train_movie_summary.loc[j, 'std'] + train_movie_summary.loc[j, 'mean']
    score = abs(np.array(test_p) - knn_plus_pred)
    score_2 = score ** 2
    rmse_knn_plus += [np.sqrt(score_2.stack().mean())]
    mae_knn_plus += [score.stack().mean()]
    fold += 1
    print('Fold', fold)
    print('RMSE: {:.4f}'.format(np.sqrt(score_2.stack().mean())))
    print('MAE: {:.4f}'.format(score.stack().mean()))
    print('-'*12)
print('-'*12)
print('Mean RMSE: {:.4f}'.format(np.mean(rmse_knn_plus)))
print('Mean MAE: {:.4f}'.format(np.mean(mae_knn_plus)))
print('-'*12)
print('-'*12)
"""
- **SVD**

According to Matrix Factorization Techniques for Recommender Systems, written by Yehoda Koren, Robert Bell, and Chris Volinsky, Our prediction looks like this:

$$\hat{r}_{ui} = \mu + b_u + b_i + q_i^Tp_u$$

Where $q_i$ and $p_u$ are the corresponding factor vectors for movie $i$ and user $u$. So $q_i ^ T p_u$ term is the interaction factor between movie $i$ and user $u$.

And $\mu + b_u + b_i$ is the bias term. $b_u$ and $b_i$ represents the factor that depends solely on user $u$ or on movie $i$.

Hence, we want to minimize the following:

$$\min_{p^*, q^*, b^*} \sum_{(u, i) \in R} (r_{ui} - \hat{r}_{ui})^2 + \lambda (\| p_u \|^2 + \| q_i \|^2 + b_u ^2 + b_i ^2)$$

And we achive this by doing stochastic gradient descent on $p_u, q_i, b_u, $ and $b_i$. Namely:

$$b_u  \leftarrow b_u + \gamma (e_{ui} - \lambda b_u)$$
$$b_i  \leftarrow b_i + \gamma (e_{ui} - \lambda b_i)$$
$$p_u  \leftarrow p_u + \gamma (e_{ui} \cdot q_i - \lambda p_u)$$
$$q_i  \leftarrow q_i + \gamma (e_{ui} \cdot p_u - \lambda q_i)$$

Where $\gamma$ is the learning rate, and $\lambda$ is the regularization, and we want to run this gradient descent by $n$ times. (So far we set $\gamma = 0.005$, $\lambda = 0.1$, and $n=100$. And we want number of factors as $k=70$.)
"""
from datetime import datetime

r_cols = ['user_id', 'movie_id', 'rating', 'unix_timestamp']
ratings = pd.read_csv('../input/ml-100k/u.data', sep='\t', names=r_cols, encoding='latin-1')
ratings['unix_timestamp'] = ratings['unix_timestamp'].apply(datetime.fromtimestamp)
ratings.columns = ['user_id', 'movie_id', 'rating', 'time']
ratings.head(10)
lr = 0.005 # Learning Rate(Gamma)
reg = 0.1 # regularization(Lambda)
n = 100 # Number of iterations
k = 70 # Number of factors
u = ratings_p.shape[0] # Number of users
i = ratings_p.shape[1] # Number of movies

bu = np.zeros(u) # A series of bias constant, one for each user
bi = np.zeros(i) # A series of bias constant, one for each movie
pu = np.random.mtrand._rand.normal(0, 0.1, (u, k)) # A series of vectors(rows) with length k, one row for each user
qi = np.random.mtrand._rand.normal(0, 0.1, (i, k)) # A series of vectors(rows) with length k, one row for each movie
ratings_array = np.array(ratings.drop('time', axis=1))
for i in range(n): # For each iteration
    for row in ratings_array: # For each instance
        user_id = row[0]-1
        movie_id = row[1]-1
        rating = row[2]
        mult = 0
        for j in range(k): # For each factor
            mult += qi[movie_id, j] * pu[user_id, j] # Compute the dot product(interaction q_i^Tp_u)
        error = rating - (mean + bu[user_id] + bi[movie_id] + mult) # error = rating - (prediction)
        bu[user_id] += lr * (error - reg * bu[user_id]) # Update Bias on user
        bi[movie_id] += lr * (error - reg * bi[movie_id]) # Update Bias on movie
        for k in range(k): # Again for each factor
            pu_k = pu[user_id, k]
            qi_k = qi[movie_id, k]
            pu[user_id, k] += 0.005 * (error * qi_k - 0.1 * pu_k) # Update this factor using the final error
            qi[movie_id, k] += 0.005 * (error * pu_k - 0.1 * qi_k) # Update this factor
U = ratings['user_id'].unique()
I = ratings['movie_id'].unique()
def svd_pred(user_id, movie_id):
    pred = mean
    if user_id in U:
        pred += bu[user_id]
    if movie_id in I:
        pred += bi[movie_id]
    if (user_id in U) and (movie_id in I):
        pred += np.dot(qi[movie_id], pu[user_id])
    return pred

svd_pred_ratings = np.array(ratings)
for row in svd_pred_ratings:
    row[3] = svd_pred(row[0]-1, row[1]-1)
svd_pred_ratings = pd.DataFrame(svd_pred_ratings, columns=['user_id', 'movie_id', 'rating', 'pred'])

print('RMSE: {:.4f}'.format(np.sqrt(((svd_pred_ratings['pred'] - svd_pred_ratings['rating']) ** 2).mean())))
print('MAE: {:.4f}'.format(abs(svd_pred_ratings['pred'] - svd_pred_ratings['rating']).mean()))
"""
Cross-Validation for SVD
"""
rmse_svd = []
mae_svd = []
fold = 0
print('Evaluating RMSE, MAE of the SVD Model. \n')
print('-'*12)
for train_index, test_index in kfolds.split(ratings):
    
    u = ratings_p.shape[0] # Number of users
    i = ratings_p.shape[1] # Number of movies
    bu = np.zeros(u) # A series of bias constant, one for each user
    bi = np.zeros(i) # A series of bias constant, one for each movie
    pu = np.random.mtrand._rand.normal(0, 0.1, (u, k)) # A series of vectors(rows) with length k, one row for each user
    qi = np.random.mtrand._rand.normal(0, 0.1, (i, k)) # A series of vectors(rows) with length k, one row for each movie
    
    ratings_array = ratings.iloc[train_index]
    ratings_array = np.array(ratings_array.drop('time', axis=1))
    for i in range(n): # For each iteration
        for row in ratings_array: # For each instance
            user_id = row[0]-1
            movie_id = row[1]-1
            rating = row[2]
            mult = 0
            for j in range(k): # For each factor
                mult += qi[movie_id, j] * pu[user_id, j] # Compute the dot product(interaction q_i^Tp_u)
            error = rating - (mean + bu[user_id] + bi[movie_id] + mult) # error = rating - (prediction)
            bu[user_id] += lr * (error - reg * bu[user_id]) # Update Bias on user
            bi[movie_id] += lr * (error - reg * bi[movie_id]) # Update Bias on movie
            for k in range(k): # Again for each factor
                pu_k = pu[user_id, k]
                qi_k = qi[movie_id, k]
                pu[user_id, k] += 0.005 * (error * qi_k - 0.1 * pu_k) # Update this factor using the final error
                qi[movie_id, k] += 0.005 * (error * pu_k - 0.1 * qi_k) # Update this factor
    
    svd_pred_ratings = np.array(ratings.iloc[test_index])
    for row in svd_pred_ratings:
        row[3] = svd_pred(row[0]-1, row[1]-1)
    svd_pred_ratings = pd.DataFrame(svd_pred_ratings, columns=['user_id', 'movie_id', 'rating', 'pred'])
    
    score = abs(svd_pred_ratings['pred'] - svd_pred_ratings['rating'])
    score_2 = score ** 2
    rmse_svd += [np.sqrt(score_2.mean())]
    mae_svd += [score.mean()]
    fold += 1
    print('Fold', fold)
    print('RMSE: {:.4f}'.format(np.sqrt(score_2.mean())))
    print('MAE: {:.4f}'.format(score.mean()))
    print('-'*12)
print('-'*12)
print('Mean RMSE: {:.4f}'.format(np.mean(rmse_svd)))
print('Mean MAE: {:.4f}'.format(np.mean(mae_svd)))
print('-'*12)
print('-'*12)
"""
- **Supervised Learning Model**

Gradient Boosting as supervised learning.

I take each user-movie combination as one instance, and hence take full use of the features from the user and movie.
"""
from xgboost import XGBRegressor
from sklearn.model_selection import train_test_split
#occupation = {'none': 0, 'administrator': 1, 'artist': 2, 'doctor': 3, 'educator': 4, 'engineer': 5, 'entertainment': 6,\
#              'executive': 7, 'healthcare': 8, 'homemaker': 9, 'lawyer': 10, 'librarian': 11, 'marketing': 12,\
#              'programmer': 13, 'salesman': 14, 'scientist': 15, 'student': 16, 'technician': 17, 'writer': 18,\
#              'retired': 19, 'other': 20}
df = ratings_p.stack(dropna=False).reset_index()
df.columns = ['user_id', 'movie_id', 'rating']
df = df.merge(users, on='user_id')
df = df.merge(movies, on='movie_id')
df['sex'] = df['sex'].replace(['F', 'M'], [1, 0])
#df['occupation'] = df['occupation'].replace(occupation)
df.drop(['release_date', 'video_release_date', 'imdb_url', 'title', 'zip_code'], axis=1, inplace=True)
df_train = df.dropna()
df.head(10)
rmse_reg = []
mae_reg = []
i = 0
print('Evaluating RMSE, MAE of the XGB_Reg Model. \n')
print('-'*12)
for train_index, test_index in kfolds.split(ratings):
    X_train = df_train.drop('rating', axis=1).iloc[train_index]
    y_train = df_train['rating'].iloc[train_index]
    X_test = df_train.drop('rating', axis=1).iloc[test_index]
    y_test = df_train['rating'].iloc[test_index]
    xgb = XGBRegressor(learning_rate=0.1, max_depth=10, min_child_weight=10, gamma=0.03).fit(X_train, y_train)
    y_pred = xgb.predict(X_test)
    score = abs(y_test - y_pred)
    score_2 = score**2
    rmse_reg += [np.sqrt(np.mean(score_2))]
    mae_reg += [np.mean(score)]
    i += 1
    print('Fold', i)
    print('RMSE: {:.4f}'.format(np.sqrt(np.mean(score_2))))
    print('MAE: {:.4f}'.format(np.mean(score)))
    print('-'*12)
print('-'*12)
print('Mean RMSE: {:.4f}'.format(np.mean(rmse_reg)))
print('Mean MAE: {:.4f}'.format(np.mean(mae_reg)))
print('-'*12)
print('-'*12)
"""
Here are the Cross Validation RMSE and MAE scores for all models in this notebook
"""
baseline_results = {'Baseline': [np.mean(rmse), np.mean(mae)], 'Baseline_Plus': [np.mean(rmse_plus), np.mean(mae_plus)],\
                    'SVM': [np.mean(rmse_svm), np.mean(mae_svm)], 'XGradientBoosting': [np.mean(rmse_xgb), np.mean(mae_xgb)],\
                    'kNN': [np.mean(rmse_knn), np.mean(mae_knn)], 'kNN_Plus': [np.mean(rmse_knn_plus), np.mean(mae_knn_plus)],\ 
                    'SVD': [np.mean(rmse_svd), np.mean(mae_svd)], 'Supervised_Learning': [np.mean(rmse_reg), np.mean(mae_reg)]}
baseline_results = pd.DataFrame(baseline_results, index=['RMSE', 'MAE']).T
baseline_results
import matplotlib.pyplot as plt

fig = plt.figure(figsize=(10, 6))
ax = plt.subplot(111)
ax.set_axisbelow(True)
plt.bar(np.arange(1, 3*baseline_results.shape[0], 3), baseline_results['RMSE']-0.7, width=1, label='RMSE')
plt.bar(np.arange(2, 3*baseline_results.shape[0], 3), baseline_results['MAE']-0.7, width=1, label='MAE')
plt.xticks(np.arange(1.5, 3*baseline_results.shape[0], 3), baseline_results.index)
plt.yticks(np.arange(0, 0.5, 0.1), [0.7, 0.8, 0.9, 1.0, 1.1])
plt.grid()
plt.legend()
plt.show()
"""
The next model takes more than 6 hours to run. So I don't commit it this time. I'll move it into a seperate notebook.
"""
"""
- **Improvement on Gradient Boosting Model (?)**

In the previous model, we use rating as the target variable, therefore the model knows nothing about what ratings that the movie has received, and what ratings the user has given. Now we might want to include all related ratings, both from the same user and for the same movie, as features in our model.

This model takes a much longer time than others.

Notice that two of the slots on each row in these new features will actually contain the target value! However, I right now have no clue that this will result in data leakage since the model has no idea where these hidden correct ratings locate.

It makes sense that this model will beat all other models, since it contains more information like the age and gender of the user, genre of the movie, etc. However, it is still not safe to say that we are free of data leakage here. So I leave a question mark here.
"""
### BUG. Need Fix. ###

#df = df.merge(ratings_p, left_on='user_id', right_index=True)
#df = df.merge(ratings_p.T, left_on='movie_id', right_index=True)
#df.head(10)
#df_train = df_train.merge(ratings_p, left_on='user_id', right_index=True)
#df_train = df_train.merge(ratings_p.T, left_on='movie_id', right_index=True)
#df_train.head(10)
#for i in range(df_train.shape[0]):
#    if i % 1000 == 0:
#        print(i)
#    row = df_train.iloc[i]
#    df_train.iloc[i][str(row[1])+'_x'] = np.NaN
#    df_train.iloc[i][str(row[0])+'_y'] = np.NaN
"""
Train-test-split score.
"""
#X_train, X_test, y_train, y_test = train_test_split(df_train.drop(['rating'], axis=1), df_train['rating'], random_state = 0)
#xgb = XGBRegressor(learning_rate=0.1, max_depth=10, min_child_weight=10, gamma=0.03).fit(X_train, y_train)
#y_pred = xgb.predict(X_test)
#score = abs(y_test - y_pred)
#score_2 = score**2
#print('RMSE: {:.4f}'.format(np.sqrt(np.mean(score_2))))
#print('MAE: {:.4f}'.format(np.mean(score)))

# RMSE: 0.7401
# MAE: 0.5674
"""
Use our model to recommond movie for user 196.
"""
"""
Here is all movies rated by user 196.
"""
#pred_196 = df[df['user_id']==196]
#pred_196 = pred_196.merge(ratings_p, left_on='user_id', right_index=True)
#pred_196 = pred_196.merge(ratings_p.T, left_on='movie_id', right_index=True)
#pred_196.head(10)
"""
Here is the recommendation to user 196.
"""
#xgb = XGBRegressor(learning_rate=0.1, max_depth=10, min_child_weight=10, gamma=0.03)\
#      .fit(df_train.drop(['rating'], axis=1), df_train['rating'])
#pred_196['rating'] = xgb.predict(pred_196.drop('rating', axis=1))
#user_196_reg = movies[['movie_id', 'title', 'release_date']]
#user_196_reg['Estimate_Score'] = np.array(pred_196['rating'])
#user_196_reg.drop('movie_id', axis=1, inplace=True)
#user_196_reg = user_196_reg.sort_values('Estimate_Score', ascending=False)
#print(user_196_reg.head(10))
"""
Cross-Validation
"""
#rmse_reg_plus = []
#mae_reg_plus = []
#i = 0
#print('Evaluating RMSE, MAE of the XGB_Reg_Plus Model. \n')
#print('-'*12)
#for train_index, test_index in kfolds.split(ratings):
#    X_train = df_train.drop('rating', axis=1).iloc[train_index]
#    y_train = df_train['rating'].iloc[train_index]
#    X_test = df_train.drop('rating', axis=1).iloc[test_index]
#    y_test = df_train['rating'].iloc[test_index]
#    xgb = XGBRegressor(learning_rate=0.1, max_depth=10, min_child_weight=10, gamma=0.03).fit(X_train, y_train)
#    y_pred = xgb.predict(X_test)
#    score = abs(y_test - y_pred)
#    score_2 = score**2
#    rmse_reg_plus += [np.sqrt(np.mean(score_2))]
#    mae_reg_plus += [np.mean(score)]
#    i += 1
#    print('Fold', i)
#    print('RMSE: {:.4f}'.format(np.sqrt(np.mean(score_2))))
#    print('MAE: {:.4f}'.format(np.mean(score)))
#    print('-'*12)
#print('-'*12)
#print('Mean RMSE: {:.4f}'.format(np.mean(rmse_reg_plus)))
#print('Mean MAE: {:.4f}'.format(np.mean(mae_reg_plus)))
#print('-'*12)
#print('-'*12)