%matplotlib inline
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns

import warnings; warnings.simplefilter('ignore')
# Build a full dataset of all datasets in the input dir with pandas
from pathlib import Path
full_dataset = pd.concat([pd.read_csv(str(x.resolve())) for x in Path("/kaggle/input/goodreads-book-datasets-10m/").glob("*k.csv")])
full_dataset.info()
# Each rating column looks like x:432411 where the x is the star count.
# since this is redundant and ruins the data typing, we can take it out.
# turn the RatingDist columns into itegers by splitting of the first part.
for i in range(1,6):
    full_dataset['RatingDist'+str(i)] = full_dataset['RatingDist'+str(i)].str.split(':').str[1]
full_dataset["RatingDistTotal"] = full_dataset["RatingDistTotal"].str.split(':').str[1]

to_convert = ["RatingDist1", "RatingDist2", "RatingDist3", "RatingDist4", "RatingDist5", "RatingDistTotal"]
for col in to_convert:
    full_dataset[col] = full_dataset[col].astype(int)

full_dataset.info()
# Make our dataset smaller in memory Thanks to : https://www.kaggle.com/aantonova/some-new-risk-and-clusters-feature
def reduce_mem_usage(df, verbose=True):
    numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']
    start_mem = df.memory_usage().sum() / 1024**2
    for col in df.columns:
        col_type = df[col].dtypes
        if col_type in numerics:
            c_min = df[col].min()
            c_max = df[col].max()
            if str(col_type)[:3] == 'int':
                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:
                    df[col] = df[col].astype(np.int8)
                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:
                    df[col] = df[col].astype(np.int16)
                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:
                    df[col] = df[col].astype(np.int32)
                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:
                    df[col] = df[col].astype(np.int64)
            else:
                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:
                    df[col] = df[col].astype(np.float16)
                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:
                    df[col] = df[col].astype(np.float32)
                else:
                    df[col] = df[col].astype(np.float64)
    end_mem = df.memory_usage().sum() / 1024**2
    if verbose: print('Mem. usage decreased to {:5.2f} Mb ({:.1f}% reduction)'.format(end_mem, 100 * (start_mem - end_mem) / start_mem))
    return df

reduce_mem_usage(full_dataset)
# look at the dates
full_dataset[["PublishYear", "PublishMonth", "PublishDay"]].describe()
# so something is seriously wrong with publish month... 31st month??? huh? lets drop it.
full_dataset.drop(columns=["PublishMonth"], inplace=True)
# take a look at the years in the dataset
full_dataset[(full_dataset['PublishYear'] < 1400) | (full_dataset['PublishYear'] > 2020)]
# pretty sure 3006 hasn't happenned yet, so let's filter those out
full_dataset = full_dataset[full_dataset['PublishYear'] > 1400]
full_dataset = full_dataset[full_dataset['PublishYear'] < 2022]
full_dataset.info(show_counts=True)
print(full_dataset.Language.unique())
full_dataset["Language"].describe()
# so it's predominatly english books, but lets change en-GB and en-CA to english
full_dataset["Language"].replace(["en-GB", "en-CA", "en-US"], "eng", inplace=True)
print(full_dataset.Language.unique())
full_dataset["Language"].describe()
# we will then drop the rest of the languages as our simple rec-sys will be english only for now.
full_dataset = full_dataset[full_dataset["Language"] == "eng"]
full_dataset.info(show_counts=True)
# lets check out deplicates
full_dataset.Name.value_counts()[:20]
# so there are alot of duplicate listings here, lets drop all the books by the excat same name and authors
# this should remove books that were re-pulished by the same author and name but possibly by a different publisher
full_dataset.drop_duplicates(subset=["Authors", "Name"], inplace=True)
full_dataset.info(show_counts=True)
full_dataset.Name.value_counts()[:20]
full_dataset.head()
# drop some columns we won't need for now
full_dataset.drop(columns=["PagesNumber", "CountsOfReview", "Count of text reviews", "pagesNumber"], inplace=True)
# fill the empty descriptions with the names of the books
full_dataset.Description.fillna(full_dataset.Name, inplace=True)

full_dataset.info(show_counts=True)
# drop remaining columns
full_dataset.dropna(inplace=True)
full_dataset.info(show_counts=True)
# use regex to clean up the descriptions as some of them have embedded HTML tags like <br>
import re
# compile once only
CLEANR = re.compile('<.*?>|&([a-z0-9]+|#[0-9]{1,6}|#x[0-9a-f]{1,6});') 

def cleanhtml(raw_html):
    cleantext = re.sub(CLEANR, '', raw_html)
    return cleantext
full_dataset["Description"] = full_dataset.Description.apply(cleanhtml)
full_dataset.Description.sample(100)
"""
## Simple Weighted Rating

Weighted Rating (WR) = $(\frac{v}{v + q} . R) + (\frac{q}{v + q} . C)$

where,
* *v* is the number of votes for the book (total)
* *q* is the minimum votes required to be listed in the chart
* *R* is the average rating of all books
* *C* is the mean vote across the entire dataset

inspired by: https://www.kaggle.com/rounakbanik/movie-recommender-systems
"""
ratings = full_dataset["Rating"]
rating_count = full_dataset["RatingDistTotal"]
MR = ratings.convert_dtypes(np.float16).mean()
print("Mean Rating (MR) accross all books: ", MR)

q = rating_count.quantile(0.98)
print("Minimum number of required ratings: ", q)

"""
### Weighting Metrics
- ``MR`` = Mean rating = 3.75 (out of 5)
- ``q`` = minimum number of ratings required
"""
# our weighted rating algorithm
def weighted_rating(book):
    v = book['RatingDistTotal']
    R = book['Rating']
    return (v/(v+q) * R) + (q/(q+v) * MR)
top_books = full_dataset[(full_dataset["RatingDistTotal"] >= q) & (full_dataset["Rating"] > MR)]
top_books["weighted"] = top_books.apply(weighted_rating, axis=1)
above_average = top_books.sort_values("weighted", ascending=False)
above_average.shape
# So clearly Harry Potter is a popular book. No surprise there.
above_average.head()
# lets take a random sample
above_average.sample(10).sort_values(by="weighted", ascending=False)
"""
# Content Based Recommendations

The goal here is to see if we can recommend books based on the similarity in the descriptions between books.

"""
from sklearn.metrics.pairwise import linear_kernel
from sklearn.feature_extraction.text import TfidfVectorizer
# sample 20000 points so we don't hit kaggle's memory limit
#samples = full_dataset(20000)

samples = full_dataset[10000:30000] # use if you want the same books to be predicted on
tf = TfidfVectorizer(analyzer='word',
                     ngram_range=(1, 2),
                     min_df=0,
                     stop_words='english')
tfidf_matrix = tf.fit_transform(samples['Description'])
"""
#### Cosine Similarity

I will be using the Cosine Similarity to calculate a numeric quantity that denotes the similarity between two books. Mathematically, it is defined as follows:

$cosine(x,y) = \frac{x. y^\intercal}{||x||.||y||} $

Since we have used the TF-IDF Vectorizer, calculating the Dot Product will directly give us the Cosine Similarity Score. Therefore, we will use sklearn's **linear_kernel** instead of cosine_similarities since it is much faster.
"""
cosine_sim = linear_kernel(tfidf_matrix, tfidf_matrix)
# write function to get recommendations based on cosine similarity
samples = samples.reset_index()
titles = samples[['Name', 'Authors', 'PublishYear', 'Publisher']]
indices = pd.Series(samples.index, index=samples['Name'])
def get_content_recommendations(title):
    try:
        # handle case in which book by same title is in dataset
        idx = indices[title][0]
    except IndexError:
        idx = indices[title]
    sim_scores = list(enumerate(cosine_sim[idx]))
    sim_scores = sorted(sim_scores, key=lambda x: x[1], reverse=True)
    sim_scores = sim_scores[1:15]
    book_indices = [i[0] for i in sim_scores]
    return titles.iloc[book_indices]
index = 17136
title = samples.iloc[index].Name
desc = samples.iloc[index].Description
author = samples.iloc[index].Authors
year = samples.iloc[index].PublishYear
print("Title:", title, "\nDescription:", desc, "\nAuthor:", author, "\nYear:", year)
get_content_recommendations(title)
index = 18457
title = samples.iloc[index].Name
desc = samples.iloc[index].Description
author = samples.iloc[index].Authors
year = samples.iloc[index].PublishYear
print("Title:", title, "\nDescription:", desc, "\nAuthor:", author, "\nYear:", year)
get_content_recommendations(title)
index = 10984
title = samples.iloc[index].Name
desc = samples.iloc[index].Description
author = samples.iloc[index].Authors
year = samples.iloc[index].PublishYear
print("Title:", title, "\nDescription:", desc, "\nAuthor:", author, "\nYear:", year)
get_content_recommendations(title)
"""
# Combined recommendation system (content + weighted rating)

So here were going to combine our content based recommendations with the weighted raiting scheme we cooked up earlier. 
"""
titles = samples[['Name', 'Authors', 'PublishYear', 'Publisher', "Rating", "RatingDistTotal"]]

def get_weighted_content_recommendations(title):
    try:
        # handle case in which book by same title is in dataset
        idx = indices[title][0]
    except IndexError:
        idx = indices[title]
    sim_scores = list(enumerate(cosine_sim[idx]))
    
    # get the top 40 recommendations
    sim_scores = sorted(sim_scores, key=lambda x: x[1], reverse=True)
    sim_scores = sim_scores[1:40]

    book_indices = [i[0] for i in sim_scores]
    books = titles.iloc[book_indices]

    # now sort by weighted rating
    ratings = books["Rating"]
    rating_count = books["RatingDistTotal"]
    MR = ratings.convert_dtypes(np.float16).mean()
    print("Mean Rating (MR) accross all books: ", MR)

    # relax to 75% quantile
    q = rating_count.quantile(.75)
    print("Minimum number of required ratings: ", q)
    
    books["wRating"] = books.apply(weighted_rating, axis=1)
    return books.sort_values(by="wRating", ascending=False)

index = 17136
title = samples.iloc[index].Name
desc = samples.iloc[index].Description
author = samples.iloc[index].Authors
year = samples.iloc[index].PublishYear
print("Title:", title, "\nDescription:", desc, "\nAuthor:", author, "\nYear:", year)
get_weighted_content_recommendations(title)[:10]
index = 18457
title = samples.iloc[index].Name
desc = samples.iloc[index].Description
author = samples.iloc[index].Authors
year = samples.iloc[index].PublishYear
print("Title:", title, "\nDescription:", desc, "\nAuthor:", author, "\nYear:", year)
get_weighted_content_recommendations(title)[:10]
index = 10984
title = samples.iloc[index].Name
desc = samples.iloc[index].Description
author = samples.iloc[index].Authors
year = samples.iloc[index].PublishYear
print("Title:", title, "\nDescription:", desc, "\nAuthor:", author, "\nYear:", year)
get_weighted_content_recommendations(title)[:10]
# comparison!
print("BASE CONTENT RECOMMENDATIONS")
print(get_content_recommendations(title)[:10])

print("\n\nWEIGHTED CONTENT RECOMMENDATIONS")
get_weighted_content_recommendations(title) # see all top 40 for ranking comparison
"""
# Conclusion

So above we can qualitatively see that our recommendations that include the weighted rating are better on average as they aren't suggesting books that have zero ratings (which are probably books that no-one has read).

There is definetly a biased towards books that are most frequently reviewed, but that can be adjusted in the weighted rating if novelty is important for the recommenation system.

Futher work could be done to include user rating data directly on a per user basis, but I will leave that as an exercise for the user.
"""
"""
# Hope you enjoyed!! 

Please feel free to use but be sure to link to any notebooks I got inspiration from (that I linked to)!
"""