import numpy as np # linear algebra
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)
%matplotlib inline
import matplotlib.pyplot as plt # visualization
!pip install seaborn as sns -q # visualization with seaborn v0.11.1
import seaborn as sns # visualization
import missingno as msno # missing values pattern visualization

import warnings # supress warnings
warnings.filterwarnings('always')
warnings.filterwarnings('ignore')

import math


plt.style.use('bmh')

# set pandas display option
pd.set_option('display.max_columns',None)
pd.set_option('display.max_rows',None)

# Load the data 
Books_df = pd.read_csv('../input/book-recommendation-dataset/Books.csv')
Ratings_df = pd.read_csv('../input/book-recommendation-dataset/Ratings.csv')
Users_df = pd.read_csv('../input/book-recommendation-dataset/Users.csv')
# display the dataset
Ratings_df.head().style.set_caption('Sample of Ratings data')
"""
# Summarize the dataset
"""
# dimension of dataset
print(f'''\t  Book_df shape is {Books_df.shape}
          Ratings_df shape is {Ratings_df.shape}
          Users_df shape is {Users_df.shape}''')
def missing_zero_values_table(df):
    mis_val=df.isnull().sum()
    mis_val_percent=round(df.isnull().mean().mul(100),2)
    mz_table=pd.concat([mis_val,mis_val_percent],axis=1)
    mz_table=mz_table.rename(
    columns={df.index.name:'col_name',0:'Missing Values',1:'% of Total Values'})
    mz_table['Data_type']=df.dtypes
    mz_table=mz_table.sort_values('% of Total Values',ascending=False)
    print(f"Your selected dataframe has "+str(df.shape[1])+" columns and "+str(df.shape[0])+" Rows.\n"
         "There are "+str(mz_table[mz_table.iloc[:,1] != 0].shape[0])+
          " columns that have missing values.")
    return mz_table.reset_index()
missing_zero_values_table(Users_df)
missing_zero_values_table(Ratings_df)
missing_zero_values_table(Books_df)
"""
Check outlier data in **Age** and **Book-Rating** column  
"""
f,ax=plt.subplots(1,2,figsize=(18,8))
sns.boxplot(y='Book-Rating', data=Ratings_df,ax=ax[0])
ax[0].set_title('Find outlier data in Rating Book column')
sns.boxplot(y='Age', data=Users_df,ax=ax[1])
ax[1].set_title('Find outlier data in Age column')
print(sorted(Users_df.Age.unique()))
"""
Age : 244 :))
"""
"""
Ok we have Outlier data in Age    
so must be fixed it   
"""
"""
OK let's find our unique value in Location column 
"""
Users_df.Location.unique()
len(Users_df.Location.unique())
"""
57339 unique Value it's really hard to understand  
so use regex and create column country
"""
Books_df['Book-Author'].describe()
"""
Say us Miss [Agatha Christie](https://www.biography.com/writer/agatha-christie) is top in Books data frame
"""
print(Books_df['Year-Of-Publication'].unique().tolist())
"""
Year of publication **2037** !!  
'**Gallimard**' , '**DK Publishing Inc**' , type of sum year is **string**  
"""
1.0 - (np.count_nonzero(Ratings_df)/float(Ratings_df.size))
"""
15 percent sparse
"""
sorted(Ratings_df['Book-Rating'].unique())
"""
0 is an invalid number in the rated books  
and rating value must be 1 to 10
"""
Ratings_df.shape[0]
usersCount=Users_df.shape[0]
booksCount=Books_df.shape[0]
print(f'Users : {usersCount}')
print(f'Books : {booksCount}')
print(f'Total : {usersCount*booksCount}')
"""
Users rated **1149780** books, but there are **271360** books        
so users did not rate all books      
and users participation that rated make two question   
1. Are the books they rated part of the book's data frame ?  
2. Are the users they rated part of the user's data frame ?  


"""
ratings_new = Ratings_df[Ratings_df.ISBN.isin(Books_df.ISBN)]
ratings_new = ratings_new[Ratings_df['User-ID'].isin(Users_df['User-ID'])]
print("Users or books aren't in dataset")
print(f'Total : {Ratings_df.shape[0] - ratings_new.shape[0]}')
sparsity = round(1.0 - len(ratings_new)/float(usersCount*booksCount),6)
sparsity
"""
Age column has 39 percent null data   
and age column has outlier data   
and I can don't use Cosine Similarity   
so let's do it together  
if any things it's not correct I'm really become happy to tell me 
"""
"""
# Visualization and Modeling
"""
"""
**Steps**
1. rename columns names :))
2. Create country column to analyze better 
3. Fill Na value in  Country column 
4. Some data in Country Column has Misspellings 
5. Create rating_Avg and number_of_rating to analyze better
6. users more in which countries
7. Age column has outlier data 
8. Fill Na value in Age column 
9. Fill Na value in Book data frame's Author column 
10. Fill Na value in Book data frame's Publisher column 
11. Book data frame's Year of Publication column has two string value and some integer value  type is string
12. Book data frame's Year of Publication has outlier data 
13. Fill Na value in Book data frame's Year of Publication 
14. join three data frames together
15. Delete user and book columns they rated but aren't in the dataset
16. Rating_book value must be 1 to 10
17. drop three unhelpful columns 'Image-URL-S', 'Image-URL-M', 'Image-URL-L' 
"""
Ratings_df.rename(columns={'User-ID':'user_id','Book-Rating':'book_rating'},inplace=True)
Users_df.rename(columns={'User-ID':'user_id'},inplace=True)
Books_df.rename(columns={'Book-Title':'Book_Title','Book-Author':'Book_Author',
                         'Year-Of-Publication':'Year_Of_Publication'},inplace=True)
"""
**Country Column**
"""
Users_df['Country']='Iran'
for i in Users_df:
    Users_df['Country']=Users_df.Location.str.extract(r'\,+\s?(\w*\s?\w*)\"*$')   
len(Users_df.Country.unique())
Users_df.isnull().sum()
"""
368 of users Country column is Nan so must be fill it 
"""
Users_df.loc[Users_df.Country.isnull(),'Country']='other'
"""
So I don't have any idea Location column has 57339 unique value    
for this I use Regex and create country column   
but we have [195 Countries in the World !!](https://www.worldometers.info/geography/how-many-countries-are-there-in-the-world/)  
But it's better than 57339 unique Location value :))  

"""
pd.crosstab(Users_df.Country,Ratings_df.book_rating).T.style.background_gradient()
"""
Some data has Misspellings 
"""
Users_df['Country'].replace(['','alachua','america','austria','autralia','cananda','geermany','italia','united kindgonm','united sates','united staes','united state','united states','us'],
                           ['other','usa','usa','australia','australia','canada','germany','italy','united kingdom','usa','usa','usa','usa','usa'],inplace=True)
"""
Create Column 'count rate'   
user participation in rated   
and even users rated the books zero   
"""
"""
Rating Average and
"""
# Create column Count_All_Rate
Ratings_df['Count_All_Rate']=Ratings_df.groupby('ISBN')['user_id'].transform('count')
"""
**Country and Users**
"""
cm=sns.light_palette('green',as_cmap=True)
popular=Users_df.Country.value_counts().to_frame()[:10]
popular.rename(columns={'Country':'Count_Users_Country'},inplace=True)
popular.style.background_gradient(cmap=cm)
"""
In the below chart there is one row has named 'other' it's mean    
location is Nan, or regex it's not able to read
"""
"""
**Age Columns**
"""
"""
In the plot and in the unique value   
we understand we have outlier data   
so for outlier data I convert it to Nan value  
"""
# outlier data became NaN
Users_df.loc[(Users_df.Age > 100 ) | (Users_df.Age < 5),'Age']=np.nan
Users_df.Age.plot.hist(bins=20,edgecolor='black',color='red')
round(Users_df.Age.skew(axis=0,skipna=True),3)
"""
Age has **positive Skewness** (right tail)      
so we I have one idea to fill Na value from **Median**   
for this we don't like to fill Na value **just for one range of age** for handle it I use **country column** to fill Na 
"""
# Series of users data live in which country 
countryUsers = Users_df.Country.value_counts()
country=countryUsers[countryUsers>=5].index.tolist()
# Range of Age users in country register in this library and had participation
RangeOfAge = Users_df.loc[Users_df.Country.isin(country)][['Country','Age']].groupby('Country').agg(np.mean).to_dict()

for k,v in RangeOfAge['Age'].items():
    Users_df.loc[(Users_df.Age.isnull())&(Users_df.Country== k),'Age'] = v
    
Users_df.isnull().sum()
"""
POF again we have 330 null Value   
for fill in it   
Age has **positive Skewness** (right tail)        
so we I have one idea to fill Na value from **Median**     
"""
medianAge = int(Users_df.Age.median())
Users_df.loc[Users_df.Age.isnull(),'Age']=medianAge
Users_df.isnull().sum()
"""
**Book Author** column has **Nan** value
"""
Books_df[Books_df.Book_Author.isnull()]
Books_df.loc[(Books_df.ISBN=='9627982032'),'Book_Author']='other'
"""
**Publisher column has Nan value**
"""
Books_df[Books_df.Publisher.isnull()]
Books_df.loc[(Books_df.ISBN=='193169656X'),'Publisher']='other'
Books_df.loc[(Books_df.ISBN=='1931696993'),'Publisher']='other'
"""
**Year of Publication**
"""
Books_df[Books_df.Year_Of_Publication=='Gallimard']
Books_df[Books_df.Year_Of_Publication=='DK Publishing Inc']
Books_df.loc[Books_df.ISBN=='2070426769','Year_Of_Publication']=2003
Books_df.loc[Books_df.ISBN=='2070426769','Book_Author']='Gallimard'
Books_df.loc[Books_df.ISBN=='0789466953','Year_Of_Publication']=2000
Books_df.loc[Books_df.ISBN=='0789466953','Book_Author']='DK Publishing Inc'
Books_df.loc[Books_df.ISBN=='078946697X','Year_Of_Publication']=2000
Books_df.loc[Books_df.ISBN=='078946697X','Book_Author']='DK Publishing Inc'
Books_df.Year_Of_Publication=Books_df.Year_Of_Publication.astype(np.int32)
print(sorted(Books_df.Year_Of_Publication.unique()))
"""
Years of publication after 2021 and 0 it's not normal   
so must be converted to Nan value
"""
Books_df.loc[(Books_df.Year_Of_Publication>=2021)|(Books_df.Year_Of_Publication==0),'Year_Of_Publication']=np.NAN
Books_df.isnull().sum()
author=Books_df[Books_df.Year_Of_Publication.isnull()].Book_Author.unique().tolist()
RangeYearOfPublication = Books_df.loc[Books_df.Book_Author.isin(author)][['Book_Author','Year_Of_Publication']].groupby('Book_Author').agg(np.mean).round(0).to_dict()
meanYear=round(Books_df.Year_Of_Publication.mean())
authorNanYear={}
authorYear={}
for k,v in RangeYearOfPublication['Year_Of_Publication'].items():
    if math.isnan(v) != True:
        authorYear[k]=v
    else:
        authorNanYear[k] = meanYear
len(authorNanYear.keys())
"""
1355 authors don't have a year of publication and the average of them is Nan   
and I forced filling Nan value with mean of all year of publication authors
"""
len(authorYear.keys())
# for k,v in authorYear.items():
#     Books_df.loc[(Books_df.Year_Of_Publication.isnull())&(Books_df.Book_Author== k),'Year_Of_Publication'] = v
"""
1959 authors don't have year of publication of them books    
and they return value   
but it's take long time to fill Nan value   
I would like to find a fast way :))  
but now I don't know   
if you know please tell me in the comment  
"""
"""
This method it's not helpful     
I must find another way      
"""
Books_df.loc[Books_df.Year_Of_Publication.isnull(),'Year_Of_Publication'] = round(Books_df.Year_Of_Publication.mean())
"""
I don't like this method, but I force to use this solution
"""
"""
**new Ratings_book dataset**
"""
ratings_new = Ratings_df[Ratings_df.ISBN.isin(Books_df.ISBN)]
ratings_new = ratings_new[ratings_new.user_id.isin(Users_df.user_id)]
"""
Separate 1 to 10 and 0 rated value
"""
ratings_0 = ratings_new[ratings_new.book_rating ==0]
ratings_1to10 = ratings_new[ratings_new.book_rating !=0]
# Create column Rating average 
ratings_1to10['rating_Avg']=ratings_1to10.groupby('ISBN')['book_rating'].transform('mean')
# Create column Rating sum
ratings_1to10['rating_sum']=ratings_1to10.groupby('ISBN')['book_rating'].transform('sum')
ratings_0.shape[0]
ratings_1to10.shape[0]
ratings_1to10.head()
dataset=Users_df.copy()
dataset=pd.merge(dataset,ratings_1to10,on='user_id')
dataset=pd.merge(dataset,Books_df,on='ISBN')
def skew_test(df):
    col = df.skew(axis = 0, skipna = True)
    val = df.skew(axis = 0, skipna = True) 
    sk_table = pd.concat([col, val], axis = 1)
    sk_table = sk_table.rename(
    columns = {0 : 'skewness'})
    print ("Your selected dataframe has " + str(df.shape[1]) + " columns and " + str(df.shape[0]) + " Rows.\n"      
        "There are " + str(sk_table.shape[0]) +
          " columns that have skewed values - Non Gaussian distribution.")
    return sk_table.drop([1], axis = 1).sort_values('skewness',ascending = False).reset_index()
skk = skew_test(dataset)
skk.style.background_gradient(cmap='Blues')
fig, ax = plt.subplots(figsize=(18,8))
sns.countplot(data=ratings_1to10,x='book_rating',ax=ax)
print(dataset.columns.tolist())
"""
We don't need 3 columns : 'Image-URL-S', 'Image-URL-M', 'Image-URL-L'
"""
dataset=dataset[['user_id', 'Location', 'Age', 'Country', 'ISBN', 'book_rating', 'rating_Avg','rating_sum', 'Count_All_Rate', 'Book_Title', 'Book_Author', 'Year_Of_Publication', 'Publisher']]
missing_zero_values_table(dataset)
"""
Ok everything's ok  
"""
"""
# Simple Popularity based Recommendation System
"""
cm=sns.light_palette('red',as_cmap=True)
# count all rate means include users rated 0 to book
popular=dataset.groupby(['Book_Title','Count_All_Rate','rating_Avg','rating_sum']).size().reset_index().sort_values(['rating_sum','rating_Avg',0],
                                                                                                            ascending=[False,False,True])[:20]
popular.rename(columns={0:'Count_Rate'},inplace=True)
popular.style.background_gradient(cmap=cm)
"""
There are 20 most popular books in dataset  
and they bought and rated it
"""
"""
What !!  
Why it's recommended 'Wild Animus' book   
avg rate is low, but sum rate is high    
this is one problem of that    
Do you know how can I fix this bug ??  
If you know say in the comment box  
"""
"""
# Collaborative Filtering 
"""
"""
I don't have great knowledge, but I try to create best :))
"""
"""
The First step is to find  persons who are similar to user  
so must be calculated distance   
and distance can calculate by those methods    
1. Manhattan distance 
2. Euclidean distance
3. Minkowski distance
"""
dataset.head()
def manhattan(rating1,rating2):
    "Computes the Manhattan distance. Both rating1 and rating2 are dictionaries"
    user1=dict(zip(dataset.loc[dataset.user_id==rating1].Book_Title,dataset.loc[dataset.user_id==rating1].book_rating))
    user2=dict(zip(dataset.loc[dataset.user_id==rating2].Book_Title,dataset.loc[dataset.user_id==rating2].book_rating))
    distance = 0
    for key in user1:
        if key in user2:
            distance += abs(user1[key] - user2[key])
    return distance
print(f'Manhattan distance between user number 8 and 11676 : {manhattan(8,11676)}')
def euclidean(rating1,rating2):
    "Computes the Euclidean distance. Both rating1 and rating2 are dictionaries"
    user1=dict(zip(dataset.loc[dataset.user_id==rating1].Book_Title,dataset.loc[dataset.user_id==rating1].book_rating))
    user2=dict(zip(dataset.loc[dataset.user_id==rating2].Book_Title,dataset.loc[dataset.user_id==rating2].book_rating))
    distance = 0
    for key in user1:
        if key in user2:
            distance += math.pow(abs(user1[key]-user2[key]),2)
    return math.sqrt(distance)
print(f'Euclidean distance between user number 8 and 11676 : {euclidean(8,11676)}')  

def minkowski(rating1,rating2,r):
    """Computes the Minkowski distance. Both rating1 and rating2 are dictionaries"""
    user1=dict(zip(dataset.loc[dataset.user_id==rating1].Book_Title,dataset.loc[dataset.user_id==rating1].book_rating))
    user2=dict(zip(dataset.loc[dataset.user_id==rating2].Book_Title,dataset.loc[dataset.user_id==rating2].book_rating))
    distance = 0
    for key in user1:
        if key in user2:
            distance += math.pow(abs(user1[key]-user2[key]),r)
    return math.pow(distance,1/r)
print(f'Minkowski distance between user number 8 and 11676 : {minkowski(8,11676,2)}') 
"""
Dataset has a lot of users had rated lower than ten books   
and  users don't paid attention to some books  
so I will drop it  
"""
counts1 = ratings_1to10['user_id'].value_counts()
ratings_1to10 = ratings_1to10[ratings_1to10['user_id'].isin(counts1[counts1 >= 100].index)]
counts = ratings_1to10['book_rating'].value_counts()
ratings_1to10 = ratings_1to10[ratings_1to10['book_rating'].isin(counts[counts >= 100].index)]
dataset.user_id.unique().tolist()[500]
def computeNearestNeighbor(username):
    """Creates a sorted list of users based on their distance 
    to username """
    #users = list(dataset.user_id.unique())
    users=dataset.user_id.unique().tolist()[:500]
    distances = []
    for user in users:
        if user != username:
            distance = manhattan(user,username)
            distances.append((distance,user))
    # sort based on distance -- closest first
    distances.sort()
    return distances
computeNearestNeighbor(192762)
def recommend(username):
    """Give list of recommendations"""
    # first find nearest neighbor
    nearest=computeNearestNeighbor(username)[0][1]
    recommendations=[]
    # now find bands neighbor rated that user didn't
    neighborRatings = dataset.loc[dataset.user_id==nearest].Book_Title.tolist()
    userRatings = dataset.loc[dataset.user_id==username].Book_Title.tolist()
    for artist in neighborRatings:
        if not artist in userRatings:
            recommendations.append((artist,int(dataset[(dataset.Book_Title==artist) & (dataset.user_id==nearest)].book_rating)))
    return sorted(recommendations,key=lambda artistTuple : artistTuple[1],reverse=True)
print(recommend(192762))
"""
It shows us Manhattan distance between user 192762 with 500 other users distance to suggest book   
and it's not helpful for high count users   
so must find another solution   
"""
"""
<hr>
"""