# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

import pandas as pd
import numpy as np
# %matplotlib inline
import scipy.stats as ss # For running statistical functions 
import sklearn.neighbors as sn
import random 
import sklearn.metrics as sm

# ## STEP 1: Reading in the data set

netflix = pd.read_csv("C:/Users/micha/Documents/MyDatasets/netflix_title.csv")
netflix

# # STEP 2: Getting The Lead Directors + Headlining Cast Member

netflix['lead_director']= netflix['director'].str.split(",", expand = True).rename(columns = {0: "lead_director"}).iloc[:,0]
netflix.head(2)

netflix['cast_lead']= netflix['cast'].str.split(",", expand = True).iloc[:,0]
netflix.head(2)

netflix = netflix[['show_id','type','title','lead_director','cast_lead','country','release_year','rating','listed_in','description']]
netflix.head(2)

# # STEP 3: Creating a variable to differentiate modern movies based on release year

netflix['is_modern'] = netflix['release_year'].apply(lambda x: 1 if x > 2000 else 0)
netflix = netflix[['show_id', 'type', "title", "lead_director", 'cast_lead', 'country', "is_modern", "rating", 'listed_in', 'description']]

# # STEP 4: Differentiate b/t Movie and TV Show in a binary 

netflix['is_movie'] = netflix['type'].apply(lambda x: 1 if x == "Movie" else 0)
netflix = netflix[["show_id", "is_movie", "title", 'lead_director', 'cast_lead', "country", 'is_modern', "rating", 'listed_in', "description"]]

# # STEP 5: Figuring out if a content is English or not
#
# The approach here will be based on three columns: listed_in, title and country. 
#
# Priority will be given to listed_in since it has a distinct differentiation that anything listing anime, korean, or spanish = non-English.  Every other genre is pretty much up in the air that needs to be further flushed out by the title for presence of non-ASCII characters + country where anything not listing English speaking = Foreign.

netflix['country'] = netflix['country'].replace(np.nan, "Unknown")

list_of_countries = pd.DataFrame(netflix['country'].str.split(",", expand = True)).rename(columns = {0:"country_1", 1:"country_2", 2:"country_3", 3:"country_4", 4:"country_5", 5:"country_6", 6:"country_7", 7:"country_8", 8:"country_9", 9:"country_10", 10:"country_11", 11:"country_12"})

list_of_countries = list_of_countries.select_dtypes(['object']).apply(lambda x: x.str.strip())
list_of_countries

# +
english_countries = "United States", "United Kingdom", "Ireland", "Canada", "Australia", "New Zealand", "Jamaica"

list_of_countries['english_check_1'] = list_of_countries['country_1'].apply(lambda x: "Yes" if x in english_countries else "no")
list_of_countries['english_check_2'] = list_of_countries['country_2'].apply(lambda x: "Yes" if x in english_countries else "no")
list_of_countries['english_check_3'] = list_of_countries['country_3'].apply(lambda x: "Yes" if x in english_countries else "no")
list_of_countries['english_check_4'] = list_of_countries['country_4'].apply(lambda x: "Yes" if x in english_countries else "no")
list_of_countries['english_check_5'] = list_of_countries['country_5'].apply(lambda x: "Yes" if x in english_countries else "no")
list_of_countries['english_check_6'] = list_of_countries['country_6'].apply(lambda x: "Yes" if x in english_countries else "no")
list_of_countries['english_check_7'] = list_of_countries['country_7'].apply(lambda x: "Yes" if x in english_countries else "no")
list_of_countries['english_check_8'] = list_of_countries['country_8'].apply(lambda x: "Yes" if x in english_countries else "no")
list_of_countries['english_check_9'] = list_of_countries['country_9'].apply(lambda x: "Yes" if x in english_countries else "no")
list_of_countries['english_check_10'] = list_of_countries['country_10'].apply(lambda x: "Yes" if x in english_countries else "no")
list_of_countries['english_check_11'] = list_of_countries['country_11'].apply(lambda x: "Yes" if x in english_countries else "no")
list_of_countries['english_check_12'] = list_of_countries['country_12'].apply(lambda x: "Yes" if x in english_countries else "no")

list_of_countries
# -

netflix = pd.concat([netflix,pd.DataFrame(list_of_countries.iloc[:,12:].select_dtypes("object").apply(lambda row: row.str.contains("Yes"), axis = 1).any(axis = 1))], axis = 1)
netflix = netflix.rename(columns = {0:"is_english"})

new_netflix = netflix[['show_id', 'title', 'is_movie', 'lead_director', 'cast_lead', 'is_english', 'is_modern', 'rating', 'listed_in', 'description']]
new_netflix.head(10)

# # STEP 6: Blow-up the genre + rating

list_of_genres = pd.DataFrame(netflix['listed_in'].str.split(",", expand = True)).rename(columns = {0:"genre_1", 1:"genre_2", 2:"genre_3"})
list_of_genres = list_of_genres.select_dtypes(['object']).apply(lambda x: x.str.strip())
list_of_genres.head(800)

list_of_genres['genre_1'].unique()

international = pd.DataFrame(list_of_genres.select_dtypes("object").apply(lambda row: row.str.contains("International Movies|International TV Shows|British TV Shows|Korean TV Shows|Spanish-Language TV Shows"), axis = 1).any(axis = 1)).rename(columns = {0:"is_international"})
drama = pd.DataFrame(list_of_genres.select_dtypes("object").apply(lambda row: row.str.contains("Dramas|TV Dramas"), axis = 1).any(axis = 1)).rename(columns = {0:"is_drama"})
horror = pd.DataFrame(list_of_genres.select_dtypes("object").apply(lambda row: row.str.contains("Horror Movies|TV Horror"), axis = 1).any(axis = 1)).rename(columns = {0:"is_horror"})
action_adventure = pd.DataFrame(list_of_genres.select_dtypes("object").apply(lambda row: row.str.contains("Action & Adventure|TV Action & Adventure"), axis = 1).any(axis = 1)).rename(columns = {0:"is_action_adventure"})
crime_mystery_thrillers = pd.DataFrame(list_of_genres.select_dtypes("object").apply(lambda row: row.str.contains("Crime TV Shows|Thrillers|TV Thrillers|TV Mysteries"), axis = 1).any(axis = 1)).rename(columns = {0:"is_crime_mystery_thrillers"})
docu = pd.DataFrame(list_of_genres.select_dtypes("object").apply(lambda row: row.str.contains("Documentaries|Docuseries"), axis = 1).any(axis = 1)).rename(columns = {0:"is_docu"})
indie = pd.DataFrame(list_of_genres.select_dtypes("object").apply(lambda row: row.str.contains("Independent Movies"), axis = 1).any(axis = 1)).rename(columns = {0:"is_indie"})
reality = pd.DataFrame(list_of_genres.select_dtypes("object").apply(lambda row: row.str.contains("Reality TV"), axis = 1).any(axis = 1)).rename(columns = {0:"is_reality"})
comedy = pd.DataFrame(list_of_genres.select_dtypes("object").apply(lambda row: row.str.contains("TV Comedies|Comedies|Stand-Up Comedy|Stand-Up Comedy & Talk Shows"), axis = 1).any(axis = 1)).rename(columns = {0:"is_comedy"})
anime = pd.DataFrame(list_of_genres.select_dtypes("object").apply(lambda row: row.str.contains("Anime Series|Anime Features"), axis = 1).any(axis = 1)).rename(columns = {0:"is_anime"})
sports = pd.DataFrame(list_of_genres.select_dtypes("object").apply(lambda row: row.str.contains("Sports Movies"), axis = 1).any(axis = 1)).rename(columns = {0:"is_sports"})
sci_fi = pd.DataFrame(list_of_genres.select_dtypes("object").apply(lambda row: row.str.contains("TV Sci-Fi & Fantasy|Sci-Fi & Fantasy"), axis = 1).any(axis = 1)).rename(columns = {0:"is_sci_fi"})
musicals = pd.DataFrame(list_of_genres.select_dtypes("object").apply(lambda row: row.str.contains("Music & Musicals"), axis = 1).any(axis = 1)).rename(columns = {0:"is_musical"})
family = pd.DataFrame(list_of_genres.select_dtypes("object").apply(lambda row: row.str.contains("Kids' TV|Children & Family Movies|Teen TV Shows|Faith & Spirituality"), axis = 1).any(axis = 1)).rename(columns = {0:"is_family"})
classic_cult = pd.DataFrame(list_of_genres.select_dtypes("object").apply(lambda row: row.str.contains("Classic Movies|Cult Movies|Classic & Cult TV"), axis = 1).any(axis = 1)).rename(columns = {0:"is_classic"})
romantic = pd.DataFrame(list_of_genres.select_dtypes("object").apply(lambda row: row.str.contains("Romantic TV Shows|Romantic Movies|LGBTQ Movies"), axis = 1).any(axis = 1)).rename(columns = {0:"is_romantic"})

new_netflix = pd.concat([new_netflix, international, drama, horror, action_adventure, crime_mystery_thrillers, docu, indie, reality, comedy, anime, sports, sci_fi, musicals, family, classic_cult, romantic], axis = 1)
new_netflix = new_netflix[['show_id', 'title', 'is_movie', 'lead_director', 'cast_lead','is_english', 'is_modern', 'rating', 'description', 'is_international', 'is_drama', 'is_horror', 'is_action_adventure', 'is_crime_mystery_thrillers', 'is_docu', 'is_indie', 'is_reality','is_comedy', 'is_anime', 'is_sports', 'is_sci_fi', 'is_musical', 'is_family', 'is_classic', 'is_romantic']]
new_netflix.head(3)

netflix['rating'].unique()

new_netflix["is_TV_MA"] = pd.DataFrame(netflix['rating'].apply(lambda x: 1 if x == "TV-MA" else 0))
new_netflix["is_R"] = pd.DataFrame(netflix['rating'].apply(lambda x: 1 if x == "R" else 0))
new_netflix["is_PG_13"] = pd.DataFrame(netflix['rating'].apply(lambda x: 1 if x == "PG-13" else 0))
new_netflix["is_TV_14"] = pd.DataFrame(netflix['rating'].apply(lambda x: 1 if x == "TV-14" else 0))
new_netflix["is_TV_PG"] = pd.DataFrame(netflix['rating'].apply(lambda x: 1 if x == "TV-PG" else 0))
new_netflix["is_PG"] = pd.DataFrame(netflix['rating'].apply(lambda x: 1 if x == "PG" else 0))
new_netflix["is_TV_G"] = pd.DataFrame(netflix['rating'].apply(lambda x: 1 if x == "TV-G" else 0))
new_netflix["is_TV_Y"] = pd.DataFrame(netflix['rating'].apply(lambda x: 1 if x == "TV-Y" else 0))
new_netflix["is_TV_Y7"] = pd.DataFrame(netflix['rating'].apply(lambda x: 1 if x == "TV-Y7" else (1 if x == "TV-Y7-FV" else 0)))
new_netflix["is_G"] = pd.DataFrame(netflix['rating'].apply(lambda x: 1 if x == "G" else 0))
new_netflix["is_NC_17"] = pd.DataFrame(netflix['rating'].apply(lambda x: 1 if x == "NC-17" else 0))
new_netflix["is_unrated"] = pd.DataFrame(netflix['rating'].apply(lambda x: 1 if x == "NR" else (1 if x == "UR" else 0)))
new_netflix.head(4)

new_netflix.columns

new_netflix = new_netflix[['show_id', 'title', 'is_movie', 'lead_director', 'cast_lead','is_english', 'is_modern', 'description', 'is_international', 'is_drama', 'is_horror','is_action_adventure', 'is_crime_mystery_thrillers', 'is_docu', 'is_indie', 'is_reality', 'is_comedy', 'is_anime', 'is_sports', 'is_sci_fi', 'is_musical', 'is_family', 'is_classic', 'is_romantic', 'is_TV_MA', 'is_R', 'is_PG_13', 'is_TV_14', 'is_TV_PG', 'is_PG', 'is_TV_G', 'is_TV_Y', 'is_TV_Y7', 'is_G', 'is_NC_17','is_unrated']]
new_netflix

new_netflix_update = new_netflix.copy()
new_netflix_update.index = new_netflix['title']

new_netflix_update = new_netflix_update[['show_id', 'is_movie', 'lead_director', 'cast_lead', 'is_english', 'is_modern', 'description', 'is_international', 'is_drama', 'is_horror', 'is_action_adventure', 'is_crime_mystery_thrillers', 'is_docu', 'is_indie', 'is_reality', 'is_comedy', 'is_anime', 'is_sports', 'is_sci_fi', 'is_musical', 'is_family', 'is_classic', 'is_romantic', 'is_TV_MA', 'is_R', 'is_PG_13', 'is_TV_14', 'is_TV_PG', 'is_PG', 'is_TV_G', 'is_TV_Y', 'is_TV_Y7', 'is_G', 'is_NC_17', 'is_unrated']]
new_netflix_update.head(10)

new_netflix_update['is_english'] = new_netflix_update['is_english'].apply(lambda x: 0 if x == False else 1)
new_netflix_update.head(3)

new_netflix_update['is_international'] = new_netflix_update['is_international'].apply(lambda x: 1 if x == True else 0)
new_netflix_update['is_drama'] = new_netflix_update['is_drama'].apply(lambda x: 1 if x == True else 0)
new_netflix_update['is_horror'] = new_netflix_update['is_horror'].apply(lambda x: 1 if x == True else 0)
new_netflix_update['is_action_adventure'] = new_netflix_update['is_action_adventure'].apply(lambda x: 1 if x == True else 0)
new_netflix_update['is_crime_mystery_thrillers'] = new_netflix_update['is_crime_mystery_thrillers'].apply(lambda x: 1 if x == True else 0)
new_netflix_update['is_docu'] = new_netflix_update['is_docu'].apply(lambda x: 1 if x == True else 0)
new_netflix_update['is_indie'] = new_netflix_update['is_indie'].apply(lambda x: 1 if x == True else 0)
new_netflix_update['is_reality'] = new_netflix_update['is_reality'].apply(lambda x: 1 if x == True else 0)
new_netflix_update['is_comedy'] = new_netflix_update['is_comedy'].apply(lambda x: 1 if x == True else 0)
new_netflix_update['is_anime'] = new_netflix_update['is_anime'].apply(lambda x: 1 if x == True else 0)
new_netflix_update['is_sports'] = new_netflix_update['is_sports'].apply(lambda x: 1 if x == True else 0)
new_netflix_update['is_sci_fi'] = new_netflix_update['is_sci_fi'].apply(lambda x: 1 if x == True else 0)
new_netflix_update['is_musical'] = new_netflix_update['is_musical'].apply(lambda x: 1 if x == True else 0)
new_netflix_update['is_family'] = new_netflix_update['is_family'].apply(lambda x: 1 if x == True else 0)
new_netflix_update['is_classic'] = new_netflix_update['is_classic'].apply(lambda x: 1 if x == True else 0)
new_netflix_update['is_romantic'] = new_netflix_update['is_romantic'].apply(lambda x: 1 if x == True else 0)
new_netflix_update

# # STEP 7: Last final fixes with missing entries for director + lead cast

new_netflix_update['lead_director'] = new_netflix_update['lead_director'].fillna("Unknown Director")
new_netflix_update['cast_lead'] = new_netflix_update['cast_lead'].fillna("Unknown Cast")
new_netflix_update.head(20)

new_netflix_update_knn = new_netflix_update[['is_movie', 'lead_director', 'cast_lead', 'is_english',
       'is_modern', 'is_international', 'is_drama', 'is_horror',
       'is_action_adventure', 'is_crime_mystery_thrillers', 'is_docu',
       'is_indie', 'is_reality', 'is_comedy', 'is_anime', 'is_sports',
       'is_sci_fi', 'is_musical', 'is_family', 'is_classic', 'is_romantic',
       'is_TV_MA', 'is_R', 'is_PG_13', 'is_TV_14', 'is_TV_PG', 'is_PG',
       'is_TV_G', 'is_TV_Y', 'is_TV_Y7', 'is_G', 'is_NC_17', 'is_unrated']]

new_netflix_update_knn

dummified_cast = pd.get_dummies(new_netflix_update_knn['cast_lead'])
dummified_director = pd.get_dummies(new_netflix_update_knn['lead_director'])

dummified_director

newer_netflix_update_knn = new_netflix_update_knn[['is_movie','is_english', 'is_modern',
       'is_international', 'is_drama', 'is_horror', 'is_action_adventure',
       'is_crime_mystery_thrillers', 'is_docu', 'is_indie', 'is_reality',
       'is_comedy', 'is_anime', 'is_sports', 'is_sci_fi', 'is_musical',
       'is_family', 'is_classic', 'is_romantic', 'is_TV_MA', 'is_R',
       'is_PG_13', 'is_TV_14', 'is_TV_PG', 'is_PG', 'is_TV_G', 'is_TV_Y',
       'is_TV_Y7', 'is_G', 'is_NC_17', 'is_unrated']]

netflix_update_knn = pd.concat([newer_netflix_update_knn, dummified_cast], axis = 1)
netflix_update_knn.head(3)

# # STEP 8: Content-Based Recommender with a KNN Algorithm

from sklearn.neighbors import NearestNeighbors

from scipy.sparse import csr_matrix

netflix_update_knn_bool = netflix_update_knn.iloc[:, :].astype("bool")

netflix_bool_feature_matrix = csr_matrix(netflix_update_knn_bool.values)
print(netflix_bool_feature_matrix)

knn_model = NearestNeighbors(algorithm = 'brute', metric = 'cosine')
knn_model.fit(netflix_bool_feature_matrix)

recommended = []

for each in netflix_update_knn_bool.index:
    
    distances, indices = knn_model.kneighbors(netflix_update_knn_bool.loc[each, :].values.reshape(1, -1), n_neighbors = 20)
    
    for num in range(0,len(distances.flatten())):  
        
        recommended.append([each, num, netflix_update_knn_bool.index[indices.flatten()[num]], distances.flatten()[num]])
    

netflix_recommendations = pd.DataFrame(recommended, columns = ["title", "recommendation_placement", "recommended_titles", "metric"])

netflix_recommendations = netflix_recommendations.drop_duplicates()

netflix_recommendations[(netflix_recommendations['title'] == "Step Brothers") & (netflix_recommendations['recommended_titles'] != "Step Brothers")].sort_values("metric")

netflix_recommendations[(netflix_recommendations['title'] == "Lucifer") & (netflix_recommendations['recommended_titles'] != "Lucifer")].sort_values("metric")

netflix_recommendations[(netflix_recommendations['title'] == "Fairy Tail") & (netflix_recommendations['recommended_titles'] != "Fairy Tail")].sort_values("metric")


# # STEP 9: Create all of this into a single function

def recommendations(title, num_recommendations):
    
    # Create a matrix where you'll be using to calculate the distance b/t each title
    some_matrix = csr_matrix(netflix_update_knn_bool.values)
    # Establish the unsupervised learner for generating recommendations ... use the brute force approach with appropriate metric
    knn_model = NearestNeighbors(algorithm = 'brute', metric = 'cosine')
    # Fit the nearest neighbors estimator from the training dataset
    knn_model.fit(some_matrix)
    
    # Have an empty list to populate with recommendations
    recommended = []
    
    # Iterate over each row in the matrix to get the actual distance metric and then populate the empty list with relevant into
    
    for each in netflix_update_knn_bool.index:
        distances, indices = knn_model.kneighbors(netflix_update_knn_bool.loc[each, :].values.reshape(1, -1), n_neighbors = num_recommendations)
    
        for num in range(0,len(distances.flatten())):  
            recommended.append([each, num, netflix_update_knn_bool.index[indices.flatten()[num]], distances.flatten()[num]])
    
    # Rename the columns of the recommendation matrix-turned data frame
    recommendations = pd.DataFrame(recommended, columns = ["title", "recommendation_placement", "recommended_titles", "metric"])
    # Remove duplicate entries 
    recommendations = recommendations.drop_duplicates()
    # REtrieve the recommendations 
    recommend = recommendations[(recommendations['title'] == title) & (recommendations['recommended_titles'] != title)].sort_values("metric")
    return recommend


recommendations("Lucifer", 15)

recommendations("The Other Guys", 10)

StarcoderdataJupyterScriptsDedupFiltered