import pandas as pd
%matplotlib inline
import matplotlib.pyplot as plt
import numpy as np

import seaborn as sns
sns.set_context('notebook')

"""
Explanation: Table of Contents
<p><div class="lev1 toc-item"><a href="#YOUR-NAME-(NEPTUN)" data-toc-modified-id="YOUR-NAME-(NEPTUN)-1"><span class="toc-item-num">1&nbsp;&nbsp;</span>YOUR NAME (NEPTUN)</a></div><div class="lev1 toc-item"><a href="#Business-Intelligence---Pandas" data-toc-modified-id="Business-Intelligence---Pandas-2"><span class="toc-item-num">2&nbsp;&nbsp;</span>Business Intelligence - Pandas</a></div><div class="lev3 toc-item"><a href="#Submission" data-toc-modified-id="Submission-201"><span class="toc-item-num">2.0.1&nbsp;&nbsp;</span>Submission</a></div><div class="lev3 toc-item"><a href="#Tips" data-toc-modified-id="Tips-202"><span class="toc-item-num">2.0.2&nbsp;&nbsp;</span>Tips</a></div><div class="lev3 toc-item"><a href="#Credits" data-toc-modified-id="Credits-203"><span class="toc-item-num">2.0.3&nbsp;&nbsp;</span>Credits</a></div><div class="lev3 toc-item"><a href="#Feedback" data-toc-modified-id="Feedback-204"><span class="toc-item-num">2.0.4&nbsp;&nbsp;</span>Feedback</a></div><div class="lev2 toc-item"><a href="#Code-quality" data-toc-modified-id="Code-quality-21"><span class="toc-item-num">2.1&nbsp;&nbsp;</span>Code quality</a></div><div class="lev2 toc-item"><a href="#PEP8-style-guide" data-toc-modified-id="PEP8-style-guide-22"><span class="toc-item-num">2.2&nbsp;&nbsp;</span>PEP8 style guide</a></div><div class="lev2 toc-item"><a href="#Figure-quality" data-toc-modified-id="Figure-quality-23"><span class="toc-item-num">2.3&nbsp;&nbsp;</span>Figure quality</a></div><div class="lev1 toc-item"><a href="#Main-imports" data-toc-modified-id="Main-imports-3"><span class="toc-item-num">3&nbsp;&nbsp;</span>Main imports</a></div><div class="lev3 toc-item"><a href="#downloading-the-dataset" data-toc-modified-id="downloading-the-dataset-301"><span class="toc-item-num">3.0.1&nbsp;&nbsp;</span>downloading the dataset</a></div><div class="lev1 toc-item"><a href="#Loading-the-dataset" data-toc-modified-id="Loading-the-dataset-4"><span class="toc-item-num">4&nbsp;&nbsp;</span>Loading the dataset</a></div><div class="lev1 toc-item"><a href="#Normalizing-the-dataset" data-toc-modified-id="Normalizing-the-dataset-5"><span class="toc-item-num">5&nbsp;&nbsp;</span>Normalizing the dataset</a></div><div class="lev2 toc-item"><a href="#the-.dt-and-the-.str-namespaces" data-toc-modified-id="the-.dt-and-the-.str-namespaces-51"><span class="toc-item-num">5.1&nbsp;&nbsp;</span>the <code>.dt</code> and the <code>.str</code> namespaces</a></div><div class="lev3 toc-item"><a href="#.dt-namespace" data-toc-modified-id=".dt-namespace-511"><span class="toc-item-num">5.1.1&nbsp;&nbsp;</span><code>.dt</code> namespace</a></div><div class="lev3 toc-item"><a href="#.str-namespace" data-toc-modified-id=".str-namespace-512"><span class="toc-item-num">5.1.2&nbsp;&nbsp;</span><code>.str</code> namespace</a></div><div class="lev2 toc-item"><a href="#Let's-extract-the-release-year-of-a-movie-into-a-separate" data-toc-modified-id="Let's-extract-the-release-year-of-a-movie-into-a-separate-52"><span class="toc-item-num">5.2&nbsp;&nbsp;</span>Let's extract the release year of a movie into a separate</a></div><div class="lev3 toc-item"><a href="#the-most-common-years-are" data-toc-modified-id="the-most-common-years-are-521"><span class="toc-item-num">5.2.1&nbsp;&nbsp;</span>the most common years are</a></div><div class="lev3 toc-item"><a href="#video_release_date-is-always-NaT-(not-a-time),-let's-drop-it" data-toc-modified-id="video_release_date-is-always-NaT-(not-a-time),-let's-drop-it-522"><span class="toc-item-num">5.2.2&nbsp;&nbsp;</span><code>video_release_date</code> is always <code>NaT</code> (not a time), let's drop it</a></div><div class="lev1 toc-item"><a href="#Basic-analysis-of-the-dataset" data-toc-modified-id="Basic-analysis-of-the-dataset-6"><span class="toc-item-num">6&nbsp;&nbsp;</span>Basic analysis of the dataset</a></div><div class="lev1 toc-item"><a href="#Basic-queries" data-toc-modified-id="Basic-queries-7"><span class="toc-item-num">7&nbsp;&nbsp;</span>Basic queries</a></div><div class="lev1 toc-item"><a href="#Which-movies-were-released-in-1956?" data-toc-modified-id="Which-movies-were-released-in-1956?-8"><span class="toc-item-num">8&nbsp;&nbsp;</span>Which movies were released in 1956?</a></div><div class="lev1 toc-item"><a href="#How-many-movies-were-released-in-the-80s?" data-toc-modified-id="How-many-movies-were-released-in-the-80s?-9"><span class="toc-item-num">9&nbsp;&nbsp;</span>How many movies were released in the 80s?</a></div><div class="lev1 toc-item"><a href="#When-were-the-Die-Hard-movies-released?" data-toc-modified-id="When-were-the-Die-Hard-movies-released?-10"><span class="toc-item-num">10&nbsp;&nbsp;</span>When were the Die Hard movies released?</a></div><div class="lev2 toc-item"><a href="#How-many-movies-are-both-action-and-romance?-What-about-action-or-romance?" data-toc-modified-id="How-many-movies-are-both-action-and-romance?-What-about-action-or-romance?-101"><span class="toc-item-num">10.1&nbsp;&nbsp;</span>How many movies are both action and romance? What about action or romance?</a></div><div class="lev1 toc-item"><a href="#Problem-Set-1:-simple-queries" data-toc-modified-id="Problem-Set-1:-simple-queries-11"><span class="toc-item-num">11&nbsp;&nbsp;</span>Problem Set 1: simple queries</a></div><div class="lev1 toc-item"><a href="#Q1.1-How-many-action-movies-were-release-before-1985-and-in-1985-or-later?-(2-points)" data-toc-modified-id="Q1.1-How-many-action-movies-were-release-before-1985-and-in-1985-or-later?-(2-points)-12"><span class="toc-item-num">12&nbsp;&nbsp;</span>Q1.1 How many <em>action</em> movies were release before 1985 and in 1985 or later? (2 points)</a></div><div class="lev1 toc-item"><a href="#Q1.2-Are-there-thrillers-for-children?-Find-an-example.-(2-points)" data-toc-modified-id="Q1.2-Are-there-thrillers-for-children?-Find-an-example.-(2-points)-13"><span class="toc-item-num">13&nbsp;&nbsp;</span>Q1.2 Are there thrillers for children? Find an example. (2 points)</a></div><div class="lev1 toc-item"><a href="#Q1.3-How-many-movies-have-title-longer-than-40-characters?-(3-points)" data-toc-modified-id="Q1.3-How-many-movies-have-title-longer-than-40-characters?-(3-points)-14"><span class="toc-item-num">14&nbsp;&nbsp;</span>Q1.3 How many movies have title longer than 40 characters? (3 points)</a></div><div class="lev1 toc-item"><a href="#pd.Series.apply:-running-arbitrary-functions-on-each-element" data-toc-modified-id="pd.Series.apply:-running-arbitrary-functions-on-each-element-15"><span class="toc-item-num">15&nbsp;&nbsp;</span><code>pd.Series.apply</code>: running arbitrary functions on each element</a></div><div class="lev1 toc-item"><a href="#Q1.4*-How-many-content-words-does-the-average-title-have?-Which-title-has-the-most/least-words?" data-toc-modified-id="Q1.4*-How-many-content-words-does-the-average-title-have?-Which-title-has-the-most/least-words?-16"><span class="toc-item-num">16&nbsp;&nbsp;</span>Q1.4* How many content words does the average title have? Which title has the most/least words?</a></div><div class="lev2 toc-item"><a href="#Step-1.-Count-words-in-the-title-(2-points)" data-toc-modified-id="Step-1.-Count-words-in-the-title-(2-points)-161"><span class="toc-item-num">16.1&nbsp;&nbsp;</span>Step 1. Count words in the title (2 points)</a></div><div class="lev2 toc-item"><a href="#Step-2.-Shortest-and-longest-titles-by-word-count-(3-points)" data-toc-modified-id="Step-2.-Shortest-and-longest-titles-by-word-count-(3-points)-162"><span class="toc-item-num">16.2&nbsp;&nbsp;</span>Step 2. Shortest and longest titles by word count (3 points)</a></div><div class="lev1 toc-item"><a href="#Q1.5*-How-many-movies-have-the-word-'and'-in-their-title?-Write-a-function-that-counts-movies-with-a-particular-word.-(3-points)" data-toc-modified-id="Q1.5*-How-many-movies-have-the-word-'and'-in-their-title?-Write-a-function-that-counts-movies-with-a-particular-word.-(3-points)-17"><span class="toc-item-num">17&nbsp;&nbsp;</span>Q1.5* How many movies have the word 'and' in their title? Write a function that counts movies with a particular word. (3 points)</a></div><div class="lev1 toc-item"><a href="#Groupby-and-visualization" data-toc-modified-id="Groupby-and-visualization-18"><span class="toc-item-num">18&nbsp;&nbsp;</span>Groupby and visualization</a></div><div class="lev3 toc-item"><a href="#we-can-groupby-on-multiple-columns" data-toc-modified-id="we-can-groupby-on-multiple-columns-1801"><span class="toc-item-num">18.0.1&nbsp;&nbsp;</span>we can groupby on multiple columns</a></div><div class="lev3 toc-item"><a href="#we-can-also-group-on-arbitrary-conditions" data-toc-modified-id="we-can-also-group-on-arbitrary-conditions-1802"><span class="toc-item-num">18.0.2&nbsp;&nbsp;</span>we can also group on arbitrary conditions</a></div><div class="lev1 toc-item"><a href="#Problem-Set-2:-Groupby-and-visualization" data-toc-modified-id="Problem-Set-2:-Groupby-and-visualization-19"><span class="toc-item-num">19&nbsp;&nbsp;</span>Problem Set 2: Groupby and visualization</a></div><div class="lev1 toc-item"><a href="#Q2.1-Write-a-function-that-takes-a-genre-and-groups-movies-of-that-genre-by-year.-Do-not-include-movies-older-than-1985.--(3-points)" data-toc-modified-id="Q2.1-Write-a-function-that-takes-a-genre-and-groups-movies-of-that-genre-by-year.-Do-not-include-movies-older-than-1985.--(3-points)-20"><span class="toc-item-num">20&nbsp;&nbsp;</span>Q2.1 Write a function that takes a genre and groups movies of that genre by year. Do not include movies older than 1985.  (3 points)</a></div><div class="lev1 toc-item"><a href="#Q2.2-Plot-the-number-of-adventure-movies-from-1985-to-1999-on-a-bar-chart.-Use-your-group_genre_by_year-function.-(2-points)" data-toc-modified-id="Q2.2-Plot-the-number-of-adventure-movies-from-1985-to-1999-on-a-bar-chart.-Use-your-group_genre_by_year-function.-(2-points)-21"><span class="toc-item-num">21&nbsp;&nbsp;</span>Q2.2 Plot the number of adventure movies from 1985 to 1999 on a <em>bar</em> chart. Use your <code>group_genre_by_year</code> function. (2 points)</a></div><div class="lev1 toc-item"><a href="#Q2.3-Plot-the-distribution-of-release-day-(day-of-month)-on-a-pie-chart." data-toc-modified-id="Q2.3-Plot-the-distribution-of-release-day-(day-of-month)-on-a-pie-chart.-22"><span class="toc-item-num">22&nbsp;&nbsp;</span>Q2.3 Plot the distribution of release day (day of month) on a pie chart.</a></div><div class="lev2 toc-item"><a href="#Step-1.-groupby-(2-points)" data-toc-modified-id="Step-1.-groupby-(2-points)-221"><span class="toc-item-num">22.1&nbsp;&nbsp;</span>Step 1. groupby (2 points)</a></div><div class="lev2 toc-item"><a href="#Step-2.-pie-chart.-Add-percent-values.-(3-points)" data-toc-modified-id="Step-2.-pie-chart.-Add-percent-values.-(3-points)-222"><span class="toc-item-num">22.2&nbsp;&nbsp;</span>Step 2. pie chart. Add percent values. (3 points)</a></div><div class="lev1 toc-item"><a href="#Q2.4-We're-building-a-traditional-lexicon-of-the-titles.-What-is-the-distribution-of-initial-letters-(i.e.-how-many-titles-start-with-S?)?-Plot-it-on-a-bar-chart." data-toc-modified-id="Q2.4-We're-building-a-traditional-lexicon-of-the-titles.-What-is-the-distribution-of-initial-letters-(i.e.-how-many-titles-start-with-S?)?-Plot-it-on-a-bar-chart.-23"><span class="toc-item-num">23&nbsp;&nbsp;</span>Q2.4 We're building a traditional lexicon of the titles. What is the distribution of initial letters (i.e. how many titles start with S?)? Plot it on a bar chart.</a></div><div class="lev2 toc-item"><a href="#Step-1.-Compute-frequencies.-(3-points)" data-toc-modified-id="Step-1.-Compute-frequencies.-(3-points)-231"><span class="toc-item-num">23.1&nbsp;&nbsp;</span>Step 1. Compute frequencies. (3 points)</a></div><div class="lev2 toc-item"><a href="#Step-2.-Plot-it-on-a-bar-chart-in-descending-order.-(3-points)" data-toc-modified-id="Step-2.-Plot-it-on-a-bar-chart-in-descending-order.-(3-points)-232"><span class="toc-item-num">23.2&nbsp;&nbsp;</span>Step 2. Plot it on a bar chart in descending order. (3 points)</a></div><div class="lev1 toc-item"><a href="#Problem-Set-3.-Handling-multiple-dataframes" data-toc-modified-id="Problem-Set-3.-Handling-multiple-dataframes-24"><span class="toc-item-num">24&nbsp;&nbsp;</span>Problem Set 3. Handling multiple dataframes</a></div><div class="lev1 toc-item"><a href="#Merging-with-the-movies-dataframe" data-toc-modified-id="Merging-with-the-movies-dataframe-25"><span class="toc-item-num">25&nbsp;&nbsp;</span>Merging with the movies dataframe</a></div><div class="lev2 toc-item"><a href="#How-many-ratings-are-timestamped-before-the-release-date?" data-toc-modified-id="How-many-ratings-are-timestamped-before-the-release-date?-251"><span class="toc-item-num">25.1&nbsp;&nbsp;</span>How many ratings are timestamped <em>before</em> the release date?</a></div><div class="lev2 toc-item"><a href="#Which-movies-were-rated-before-the-release-date?" data-toc-modified-id="Which-movies-were-rated-before-the-release-date?-252"><span class="toc-item-num">25.2&nbsp;&nbsp;</span>Which movies were rated before the release date?</a></div><div class="lev1 toc-item"><a href="#Q3.1.-How-many-movies-were-rated-4+-at-least-once?-(3-points)" data-toc-modified-id="Q3.1.-How-many-movies-were-rated-4+-at-least-once?-(3-points)-26"><span class="toc-item-num">26&nbsp;&nbsp;</span>Q3.1. How many movies were rated 4+ at least once? (3 points)</a></div><div class="lev1 toc-item"><a href="#Q3.2.-How-many-time-was-each-movie-rated?" data-toc-modified-id="Q3.2.-How-many-time-was-each-movie-rated?-27"><span class="toc-item-num">27&nbsp;&nbsp;</span>Q3.2. How many time was each movie rated?</a></div><div class="lev2 toc-item"><a href="#Step-1.-Compute-the-frequencies-of-movie-ratings.-Use-movie-titles-instead-of-ids.-(2-points)" data-toc-modified-id="Step-1.-Compute-the-frequencies-of-movie-ratings.-Use-movie-titles-instead-of-ids.-(2-points)-271"><span class="toc-item-num">27.1&nbsp;&nbsp;</span>Step 1. Compute the frequencies of movie ratings. Use movie titles instead of ids. (2 points)</a></div><div class="lev2 toc-item"><a href="#Step-2.-Plot-the-frequencies-on-a-histogram.-(2-points)" data-toc-modified-id="Step-2.-Plot-the-frequencies-on-a-histogram.-(2-points)-272"><span class="toc-item-num">27.2&nbsp;&nbsp;</span>Step 2. Plot the frequencies on a histogram. (2 points)</a></div><div class="lev1 toc-item"><a href="#Q3.3.-How-many-ratings-were-submitted-by-each-day-of-the-week?-What-were-their-averages?" data-toc-modified-id="Q3.3.-How-many-ratings-were-submitted-by-each-day-of-the-week?-What-were-their-averages?-28"><span class="toc-item-num">28&nbsp;&nbsp;</span>Q3.3. How many ratings were submitted by each day of the week? What were their averages?</a></div><div class="lev2 toc-item"><a href="#Step-1.-groupby-(3-points)" data-toc-modified-id="Step-1.-groupby-(3-points)-281"><span class="toc-item-num">28.1&nbsp;&nbsp;</span>Step 1. groupby (3 points)</a></div><div class="lev2 toc-item"><a href="#Step-2.-number-of-ratings-per-day-(2-points)" data-toc-modified-id="Step-2.-number-of-ratings-per-day-(2-points)-282"><span class="toc-item-num">28.2&nbsp;&nbsp;</span>Step 2. number of ratings per day (2 points)</a></div><div class="lev2 toc-item"><a href="#Step-3.-mean-rating-by-day-(2-points)" data-toc-modified-id="Step-3.-mean-rating-by-day-(2-points)-283"><span class="toc-item-num">28.3&nbsp;&nbsp;</span>Step 3. mean rating by day (2 points)</a></div><div class="lev1 toc-item"><a href="#Q3.4**-What-is-the-mean-of-ratings-by-genre?" data-toc-modified-id="Q3.4**-What-is-the-mean-of-ratings-by-genre?-29"><span class="toc-item-num">29&nbsp;&nbsp;</span>Q3.4** What is the mean of ratings by genre?</a></div><div class="lev2 toc-item"><a href="#Step-1.-Compute-the-mean-scores.-(5-points)" data-toc-modified-id="Step-1.-Compute-the-mean-scores.-(5-points)-291"><span class="toc-item-num">29.1&nbsp;&nbsp;</span>Step 1. Compute the mean scores. (5 points)</a></div><div class="lev2 toc-item"><a href="#Step-2.-Plot-it-on-a-bar-chart-in-descending-order-by-score.-Set-the-limits-of-the-y-axis-to-(2.5,-4).-(3-points)" data-toc-modified-id="Step-2.-Plot-it-on-a-bar-chart-in-descending-order-by-score.-Set-the-limits-of-the-y-axis-to-(2.5,-4).-(3-points)-292"><span class="toc-item-num">29.2&nbsp;&nbsp;</span>Step 2. Plot it on a bar chart in descending order by score. Set the limits of the y-axis to (2.5, 4). (3 points)</a></div><div class="lev1 toc-item"><a href="#Problem-Set-4.-User-demographics" data-toc-modified-id="Problem-Set-4.-User-demographics-30"><span class="toc-item-num">30&nbsp;&nbsp;</span>Problem Set 4. User demographics</a></div><div class="lev1 toc-item"><a href="#Q4.1-Load-the-users-table-from-the-file-ml-100k/u.user.-(3-points)" data-toc-modified-id="Q4.1-Load-the-users-table-from-the-file-ml-100k/u.user.-(3-points)-31"><span class="toc-item-num">31&nbsp;&nbsp;</span>Q4.1 Load the users table from the file <code>ml-100k/u.user</code>. (3 points)</a></div><div class="lev1 toc-item"><a href="#Q4.2-Merge-the-users-table-with-ratings.-Do-not-discard-any-columns.-(3-points)" data-toc-modified-id="Q4.2-Merge-the-users-table-with-ratings.-Do-not-discard-any-columns.-(3-points)-32"><span class="toc-item-num">32&nbsp;&nbsp;</span>Q4.2 Merge the <code>users</code> table with <code>ratings</code>. Do not discard any columns. (3 points)</a></div><div class="lev1 toc-item"><a href="#Q-4.3-How-strict-are-people-by-occupation?-Compute-the-average-of-ratings-by-occupation.-Plot-it-on-a-bar-chart-in-descending-order." data-toc-modified-id="Q-4.3-How-strict-are-people-by-occupation?-Compute-the-average-of-ratings-by-occupation.-Plot-it-on-a-bar-chart-in-descending-order.-33"><span class="toc-item-num">33&nbsp;&nbsp;</span>Q 4.3 How strict are people by occupation? Compute the average of ratings by occupation. Plot it on a bar chart in descending order.</a></div><div class="lev2 toc-item"><a href="#Step-1.-Compute-the-averages-by-occupation.-(2-points)" data-toc-modified-id="Step-1.-Compute-the-averages-by-occupation.-(2-points)-331"><span class="toc-item-num">33.1&nbsp;&nbsp;</span>Step 1. Compute the averages by occupation. (2 points)</a></div><div class="lev2 toc-item"><a href="#Step-2.-Plot-it-on-a-bar-chart.-(2-points)" data-toc-modified-id="Step-2.-Plot-it-on-a-bar-chart.-(2-points)-332"><span class="toc-item-num">33.2&nbsp;&nbsp;</span>Step 2. Plot it on a bar chart. (2 points)</a></div><div class="lev1 toc-item"><a href="#Q4.4*-Plot-the-averages-by-occupation-and-gender-on-a-multiple-bar-plot.-(4-points)" data-toc-modified-id="Q4.4*-Plot-the-averages-by-occupation-and-gender-on-a-multiple-bar-plot.-(4-points)-34"><span class="toc-item-num">34&nbsp;&nbsp;</span>Q4.4* Plot the averages by occupation <em>and</em> gender on a multiple bar plot. (4 points)</a></div><div class="lev1 toc-item"><a href="#Q4.5-How-likely-are-different-age-groups-to-rate-movies?-Compute-the-number-of-ratings-by-age-grouped-into-10-19,-20-29,-etc.-Plot-it-on-a-bar-chart." data-toc-modified-id="Q4.5-How-likely-are-different-age-groups-to-rate-movies?-Compute-the-number-of-ratings-by-age-grouped-into-10-19,-20-29,-etc.-Plot-it-on-a-bar-chart.-35"><span class="toc-item-num">35&nbsp;&nbsp;</span>Q4.5 How likely are different age groups to rate movies? Compute the number of ratings by age grouped into 10-19, 20-29, etc. Plot it on a bar chart.</a></div><div class="lev2 toc-item"><a href="#Step-1.-Number-of-ratings-by-age-group-(3-points)" data-toc-modified-id="Step-1.-Number-of-ratings-by-age-group-(3-points)-351"><span class="toc-item-num">35.1&nbsp;&nbsp;</span>Step 1. Number of ratings by age group (3 points)</a></div><div class="lev2 toc-item"><a href="#Step-2.-Plot-it-on-a-bar-chart.-(2-points)" data-toc-modified-id="Step-2.-Plot-it-on-a-bar-chart.-(2-points)-352"><span class="toc-item-num">35.2&nbsp;&nbsp;</span>Step 2. Plot it on a bar chart. (2 points)</a></div><div class="lev1 toc-item"><a href="#Q4.6-What-hour-of-the-day-do-different-occupations-rate?-(3-points)" data-toc-modified-id="Q4.6-What-hour-of-the-day-do-different-occupations-rate?-(3-points)-36"><span class="toc-item-num">36&nbsp;&nbsp;</span>Q4.6 What hour of the day do different occupations rate? (3 points)</a></div><div class="lev2 toc-item"><a href="#Create-a-function-that-computes-the-number-of-ratings-per-hour-for-a-single-occupation." data-toc-modified-id="Create-a-function-that-computes-the-number-of-ratings-per-hour-for-a-single-occupation.-361"><span class="toc-item-num">36.1&nbsp;&nbsp;</span>Create a function that computes the number of ratings per hour for a single occupation.</a></div><div class="lev1 toc-item"><a href="#Q4.7*-Plot-the-rating-hours-of-marketing-employees-and-programmers-on-two-pie-charts.-(4-points)" data-toc-modified-id="Q4.7*-Plot-the-rating-hours-of-marketing-employees-and-programmers-on-two-pie-charts.-(4-points)-37"><span class="toc-item-num">37&nbsp;&nbsp;</span>Q4.7* Plot the rating hours of marketing employees and programmers on two pie charts. (4 points)</a></div><div class="lev1 toc-item"><a href="#Q4.8*-Do-older-people-prefer-movies-with-longer-titles?-Compute-the-average-title-length-by-age-group-(0-10,-10-20)." data-toc-modified-id="Q4.8*-Do-older-people-prefer-movies-with-longer-titles?-Compute-the-average-title-length-by-age-group-(0-10,-10-20).-38"><span class="toc-item-num">38&nbsp;&nbsp;</span>Q4.8* Do older people prefer movies with longer titles? Compute the average title length by age group (0-10, 10-20).</a></div><div class="lev2 toc-item"><a href="#Step1.-compute-mean-length-(4-points)" data-toc-modified-id="Step1.-compute-mean-length-(4-points)-381"><span class="toc-item-num">38.1&nbsp;&nbsp;</span>Step1. compute mean length (4 points)</a></div><div class="lev2 toc-item"><a href="#Step-2.-Plot-it-on-a-bar-chart.-Choose-a-reasonable-range-for-the-y-axis.-(2-points)" data-toc-modified-id="Step-2.-Plot-it-on-a-bar-chart.-Choose-a-reasonable-range-for-the-y-axis.-(2-points)-382"><span class="toc-item-num">38.2&nbsp;&nbsp;</span>Step 2. Plot it on a bar chart. Choose a reasonable range for the y-axis. (2 points)</a></div><div class="lev1 toc-item"><a href="#Problem-Set-5.-A-simple-recommendation-system" data-toc-modified-id="Problem-Set-5.-A-simple-recommendation-system-39"><span class="toc-item-num">39&nbsp;&nbsp;</span>Problem Set 5. A simple recommendation system</a></div><div class="lev1 toc-item"><a href="#Q5.1.-Extract-genre-information-as-a-matrix.-(2-points)" data-toc-modified-id="Q5.1.-Extract-genre-information-as-a-matrix.-(2-points)-40"><span class="toc-item-num">40&nbsp;&nbsp;</span>Q5.1. Extract genre information as a matrix. (2 points)</a></div><div class="lev1 toc-item"><a href="#Q5.2-Run-the-k-nearest-neighbor-algorithm-on-X.-(3-points)" data-toc-modified-id="Q5.2-Run-the-k-nearest-neighbor-algorithm-on-X.-(3-points)-41"><span class="toc-item-num">41&nbsp;&nbsp;</span>Q5.2 Run the k-nearest neighbor algorithm on X. (3 points)</a></div><div class="lev1 toc-item"><a href="#Q5.3-Increment-by-one-(3-points)" data-toc-modified-id="Q5.3-Increment-by-one-(3-points)-42"><span class="toc-item-num">42&nbsp;&nbsp;</span>Q5.3 Increment by one (3 points)</a></div><div class="lev1 toc-item"><a href="#Q5.4*-Find-the-movies-corresponding-to-these-indices-(5-points)" data-toc-modified-id="Q5.4*-Find-the-movies-corresponding-to-these-indices-(5-points)-43"><span class="toc-item-num">43&nbsp;&nbsp;</span>Q5.4* Find the movies corresponding to these indices (5 points)</a></div><div class="lev1 toc-item"><a href="#Q5.5*-Replace-the-index-of-the-movie-by-its-title.-(2-points)" data-toc-modified-id="Q5.5*-Replace-the-index-of-the-movie-by-its-title.-(2-points)-44"><span class="toc-item-num">44&nbsp;&nbsp;</span>Q5.5* Replace the index of the movie by its title. (2 points)</a></div><div class="lev1 toc-item"><a href="#Q5.6**-Improve-your-recommedation-system-by-adding-other-columns.-(5-points)" data-toc-modified-id="Q5.6**-Improve-your-recommedation-system-by-adding-other-columns.-(5-points)-45"><span class="toc-item-num">45&nbsp;&nbsp;</span>Q5.6** Improve your recommedation system by adding other columns. (5 points)</a></div>

# YOUR NAME (NEPTUN)

# Business Intelligence - Pandas

February 26, 2020

## General information

This goal of this notebook is to give a brief introduction to the `pandas` library, a popular data manipulation and analysis tool for Python.

Problems are numbered from **Q1** to **Q5** with many subproblems such as **Q1.1**. The scores range from 2 to 5 based on the difficulty of the problem. Grades are determined using this table (TODO update):

| score | grade |
| ---- | ----|
| 80+ | 5 |
| 60+ | 4 |
| 40+ | 3 |
| 20+ | 2 |
| 20- | 1 |

Your answer should go in place of `YOUR CODE HERE`. Please remove `raise NotImplementedError`.
Most of the tasks are automatically graded using visible and hidden tests.
Visible tests are available in this version, hidden tests are not available to you.
This means that passing all visible tests does not ensure that your answer is correct.
Not passing the visible tests means that your answer is incorrect.
Do not delete or copy cells and do not edit the source code.
You may add cells but they will not be graded.
You will not find the hidden tests in the source code but you can mess up the autograder if you manually edit it.

**VERY IMPORTANT** Do not edit cell metadata or the raw `.ipynb`. Autograde will fail if you change something carelessly.

Advanced exercises are marked with \*.
More advanced ones have more stars.
Some problems build on each other - it should be obvious how from the code - but advanced problems can safely be skipped.
Completing all non-advanced exercises correctly is worth 63 points.

### Submission

You only need to submit this notebook (no separate report).
Make sure that you save the last version of your notebook.
Please rename the notebook to your neptun code (`ABC123.ipynb`), package it in an archive (`.zip`) and upload it to the class website.
You are free to continue working on this problem set after class but make sure that you upload it by the end of the week (Sunday).

**VERY IMPORTANT** Run Kernel->Restart & Run All and make sure that it finishes without errors.
If you skip exercises, you need to manually run the remaining cells.
Skipping exercises won't affect the autograder.

### Tips

You generally don't need to leave any DataFrames printed as cell outputs. You can do it for debug purposes and it won't affect the autograder but please don't leave long tables in the output. Use `.head()` instead.

Be concise. All exercises can be solved with less than 5 lines of code.

Avoid for loops. Almost all tasks can be solved with efficient pandas operations.

Avoid overriding Python built-in functions with your own variables (`max = 2`).

If you mess up, you can always do one of the following
  1. Kernel -> Restart & Run All - this will run all cells from top to bottom until an exception is thrown
  1. Kernel -> Restart, Cell -> Run All Above - this will run all cells from top to bottom until the current cell is reached or and exception is thrown

If your notebook runs for longer than a minute, one or more of your solutions is very inefficient.

### Credits

This assignment was created by Judit Ács using [nbgrader](https://nbgrader.readthedocs.io/).

### Feedback

Please fill out this [short survey](https://forms.gle/i717e8Mzijjy3Pfk9) after you completed the problems.

## Code quality

You can get 3 extra points for code quality.

## PEP8 style guide

You can get 2 extra points for adhering to the PEP8 Python style guide.

## Figure quality

You can get 5 extra points for the quality of your figures. Good figures have labeled axes with meaningful names, reasonable figure size and reasonable axes limits.
Extra attention to details also helps.

# Main imports
End of explanation
"""


import os

data_dir = os.getenv("MOVIELENS")
if data_dir is None:
    data_dir = ""

ml_path = os.path.join(data_dir, "ml.zip")

if os.path.exists(ml_path):
    print("File already exists, skipping download step.")
else:
    print("Downloading the Movielens dataset")
    import urllib
    u = urllib.request.URLopener()
    u.retrieve("http://files.grouplens.org/datasets/movielens/ml-100k.zip", ml_path)

unzip_path = os.path.join(data_dir, "ml-100k")

if os.path.exists(unzip_path):
    print("Dataset already unpacked, skipping unpacking step.")
else:
    print("Unziping the dataset.")
    from zipfile import ZipFile
    with ZipFile(ml_path) as myzip:
        myzip.extractall(data_dir)
        
data_dir = unzip_path

"""
Explanation: downloading the dataset
End of explanation
"""


# df = pd.read_table("ml-100k/u.item")  # UnicodeDecodeErrort kapunk, mert rossz dekódert használ
df = pd.read_table(os.path.join(data_dir, "u.item"), encoding="latin1")
df.head()

"""
Explanation: Loading the dataset
pd.read_table loads a tabular dataset. The full function signature is:
~~~
pandas.read_table(filepath_or_buffer: Union[str, pathlib.Path, IO[~AnyStr]], sep='t', delimiter=None, header='infer', names=None, index_col=None, usecols=None, squeeze=False, prefix=None, mangle_dupe_cols=True, dtype=None, engine=None, converters=None, true_values=None, false_values=None, skipinitialspace=False, skiprows=None, skipfooter=0, nrows=None, na_values=None, keep_default_na=True, na_filter=True, verbose=False, skip_blank_lines=True, parse_dates=False, infer_datetime_format=False, keep_date_col=False, date_parser=None, dayfirst=False, cache_dates=True, iterator=False, chunksize=None, compression='infer', thousands=None, decimal: str = '.', lineterminator=None, quotechar='"', quoting=0, doublequote=True, escapechar=None, comment=None, encoding=None, dialect=None, error_bad_lines=True, warn_bad_lines=True, delim_whitespace=False, low_memory=True, memory_map=False, float_precision=None
~~~
let's try it with defaults
End of explanation
"""


column_names = [
    "movie_id", "title", "release_date", "video_release_date", "imdb_url", "unknown", "action", "adventure", "animation",
    "children", "comedy", "crime", "documentary", "drama", "fantasy", "film_noir", "horror", "musical", "mystery",
    "romance", "sci_fi", "thriller", "war", "western"]
df = pd.read_table(
    os.path.join(data_dir, "u.item"), sep="|",
    names=column_names, encoding="latin1", index_col='movie_id',
    parse_dates=['release_date', 'video_release_date']
)
df.head()

"""
Explanation: A couple improvements:

Use a different separator. | instead of \t.
The first line of the file is used as the header. The real names of the columns are listed in the README, they can be specified with the names parameters.
read_table added an index (0..N-1), but the dataset already has an index, let's use that one (index_col='movie_id')
two columns, release_date and video_release_date are dates, pandas can parse them and create its own datetype.
End of explanation
"""


print(", ".join([d for d in dir(df.release_date.dt) if not d.startswith('_')]))

"""
Explanation: Normalizing the dataset
the .dt and the .str namespaces
The year in the title seems redundant, let's check if it's always the same as the release date. The .dt namespace has various methods and attributes for handling dates, and the .str namespace has string handling methods.
.dt namespace
End of explanation
"""


print(", ".join([d for d in dir(df.title.str) if not d.startswith('_')]))

"""
Explanation: .str namespace
End of explanation
"""


df['year'] = df.release_date.dt.year

"""
Explanation: Let's extract the release year of a movie into a separate
End of explanation
"""


df.year.value_counts().head()

"""
Explanation: the most common years are
End of explanation
"""


df.video_release_date.isnull().value_counts()

df = df.drop('video_release_date', axis=1)

"""
Explanation: video_release_date is always NaT (not a time), let's drop it
End of explanation
"""


df.describe()

"""
Explanation: Basic analysis of the dataset
describe generate descriptive statistics.
End of explanation
"""


df.release_date.describe()

"""
Explanation: Only numeric columns are included by default. A single column (pd.Series) has a describe function too
End of explanation
"""


df.mean()

"""
Explanation: Numberic statistics are available as separate functions too:

count: number of non-NA cells. NA is NOT the same as 0
mean: average
std: standard deviation
var: variance
min, max

etc.
End of explanation
"""


df[df.release_date.dt.year==1956]

"""
Explanation: Basic queries
Which movies were released in 1956?
End of explanation
"""


d = df[(df.release_date.dt.year >=  1980) & (df.release_date.dt.year < 1990)]
print(f"{len(d)} movies were released in the 80s.")
print("\nA few examples:")
print("\n".join(d.sample(5).title))

"""
Explanation: How many movies were released in the 80s?
Let's print 5 examples too.
End of explanation
"""


df[df.title.str.contains('Die Hard')]

"""
Explanation: When were the Die Hard movies released?
End of explanation
"""


df.release_date.min(), df.release_date.max()

"""
Explanation: Die Hard 4 and 5 are missing. This is because the dataset only contains movies released between:
End of explanation
"""


print("Action and romance:", len(df[(df.action==1) & (df.romance==1)]))
print("Action or romance:", len(df[(df.action==1) | (df.romance==1)]))

"""
Explanation: and Die Hard 4 and 5 were released in 2007 and 2013 respectively.
How many movies are both action and romance? What about action or romance?
Make sure you parenthesize the conditions
End of explanation
"""


def count_movies_before_1985(df):
    # YOUR CODE HERE
    raise NotImplementedError()
    
def count_movies_after_1984(df):
    # YOUR CODE HERE
    raise NotImplementedError()

before = count_movies_before_1985(df)
assert isinstance(before, int) or isinstance(before, np.integer)
# action movies only, not all!
assert before != 272

after = count_movies_after_1984(df)
assert isinstance(after, int) or isinstance(after, np.integer)

"""
Explanation: Problem Set 1: simple queries
Q1.1 How many action movies were release before 1985 and in 1985 or later? (2 points)
End of explanation
"""


def child_thriller(df):
    # YOUR CODE HERE
    raise NotImplementedError()

title = child_thriller(df)
assert isinstance(title, str)

"""
Explanation: Q1.2 Are there thrillers for children? Find an example. (2 points)
End of explanation
"""


def long_titles(df):
    # YOUR CODE HERE
    raise NotImplementedError()

title_cnt = long_titles(df)
assert isinstance(title_cnt, int) or isinstance(title_cnt, np.integer)

"""
Explanation: Q1.3 How many movies have title longer than 40 characters? (3 points)
End of explanation
"""


def number_of_words(field):
    return len(field.split(" "))

df.title.apply(number_of_words).value_counts().sort_index()
# or
# df.title.apply(lambda t: len(t.split(" "))).value_counts().sort_index()

df.title.apply(number_of_words).value_counts().sort_index().plot(kind='bar')

"""
Explanation: pd.Series.apply: running arbitrary functions on each element
The apply function allow running arbitrary functions on pd.Series:
End of explanation
"""


#df['title_word_cnt'] = ...
def count_content_words(title):
    # YOUR CODE HERE
    raise NotImplementedError()

df['title_word_cnt'] = df.title.apply(count_content_words)
assert 'title_word_cnt' in df.columns
assert df['title_word_cnt'].dtype == int
assert df.loc[1424, 'title_word_cnt'] == 5
assert df.loc[1170, 'title_word_cnt'] == 2

df.title_word_cnt.value_counts().sort_index().plot(kind='bar')

"""
Explanation: Q1.4* How many content words does the average title have? Which title has the most/least words?
Content words are capitalized. The opposite of content words are function words (and, a, an, the, more etc.). 
We should not include the release year in the word count.
Step 1. Count words in the title (2 points)
End of explanation
"""


#shortest_title = ...
#longest_title = ...
#shortest_title_len = ...
#longest_title_len = ...
# YOUR CODE HERE
raise NotImplementedError()

assert isinstance(shortest_title, str)
assert isinstance(longest_title, str)
assert isinstance(shortest_title_len, np.int64)
assert isinstance(longest_title_len, np.int64)
assert shortest_title_len == 0
assert longest_title == 'Englishman Who Went Up a Hill, But Came Down a Mountain, The (1995)'
assert shortest_title == 'unknown'
assert longest_title_len == 10

"""
Explanation: Step 2. Shortest and longest titles by word count (3 points)
End of explanation
"""


def movies_with_word(df, word):
    # YOUR CODE HERE
    raise NotImplementedError()

and_movies = movies_with_word(df, "and")
assert isinstance(and_movies, pd.DataFrame)
assert and_movies.shape == (66, 24)
assert 'Mr. Holland\'s Opus (1995)' not in and_movies.title.values

"""
Explanation: Q1.5* How many movies have the word 'and' in their title? Write a function that counts movies with a particular word. (3 points)
Disregard case and avoid matching subwords for examples. For example 'and' should not match 'Holland' nor should it match the movie 'Andrew'.
End of explanation
"""


df.groupby('year').size().plot()

"""
Explanation: Groupby and visualization
How many movies are released each year?
End of explanation
"""


df.year.value_counts().sort_index().plot()

"""
Explanation: another option is the use pd.Series.value_counts
End of explanation
"""


fig, ax = plt.subplots(1, figsize=(10, 6))
d = df[df.year>1985]
d.groupby('year').size().plot(kind='bar', ax=ax)

"""
Explanation: most movies were released in the late 80s and 90s, let's zoom in. Let's also change the figure size.
We create the plot object with one subplot, we then specify which axis pandas should use for plotting (ax=ax).
End of explanation
"""


df.groupby(['action', 'romance']).size()

"""
Explanation: we can groupby on multiple columns
Back to the romance-action combinations
End of explanation
"""


df.groupby(df.year // 10 * 10).size()

"""
Explanation: we can also group on arbitrary conditions
how many movies were released each decade?
End of explanation
"""


def group_genre_by_year(df, genre):
    # YOUR CODE HERE
    raise NotImplementedError()

crime = group_genre_by_year(df, 'crime')
assert type(crime) == pd.core.groupby.DataFrameGroupBy
assert len(crime) <= 15  # movies between 1985-1999

"""
Explanation: Problem Set 2: Groupby and visualization
Q2.1 Write a function that takes a genre and groups movies of that genre by year. Do not include movies older than 1985.  (3 points)
End of explanation
"""


# YOUR CODE HERE
raise NotImplementedError()

"""
Explanation: Q2.2 Plot the number of adventure movies from 1985 to 1999 on a bar chart. Use your group_genre_by_year function. (2 points)
End of explanation
"""


def groupby_release_day(df):
    # YOUR CODE HERE
    raise NotImplementedError()

by_day = groupby_release_day(df)
assert type(by_day) == pd.core.groupby.DataFrameGroupBy

# the longest month is 31 days
assert len(by_day) < 32

# shouldn't group on day of week
assert len(by_day) > 7

"""
Explanation: Q2.3 Plot the distribution of release day (day of month) on a pie chart.
Step 1. groupby (2 points)
End of explanation
"""


# YOUR CODE HERE
raise NotImplementedError()

"""
Explanation: Step 2. pie chart. Add percent values. (3 points)
End of explanation
"""


def compute_initial_letter_frequencies(df):
    # YOUR CODE HERE
    raise NotImplementedError()

initial = compute_initial_letter_frequencies(df)

assert type(initial) == pd.Series

# frequency counts should be >= 1
assert initial.min() >= 1

# the largest one cannot be larger than the full dataframe
assert initial.max() <= len(df)

"""
Explanation: Q2.4 We're building a traditional lexicon of the titles. What is the distribution of initial letters (i.e. how many titles start with S?)? Plot it on a bar chart.
Step 1. Compute frequencies. (3 points)
End of explanation
"""


# YOUR CODE HERE
raise NotImplementedError()

"""
Explanation: Step 2. Plot it on a bar chart in descending order. (3 points)
The most common letter should be the first bar.
End of explanation
"""


cols = ['user', 'movie_id', 'rating', 'timestamp']
ratings = pd.read_table(os.path.join(data_dir, "u.data"), names=cols)
ratings.head()

"""
Explanation: Problem Set 3. Handling multiple dataframes
The main table of this dataset is u.data with 100000 ratings.
End of explanation
"""


ratings['timestamp'] = pd.to_datetime(ratings.timestamp, unit='s')
ratings.head()

"""
Explanation: The timestamp column is a Unix timestamp, let's convert it to pd.DateTime:
End of explanation
"""


movies = df
ratings = pd.merge(ratings, movies, left_on='movie_id', right_index=True)

"""
Explanation: Merging with the movies dataframe
We overwrite ratings:
End of explanation
"""


(ratings.timestamp <= ratings.release_date).value_counts()

"""
Explanation: How many ratings are timestamped before the release date?
End of explanation
"""


ratings[ratings.timestamp <= ratings.release_date].title.value_counts()

"""
Explanation: Which movies were rated before the release date?
End of explanation
"""


def count_greater_than_4(df):
    # YOUR CODE HERE
    raise NotImplementedError()

greater = count_greater_than_4(ratings)

assert isinstance(greater, np.integer) or isinstance(greater, int)

"""
Explanation: Q3.1. How many movies were rated 4+ at least once? (3 points)
End of explanation
"""


def compute_movie_rated_frequencies(df):
    # YOUR CODE HERE
    raise NotImplementedError()

title_freq = compute_movie_rated_frequencies(ratings)
assert isinstance(title_freq, pd.Series)

# use titles
assert 'Lone Star (1996)' in title_freq.index

"""
Explanation: Q3.2. How many time was each movie rated?
Step 1. Compute the frequencies of movie ratings. Use movie titles instead of ids. (2 points)
End of explanation
"""


# YOUR CODE HERE
raise NotImplementedError()

"""
Explanation: Step 2. Plot the frequencies on a histogram. (2 points)
pd.Series has a hist function. It uses 10 bins by default, use more.
End of explanation
"""


def groupby_day_of_week(ratings):
    # YOUR CODE HERE
    raise NotImplementedError()

by_day = groupby_day_of_week(ratings)
assert isinstance(by_day, pd.core.groupby.generic.DataFrameGroupBy)
# there are 7 days
assert len(by_day) == 7
# use names of the days
assert 'Monday' in by_day.groups

"""
Explanation: Q3.3. How many ratings were submitted by each day of the week? What were their averages?
Step 1. groupby (3 points)
Tip: look around in the .dt namespace.
End of explanation
"""


#number_of_ratings_per_day = ...
# YOUR CODE HERE
raise NotImplementedError()

assert isinstance(number_of_ratings_per_day, pd.Series)
assert len(number_of_ratings_per_day) == 7
assert number_of_ratings_per_day.min() > 10000

"""
Explanation: Step 2. number of ratings per day (2 points)
End of explanation
"""


#mean_rating_by_day = ...
# YOUR CODE HERE
raise NotImplementedError()

assert isinstance(mean_rating_by_day, pd.Series)

"""
Explanation: Step 3. mean rating by day (2 points)
End of explanation
"""


genres = ['unknown', 'action', 'adventure', 'animation',
       'children', 'comedy', 'crime', 'documentary', 'drama', 'fantasy',
       'film_noir', 'horror', 'musical', 'mystery', 'romance', 'sci_fi',
       'thriller', 'war', 'western']

def compute_mean_rating_by_genre(ratings):
    # YOUR CODE HERE
    raise NotImplementedError()

genre_rating = compute_mean_rating_by_genre(ratings)
assert len(genre_rating) == len(genres)
# all means are between 3 and 4
assert genre_rating.min() > 3.0
assert genre_rating.max() < 4.0
# film noir is rated highest
assert genre_rating.idxmax() == 'film_noir'
for g in genres:
    assert g in genre_rating.index

"""
Explanation: Q3.4** What is the mean of ratings by genre?
If a movie has multiple genres, include it in every genre. 
Step 1. Compute the mean scores. (5 points)
There are many ways to solve this problem. Try to do it without explicit for loops.
End of explanation
"""


# YOUR CODE HERE
raise NotImplementedError()

"""
Explanation: Step 2. Plot it on a bar chart in descending order by score. Set the limits of the y-axis to (2.5, 4). (3 points)
End of explanation
"""


# users = ...
# YOUR CODE HERE
raise NotImplementedError()

assert type(users) == pd.DataFrame

# users are indexed from 0
assert 0 not in users.index

"""
Explanation: Problem Set 4. User demographics
Q4.1 Load the users table from the file ml-100k/u.user. (3 points)
u.users has the following columns: user_id, age, gender, occupation, zip. Use user_id as the index.
End of explanation
"""


# ratings = ratings.merge...
# YOUR CODE HERE
raise NotImplementedError()

assert type(ratings) == pd.DataFrame
# all movies have ratings (nunique return the number of unique elements)
assert ratings.movie_id.nunique() == 1682

"""
Explanation: Q4.2 Merge the users table with ratings. Do not discard any columns. (3 points)
End of explanation
"""


def compute_mean_by_occupation(ratings):
    # YOUR CODE HERE
    raise NotImplementedError()

mean_by_occupation = compute_mean_by_occupation(ratings)
assert isinstance(mean_by_occupation, pd.Series)
# ratings are between 1 and 5
assert mean_by_occupation.min() > 1
assert mean_by_occupation.max() < 5

"""
Explanation: Q 4.3 How strict are people by occupation? Compute the average of ratings by occupation. Plot it on a bar chart in descending order.
Step 1. Compute the averages by occupation. (2 points)
End of explanation
"""


# YOUR CODE HERE
raise NotImplementedError()

"""
Explanation: Step 2. Plot it on a bar chart. (2 points)
Extra point: make the bar chart wider and restring the y-axis to (2.5, 4).
End of explanation
"""


# YOUR CODE HERE
raise NotImplementedError()

"""
Explanation: Q4.4* Plot the averages by occupation and gender on a multiple bar plot. (4 points)
Tip: there is an example of a multiple bar plot here
Tip 2: there are many ways to solve this problem, one is a one-liner using DataFrame.unstack. It's a little longer if you make the figure nicer.
End of explanation
"""


def count_ratings_by_age_group(ratings):
    # YOUR CODE HERE
    raise NotImplementedError()

rating_by_age_group = count_ratings_by_age_group(ratings)

assert isinstance(rating_by_age_group, pd.Series)
assert 20 in rating_by_age_group
assert 25 not in rating_by_age_group

"""
Explanation: Q4.5 How likely are different age groups to rate movies? Compute the number of ratings by age grouped into 10-19, 20-29, etc. Plot it on a bar chart.
Step 1. Number of ratings by age group (3 points)
End of explanation
"""


# YOUR CODE HERE
raise NotImplementedError()

"""
Explanation: Step 2. Plot it on a bar chart. (2 points)
End of explanation
"""


def count_rating_by_hour_occupation(ratings, occupation):
    # YOUR CODE HERE
    raise NotImplementedError()

marketing = count_rating_by_hour_occupation(ratings, "marketing")
assert isinstance(marketing, pd.Series)

# there are only 24 hours
assert len(marketing) < 25

"""
Explanation: Q4.6 What hour of the day do different occupations rate? (3 points)
Create a function that computes the number of ratings per hour for a single occupation.
End of explanation
"""


fig, ax = plt.subplots(1, 2, figsize=(12, 6))
# YOUR CODE HERE
raise NotImplementedError()

"""
Explanation: Q4.7* Plot the rating hours of marketing employees and programmers on two pie charts. (4 points)
A two-subplot figure is created. ax is an array of the two subplots, use ax[0] for marketing employees and ax[1] for programmers. Set the titles of the subplots accordingly.
End of explanation
"""


def get_mean_length_by_age_group(ratings):
    # YOUR CODE HERE
    raise NotImplementedError()

title_len_by_age = get_mean_length_by_age_group(ratings)
assert isinstance(title_len_by_age, pd.Series)
assert len(title_len_by_age) == 8
# titles are long
assert title_len_by_age.min() >= 20

"""
Explanation: Q4.8* Do older people prefer movies with longer titles? Compute the average title length by age group (0-10, 10-20).
Step1. compute mean length (4 points)
Tip: You should probably create a copy of some of the columns.
End of explanation
"""


# YOUR CODE HERE
raise NotImplementedError()

"""
Explanation: Step 2. Plot it on a bar chart. Choose a reasonable range for the y-axis. (2 points)
End of explanation
"""


#X = ...
# YOUR CODE HERE
raise NotImplementedError()

assert isinstance(X, np.ndarray)
assert X.shape[0] == len(movies)

"""
Explanation: Problem Set 5. A simple recommendation system
Let's build a simple recommendation system that finds similar movies based on genre.
Q5.1. Extract genre information as a matrix. (2 points)
The .values attribute represents the underlying values as a Numpy ndarray.
End of explanation
"""


from sklearn.neighbors import NearestNeighbors
# YOUR CODE HERE
raise NotImplementedError()

assert run_knn(X, 2).shape == (1682, 2)
K = 4
indices = run_knn(X, K)
assert isinstance(indices, np.ndarray)
assert indices.shape[1] == K

"""
Explanation: Q5.2 Run the k-nearest neighbor algorithm on X. (3 points)
Find a usage example in the documentation of NearestNeighbors.
Store the indices in a variable names indices.
K is the number of nearest neighbors. It should be a parameter of your function.
End of explanation
"""


ind = pd.DataFrame(indices)
ind.head()

"""
Explanation: indices is more convenient as a DataFrame
End of explanation
"""


def increment_table(df):
    # YOUR CODE HERE
    raise NotImplementedError()

indices = increment_table(ind)
assert indices.shape[1] == 4
assert indices.index[0] == 1
assert indices.index[-1] == len(indices)

indices.head()

"""
Explanation: Q5.3 Increment by one (3 points)
The index of this DataFrame refers to a particular movie and the rest of the rows are the indices of similar movies. The problem is that this matrix is zero-indexed, while the dataset (movies table) is indexed from 1.
Both the index and all values should be increased by one.
End of explanation
"""


def find_neighbor_titles(movies, indices):
    # YOUR CODE HERE
    raise NotImplementedError()

neighbors = find_neighbor_titles(movies, indices)
assert isinstance(neighbors, pd.DataFrame)
assert neighbors.shape[1] == K

neighbors.head()

"""
Explanation: Q5.4* Find the movies corresponding to these indices (5 points)
You'll need multiple merge operations.
Tip: the names of the columns in indices are not strings but integers, you can rename the columns of a dataframe:
~~~
df = df.rename(columns={'old': 'new', 'other old': 'other new'})
~~~
You can discard all other columns.
End of explanation
"""


def recover_titles(movies, neighbors):
    # YOUR CODE HERE
    raise NotImplementedError()

most_similar = recover_titles(movies, neighbors)

assert type(most_similar) == pd.DataFrame
assert "Toy Story (1995)" in most_similar.index

"""
Explanation: Q5.5* Replace the index of the movie by its title. (2 points)
End of explanation
"""


# YOUR CODE HERE
raise NotImplementedError()

"""
Explanation: Q5.6** Improve your recommedation system by adding other columns. (5 points)
Tips: you can add the average rating of a movie by occupation/age group/gender
Please fit your solution in one cell.
End of explanation
"""


