# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# + [markdown] papermill={"duration": 0.018849, "end_time": "2020-11-25T17:42:25.365369", "exception": false, "start_time": "2020-11-25T17:42:25.346520", "status": "completed"} tags=[]
# **This notebook is an exercise in the [Data Cleaning](https://www.kaggle.com/learn/data-cleaning) course.  You can reference the tutorial at [this link](https://www.kaggle.com/alexisbcook/inconsistent-data-entry).**
#
# ---
#

# + [markdown] papermill={"duration": 0.015082, "end_time": "2020-11-25T17:42:25.396013", "exception": false, "start_time": "2020-11-25T17:42:25.380931", "status": "completed"} tags=[]
# In this exercise, you'll apply what you learned in the **Inconsistent data entry** tutorial.
#
# # Setup
#
# The questions below will give you feedback on your work. Run the following cell to set up the feedback system.

# + papermill={"duration": 0.130998, "end_time": "2020-11-25T17:42:25.542254", "exception": false, "start_time": "2020-11-25T17:42:25.411256", "status": "completed"} tags=[]
from learntools.core import binder
binder.bind(globals())
from learntools.data_cleaning.ex5 import *
print("Setup Complete")

# + [markdown] papermill={"duration": 0.01555, "end_time": "2020-11-25T17:42:25.575006", "exception": false, "start_time": "2020-11-25T17:42:25.559456", "status": "completed"} tags=[]
# # Get our environment set up
#
# The first thing we'll need to do is load in the libraries and dataset we'll be using.  We use the same dataset from the tutorial.

# + papermill={"duration": 0.036583, "end_time": "2020-11-25T17:42:25.627455", "exception": false, "start_time": "2020-11-25T17:42:25.590872", "status": "completed"} tags=[]
# modules we'll use
import pandas as pd
import numpy as np

# helpful modules
import fuzzywuzzy
from fuzzywuzzy import process
import chardet

# read in all our data
professors = pd.read_csv("../input/pakistan-intellectual-capital/pakistan_intellectual_capital.csv")

# set seed for reproducibility
np.random.seed(0)

# + [markdown] papermill={"duration": 0.015512, "end_time": "2020-11-25T17:42:25.658975", "exception": false, "start_time": "2020-11-25T17:42:25.643463", "status": "completed"} tags=[]
# Next, we'll redo all of the work that we did in the tutorial.

# + papermill={"duration": 0.039768, "end_time": "2020-11-25T17:42:25.714598", "exception": false, "start_time": "2020-11-25T17:42:25.674830", "status": "completed"} tags=[]
# convert to lower case
professors['Country'] = professors['Country'].str.lower()
# remove trailing white spaces
professors['Country'] = professors['Country'].str.strip()

# get the top 10 closest matches to "south korea"
countries = professors['Country'].unique()
matches = fuzzywuzzy.process.extract("south korea", countries, limit=10, scorer=fuzzywuzzy.fuzz.token_sort_ratio)

def replace_matches_in_column(df, column, string_to_match, min_ratio = 47):
    # get a list of unique strings
    strings = df[column].unique()
    
    # get the top 10 closest matches to our input string
    matches = fuzzywuzzy.process.extract(string_to_match, strings, 
                                         limit=10, scorer=fuzzywuzzy.fuzz.token_sort_ratio)

    # only get matches with a ratio > 90
    close_matches = [matches[0] for matches in matches if matches[1] >= min_ratio]

    # get the rows of all the close matches in our dataframe
    rows_with_matches = df[column].isin(close_matches)

    # replace all rows with close matches with the input matches 
    df.loc[rows_with_matches, column] = string_to_match
    
    # let us know the function's done
    print("All done!")
    
replace_matches_in_column(df=professors, column='Country', string_to_match="south korea")
countries = professors['Country'].unique()

# + [markdown] papermill={"duration": 0.016291, "end_time": "2020-11-25T17:42:25.747494", "exception": false, "start_time": "2020-11-25T17:42:25.731203", "status": "completed"} tags=[]
# # 1) Examine another column
#
# Write code below to take a look at all the unique values in the "Graduated from" column.

# + papermill={"duration": 0.024416, "end_time": "2020-11-25T17:42:25.788479", "exception": false, "start_time": "2020-11-25T17:42:25.764063", "status": "completed"} tags=[]
# TODO: Your code here

# + [markdown] papermill={"duration": 0.016296, "end_time": "2020-11-25T17:42:25.821637", "exception": false, "start_time": "2020-11-25T17:42:25.805341", "status": "completed"} tags=[]
# Do you notice any inconsistencies in the data?  Can any of the inconsistencies in the data be fixed by removing white spaces at the beginning and end of cells?
#
# Once you have answered these questions, run the code cell below to get credit for your work.

# + papermill={"duration": 0.029941, "end_time": "2020-11-25T17:42:25.868249", "exception": false, "start_time": "2020-11-25T17:42:25.838308", "status": "completed"} tags=[]
# Check your answer (Run this code cell to receive credit!)
q1.check()

# + papermill={"duration": 0.025266, "end_time": "2020-11-25T17:42:25.911672", "exception": false, "start_time": "2020-11-25T17:42:25.886406", "status": "completed"} tags=[]
# Line below will give you a hint
#q1.hint()

# + [markdown] papermill={"duration": 0.018189, "end_time": "2020-11-25T17:42:25.948671", "exception": false, "start_time": "2020-11-25T17:42:25.930482", "status": "completed"} tags=[]
# # 2) Do some text pre-processing
#
# Convert every entry in the "Graduated from" column in the `professors` DataFrame to remove white spaces at the beginning and end of cells.

# + papermill={"duration": 0.031389, "end_time": "2020-11-25T17:42:25.998670", "exception": false, "start_time": "2020-11-25T17:42:25.967281", "status": "completed"} tags=[]
# TODO: Your code here
____

# Check your answer
q2.check()

# + papermill={"duration": 0.030521, "end_time": "2020-11-25T17:42:26.049652", "exception": false, "start_time": "2020-11-25T17:42:26.019131", "status": "completed"} tags=[]
# Lines below will give you a hint or solution code
#q2.hint()
#q2.solution()

# + [markdown] papermill={"duration": 0.019878, "end_time": "2020-11-25T17:42:26.090012", "exception": false, "start_time": "2020-11-25T17:42:26.070134", "status": "completed"} tags=[]
# # 3) Continue working with countries
#
# In the tutorial, we focused on cleaning up inconsistencies in the "Country" column.  Run the code cell below to view the list of unique values that we ended with.

# + papermill={"duration": 0.031323, "end_time": "2020-11-25T17:42:26.141457", "exception": false, "start_time": "2020-11-25T17:42:26.110134", "status": "completed"} tags=[]
# get all the unique values in the 'City' column
countries = professors['Country'].unique()

# sort them alphabetically and then take a closer look
countries.sort()
countries

# + [markdown] papermill={"duration": 0.020166, "end_time": "2020-11-25T17:42:26.182465", "exception": false, "start_time": "2020-11-25T17:42:26.162299", "status": "completed"} tags=[]
# Take another look at the "Country" column and see if there's any more data cleaning we need to do.
#
# It looks like 'usa' and 'usofa' should be the same country.  Correct the "Country" column in the dataframe so that 'usofa' appears instead as 'usa'.
#
# **Use the most recent version of the DataFrame (with the whitespaces at the beginning and end of cells removed) from question 2.**

# + papermill={"duration": 0.033996, "end_time": "2020-11-25T17:42:26.236985", "exception": false, "start_time": "2020-11-25T17:42:26.202989", "status": "completed"} tags=[]
# TODO: Your code here!
____

# Check your answer
q3.check()

# + papermill={"duration": 0.029375, "end_time": "2020-11-25T17:42:26.288656", "exception": false, "start_time": "2020-11-25T17:42:26.259281", "status": "completed"} tags=[]
# Lines below will give you a hint or solution code
#q3.hint()
#q3.solution()

# + [markdown] papermill={"duration": 0.021979, "end_time": "2020-11-25T17:42:26.333104", "exception": false, "start_time": "2020-11-25T17:42:26.311125", "status": "completed"} tags=[]
# # Congratulations!
#
# Congratulations for completing the **Data Cleaning** course on Kaggle Learn!
#
# To practice your new skills, you're encouraged to download and investigate some of [Kaggle's Datasets](https://www.kaggle.com/datasets).

# + [markdown] papermill={"duration": 0.022617, "end_time": "2020-11-25T17:42:26.378177", "exception": false, "start_time": "2020-11-25T17:42:26.355560", "status": "completed"} tags=[]
# ---
#
#
#
#
# *Have questions or comments? Visit the [Learn Discussion forum](https://www.kaggle.com/learn-forum/172650) to chat with other Learners.*

StarcoderdataJupyterScriptsDedupFiltered