# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# ## Data Cleansing
# Created by: <NAME>

# ### Load library and data set

# +
# load pandas library
import pandas as pd

# load csv file
url = 'http://bit.ly/2kJwy5N'
imdb = pd.read_csv(url)
# -

imdb.head()

imdb.shape

# ### Check existing NaN values

# +
#imdb.isna().head()
# -

# ### Count NaN values

imdb.isna().sum()

imdb.isna().sum().sum()

# ### Drop records with NaN values

clean_imdb1 = imdb.dropna(how='any', axis=0)
#clean_imdb1 = imdb.dropna(how='all')

print('records number before dropping NaN values: {}'.format(imdb.shape[0]))
print('records number after dropping NaN values: {}'.format(clean_imdb1.shape[0]))

clean_imdb1.isna().sum().sum()

# ### Drop records with all values is NaN

clean_imdb2 = imdb.dropna(how='all')

print('records number before dropping NaN values: {}'.format(imdb.shape[0]))
print('records number after dropping NaN values: {}'.format(clean_imdb2.shape[0]))

# ### Replace NaN values by 5

clean_imdb3 = imdb.fillna(5)

clean_imdb3.head()

# ### Replace NaN values by median 

clean_imdb4 = imdb.fillna(imdb.median())

clean_imdb4.head()

# interpolation -> time series
# 5,4,3,5,4, NaN=(4+5)/2 ,5,...
imdb_2 = imdb.copy()
imdb_2.interpolate(method ='linear', inplace=True)
imdb_2.head()

# ### Check the existing of duplicate data

duplicated_imdb = imdb[imdb.duplicated()]

duplicated_imdb.shape

duplicated_imdb.head()

imdb[imdb['director_name'] == '<NAME>']

# ### Drop duplicate data

drop_dup_imdb = imdb.drop_duplicates()

print('records number before dropping duplicate data: {}'.format(imdb.shape[0]))
print('records number after dropping duplicate data: {}'.format(drop_dup_imdb.shape[0]))



StarcoderdataJupyterScriptsDedupFiltered