"""
In this kernel we use FİFA19 dataset yo explore python data science toolbox and data cleaning. This kernel has been prepared for only learning and practice purposes
"""
# This Python 3 environment comes with many helpful analytics libraries installed
# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python
# For example, here's several helpful packages to load in 

import numpy as np # linear algebra
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)
import matplotlib.pyplot as plt
import seaborn as sns

# Input data files are available in the "../input/" directory.
# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory

import os
print(os.listdir("../input"))

# Any results you write to the current directory are saved as output.
data = pd.read_csv("../input/data.csv")
data.info()
data.shape
data.head()
data.tail()
"""
USER DEFINED FUNCTION
"""
def tuple_overall():
    '''print top 5 players' overall'''
    oa = tuple(data[:5]['Overall'])
    return oa
print(tuple_overall())
"""
SCOPE
"""
name = 'Messi'
def player():
    name = 'Neymar'
    return name
print(name)    # global scope
print(player())    # local scope

# what if there is no local scope
x = 5
def f():
    y = 2*x  # there is no local scope for x
    return y
print(f())
#First local scopesearched, then global scope searched, 
#if two of them cannot be found lastly built in scope searched.
# how to learn builtins
import builtins
dir(builtins)
"""
NESTED FUNCTION
"""
def average_age():
    '''return the average of top 100 players'''
    def total_age():
        '''return the total age of top 100 players'''
        t = sum(data[:100]['Age'])
        return t
    return total_age()/100
print(average_age())
DEFAULT AND FLEXIBLE ARGUMENTS
# Default Arguments
def f(a, b=1, c=2):
    y = a + b + c
    return y
print(f(5))
# what if we change default arguments
print(f(5,10,5))
# flexible arguments
def f(*args):
    for i in args:
        print(i)
f(1)
print('')
f(5,6,7,8)

#flexible arguments for dictionary
def f(**kwargs):
    '''print key and value for dictionary'''
    for key,value in kwargs.items():
        print(key, ' ', value)
f(Name = 'Messi', nationality = 'Argentine', overall = 94 )
        
"""
LAMBDA FUNCTION
"""
tot = lambda x,y,z : x + y + z
print(tot(5,6,8))
"""
ANONYMOUS FUNCTİON

Like lambda function but it can take more than one arguments.

map(func,seq) : applies a function to all the items in a list


"""
number_list = [3,4,5]
y = map(lambda x:x**3, number_list)
print(list(y))
ITERATORS
name = "pele"
it = iter(name)
print(next(it))    #print next iteration
print(*it)    # print remaining iteration

ZIP - UNZIP
list1 = data[0:5]['Name']
list2 = data[0:5]['Overall']
z = zip(list1,list2)
z_list = list(z)
print(z_list)
unzip = zip(*z_list)
unlist1, unlist2 = list(unzip)    # unlist return tuble
print(unlist1)
print(unlist2)
print(type(unlist2))
"""
LIST COMPREHENSION
"""
# example of list comprehension
p = data[:5]['Potential']
miss = [100-i for i in p]
print(miss)
# conditionals on iterable
age1 = data[:10]['Age']
age2 = ['young' if i<26 else 'middle' if 30>i>26 else 'old' for i in age1]
print(age2)
avg = sum(data.Overall)/len(data.Overall)
data['level'] = ['high' if i>avg else 'low' for i in data.Overall]
data.loc[0:10,['Overall','level']]
"""
**CLEANING DATA**
"""
"""
EXPLORATORY DATA ANALYSIS

value_counts(): Frequency counts 
outliers: the value that is considerably higher or lower from rest of the data

* Lets say value at 75% is Q3 and value at 25% is Q1.
* Outlier are smaller than Q1 - 1.5(Q3-Q1) and bigger than Q3 + 1.5(Q3-Q1). (Q3-Q1) = IQR 
* We will use describe() method. Describe method includes:
* count: number of entries
* mean: average of entries
* std: standart deviation
* min: minimum entry
* 25%: first quantile
* 50%: median or second quantile
* 75%: third quantile
* max: maximum entry
"""
# lets have a look at the frequency of positions of players
print(data['Position'].value_counts(dropna=False))    # nan values will be counted
# for statistics of the dataframe
data.describe()    # ignore null entries
"""
VISUAL EXPLORATORY DATA ANALYSIS

Box plots: visualize basic statistics like outliers, min/max or quantiles
"""
# For example: compare potential of players that are with real face or nor
# Black line at top is max
# Blue line at top is 75%
# Green line is median (50%)
# Blue line at bottom is 25%
# Black line at bottom is min
# no outliers with real face
data.boxplot(column='Potential',by = 'Real Face')
"""
TIDY DATA
"""
# Firstly I create new data from pokemons data to explain melt nore easily.
data_new = data.head()    # I only take 5 rows into new data
data_new
# id_vars = what we do not wish to melt
# value_vars = what we want to melt
melted = pd.melt(frame=data_new, id_vars='Name', value_vars=['Nationality','Position'])
melted
"""
PIVOTING DATA

reverse of melting
"""
melted.pivot(index='Name', columns = 'variable', values = 'value')
"""
CONCATENATING DATA
"""
data1 = data.head()
data2 = data.tail()
conc_data_row = pd. concat([data1,data2], axis=0, ignore_index=True)   #axis=0 adds dataframes in row
conc_data_row
data1 = data['Nationality'].head()
data2 = data['Position'].head()
conc_data_col = pd.concat([data1,data2],axis=1)    # axis = 0 adds dataframes in columns
conc_data_col
"""
DATA TYPES
"""
data.dtypes
# lets convert object(str) to categorical and int to float.
data['Preferred Foot'] = data['Preferred Foot'].astype('category')
data['Potential'] = data['Potential'].astype('float')
data.dtypes
data['Potential'] = data['Potential'].astype('int')

"""
MISSING DATA and TESTING WITH ASSERT
"""
data.info()
data.Position.value_counts(dropna=False)
we have 60 nan values for position of players
data['Position'].dropna(inplace=True)    # inplace = True means we do not assign it to new variable. Changes automatically assigned to data
# to check we use assert
assert 1==2    # returns error
assert 1==1 # returns nothing
# we check with assert
assert data['Position'].notnull().all() # returns nothing because we dropped nan values
data.Position.value_counts(dropna=False)
# we can not see NAN values anymore
data['Weak Foot'].value_counts(dropna=False)
# we have 48 NAN values
data['Weak Foot'].fillna('empty', inplace=True)
assert data['Weak Foot'].notnull().all() # returns nothing
data['Weak Foot'].value_counts(dropna=False)
# we can not see NAN values anymore
"""
BUILDING DATA FRAMES FROM SCRATCH
"""
# data frames from dictionary
country = ["Italy","Germany"]
population = ["60","70"]
list_label = ["country", "population"]
list_col = [country,population]
zipped = list(zip(list_label,list_col))
data_dict = dict(zipped)
df = pd.DataFrame(data_dict)
df
# add new columns
df["capital"] = ["rome", "berlin"]
df
# broadcasting
df["income"] = 0    # broadcasting entire column
df
VISUAL EXPLORATORY DATA ANALYSIS
# plotting all data
data1 = data.loc[0:1000,["Crossing","Finishing","BallControl"]]
data1.plot()
#subplots
data1.plot(subplots = True)
plt.show()
# scatter plot  
data1.plot(kind="scatter", x = "Crossing", y = "Finishing")
plt.show()
# hist plot
data1.plot(kind="hist", y = "Crossing", bins = 50, range = (0,100), normed = True)
plt.show()
# histogram subplot with cumulative and non-cumulative
fig, axes = plt.subplots(nrows=2,ncols=1)
data1.plot(kind = "hist",y = "Crossing",bins = 30,range= (0,100),normed = True,ax = axes[0])
data1.plot(kind = "hist",y = "Crossing",bins = 30,range= (0,100),normed = True,ax = axes[1],cumulative = True)
plt.savefig('graph.png')
plt
"""
STATISTICAL EXPLORATORY DATA ANALYSIS
"""
data.describe()
INDEXING PANDAS TIME SERIES
time_list = ["1992-03-08","1992-04-12"]
print(type(time_list[1]))    #string
datetime_object = pd.to_datetime(time_list)
print(type(datetime_object))
data2 = data.head()
date_list = ["1992-01-10","1992-02-10","1992-03-10","1993-03-15","1993-03-16"]
datetime_object = pd.to_datetime(date_list)
data2["date"] = datetime_object
# lets make date as index
data2= data2.set_index("date")
data2 
# Now we can select according to our date index
print(data2.loc["1993.03.16"])
print(data2.loc["1992.03.10":"1993.03.16"])
"""
RESAMPLING PANDAS TIME SERIES
* Resampling: statistical method over different time intervals
* Needs string to specify frequency like "M" = month or "A" = year
* Downsampling: reduce date time rows to slower frequency like from daily to weekly
* Upsampling: increase date time rows to faster frequency like from daily to hourly
* Interpolate: Interpolate values according to different methods like ‘linear’, ‘time’ or index’
"""
data2.resample("A").mean()
data2.resample("M").mean()
# We can interpolete from first value
data2.resample("M").first().interpolate("linear")
# Or we can interpolate with mean()
data2.resample("M").mean().interpolate("linear")
"""
MANIPULATING DATA FRAMES WITH PANDAS
"""
# read data
data = pd.read_csv('../input/data.csv')
#data= data.set_index("index")
data.head()
# indexing using square brackets
data["Overall"][1]
# using column attribute and row label
data.Overall[1]
# using loc accessor
data.loc[1,['Overall']]
# selecting only some columns
data[["Overall","Potential"]]
"""
SLICING DATA FRAMES
"""
# Difference between selecting columns: series and dataframes
print(type(data["Overall"]))    # series
print(type(data[['Overall']]))    # data frame
# slicing and indexing series
data.loc[0:10,"Overall":"Value"]    # 10 and Value are inclusive
# reverse slicing
data.loc[10:0:-1,"Overall":"Name":-1]
# from something to end
data.loc[0:10,"Value":]
"""
FILTERING DATA FRAMES
"""
# creating boolean series
boolean = data.Overall > 93
data[boolean]
# combining filters 
first_filter = data.Overall > 90
second_filter = data.Potential > 90
data[first_filter & second_filter]
# filtering column based others
data.Overall[data.Potential>90]
"""
TRANSFORMING DATA
"""
# plain python functions
def div(n):
    return n/2
data.Overall.apply(div)
# Or we can use lambda function
data.Overall.apply(lambda n: n/2)
# defining columns using other columns
data["Strike_Power"] = (data.Crossing + data.Finishing) / 2
data.head()
"""
INDEX OBJECTS AND LABELED DATA
"""
# our index name is this:
print(data.index.name)
#lets change it
data.index.name = "index name"
data.head()
# Overwrite index
# first copy of our data to data3 then change index 
data3 = data.copy()
# lets make index start from 100. It is not remarkable change but it is just example
data3.index = range(100,18307,1)
data3.head()
HIERARCHICAL INDEXING
# Setting index : type 1 is outer type 2 is inner index
data1 = data.set_index(["Nationality","Club"])
data1.head(100)
PIVOTING DATA FRAMES
dic = {"nationality":["TR","TR","UK","UK"],"position":["RF","ST","RF","ST"],"skill":[70,45,75,90],"age":[25,40,32,27]}
df = pd.DataFrame(dic)
df
# pivoting
df.pivot(index = "nationality", columns = "position", values = "skill")
"""
STACKING and UNSTACKING DATAFRAME
* deal with multi label indexes
* level: position of unstacked index
* swaplevel: change inner and outer level index position
"""
df1 = df.set_index(["nationality","position"])
df1
# level determines indexes
df1.unstack(level=0)
df1.unstack(level=1)
# change inner and outer level index position
df2 = df1.swaplevel(0,1)
df2
"""
MELTING DATA FRAMES
* reverse of pivoting
"""
df
pd.melt(df,id_vars = "nationality", value_vars = ["age","skill"])
"""

CATEGORICALS AND GROUPBY
"""
df.groupby("nationality").mean()
df.groupby("nationality").age.max()
df.groupby("nationality")[["age","skill"]].min()