# This Python 3 environment comes with many helpful analytics libraries installed
# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python
# For example, here's several helpful packages to load in 

import numpy as np # linear algebra
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)
import matplotlib.pyplot as plt #  plotting tool
import seaborn as sns  #graphing tool 
import math


# Input data files are available in the "../input/" directory.
# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory

import os
print(os.listdir("../input"))

# Any results you write to the current directory are saved as output.
df = pd.read_csv('../input/oneYear.csv', encoding='latin-1')
df.head()
"""
# We might have other problematic values that aren't necessarily null, but we have to check for null values

"""
pd.isnull(df).any()   
"""
# It can never hurt to try and run a few different graphs, but in this case, the bar graph wasn't particularly appropriate for our case. We'll get to some other types later
"""

ax = df.plot(kind = 'bar')
"""
# The often forgotten problem of duplicates. Therefore we group by economy name and remove any duplicates there
"""
df.groupby(['Economy']).head()

df.drop_duplicates('Economy', keep=False)
"""
# Perhaps viewing a scatterplot of our data could give us a little bit better visualization, and it seems to be that even the countries which scored low overall still had decent scores regarding intercountry trade. 
"""
import matplotlib.ticker as ticker
tick_spacing = 5
plt.figure(figsize=(15,15))
x = df['Ease of doing business rank global (DB19)']
y = df['Score-Trading across borders(DB16-19 methodology)']
plt.scatter(df['Ease of doing business rank global (DB19)'], df['Score-Trading across borders(DB16-19 methodology)'])
plt.xlabel('Rank')
plt.ylabel('Score')
plt.xticks(min(x), max(x))
plt.show()
"""
# The easiest way to find outliers is very often through a boxplot, and I noticed I had two small outliers but I didn't think they would affect the overall outcome very much. 
"""
fig1, ax1 = plt.subplots()
ax1.set_title('Breakdown of how easy it is to do business globally')
ax1.boxplot(df['Ease of doing business score global (DB17-19 methodology)'])
"""
# A good way to check the visualization of our data is through a histogram. Our histogram seems to be left skewed which means most of the countries are towards the top
"""
plt.hist(df['Ease of doing business score global (DB17-19 methodology)'])
"""
# Sometimes trying to run a log function on the data can affect the Skew. I tried that here but it didn't help my skew
"""
def logFunction(x):
  return np.log(x)

df1 = df['Ease of doing business score global (DB17-19 methodology)'].apply(logFunction)
plt.hist(np.log(df1))
"""
# Even through all the visualization of certain columns and coordinates, we have to look at our full dataset to verify that the columns we assume are integers are actually integers. It turns out that while we assumed everything was integers, in actuality it wasn't.
"""
df.apply(lambda s: pd.to_numeric(s, errors='coerce').notnull().all())
"""
# Once we know that we have many issues with Strings, we have to drop the columns that have no data associated with them
"""
df.drop(df.loc[df['Trading across Borders - Time to export: Documentary compliance (hours) (DB16-19 methodology)']=='No Practice'].index, inplace=True)
df.drop(df.loc[df['Rank-Trading across borders (DB19)']== ' '].index, inplace=True)
"""
# After this stage, it turns out that all the columns are properly stored as integers which allows us to continue wrangling
"""
df.apply(lambda s: pd.to_numeric(s, errors='coerce').notnull().all())
"""
# Aside from economy and year which are fine as strings, everything is now an integer
"""
"""
# I found my minimum values which contained some of the outliers, and dropped those from the entire dataset to see if I could get more accurate readings. 
"""
temp = df['Ease of doing business score global (DB17-19 methodology)'].idxmin()
df.drop(df.loc[df['Economy']== 'Tonga'].index, inplace=True)
df.drop(df.loc[df['Economy']== 'Trinidad and Tobago'].index, inplace=True)
"""
# After running my boxplot again without the outliers, not much changed when I omitted the outliers, but that's alright. 
"""
fig1, ax1 = plt.subplots()
ax1.set_title('Basic Plot')
ax1.boxplot(df['Ease of doing business score global (DB17-19 methodology)'], showfliers = False)
"""
# I was curious to see at least one other distribution and boxplot to see how they were distributed, so I decided to check out time to export across borders, and interestingly enough it was not as nicely distributed as the other column was.
"""
fig1, ax1 = plt.subplots()
ax1.set_title('Basic Plot')
ax1.boxplot(df['Trading across Borders - Time to export: Documentary compliance (hours) (DB16-19 methodology)'].astype(np.float))
"""
# And then just to check out the histogram of that column as well, it is not well distributed, and is very right skewed.
"""
plt.hist(df['Trading across Borders - Time to export: Documentary compliance (hours) (DB16-19 methodology)'].astype(np.float))
"""
### But clearly from here, just because some columns of your data are nicely distributed, it doesn't mean that all the columns of the data are going to be normally distributed.
"""
"""
# Finally, just to get a better look at some of my statistics of the dataset one last time now that it's been wrangled, I had to describe it all leaving out duplicates. It turns out that hte top 25% aren't so far from each other in terms of score, and it doesn't make the biggest difference whether you're at the top or at the 75% mark. 
"""
df.describe()
"""
# Finally, I wanted to enjoy some of my hardwork and see just how correlated the tariffs were to the overall score of the specific country
"""
df[pd.to_numeric(df['Ease of doing business rank global (DB19)'], errors = 'coerce').notnull()]
corr = df['Score-Trading across borders(DB16-19 methodology)'].corr(df['Ease of doing business score global (DB17-19 methodology)'])
print(corr)
"""
# Conclusion: After wrangling through the dataset and running some basic correlation statistics on my dataset I could enjoy the fruits of my labor and see that there was strong correlation betweeen the ease of trading across borders as well as the overall ranking of a country's business plan.  
"""