import urllib3
import pandas as pd

url = "https://raw.githubusercontent.com/jpatokal/openflights/master/data/airports.dat"

#load the csv
airports = pd.read_csv(url,header=None)

print("Check DataFrame types")
display(airports.dtypes)

"""
Explanation: Exploratory Data Analysis with Pandas
Exploratory data analysis (EDA) is an approach to analyzing data sets to summarize their main characteristics, often with visual methods.
We will go through this notebook to:
1. Learn to reshape our data and see some features by performing operations over it
2. Discover visualization methods
Transforming and summarizing data
End of explanation
"""


import numpy as np

print("-> Original DF")
display(airports.head())

#we can add a name to each variable
h = ["airport_id","name","city","country","IATA","ICAO","lat","lon","alt","tz","DST","tz_db"]
airports = airports.iloc[:,:12]
airports.columns = h

print("-> Original DF with proper names")
display(airports.head())

print("-> With the proper names it is easier to check correctness")
display(airports.dtypes)

"""
Explanation: Here you can find an explanation of each variable:

Airport ID   Unique OpenFlights identifier for this airport.
Name Name of airport. May or may not contain the City name.
City Main city served by airport. May be spelled differently from Name.
Country  Country or territory where airport is located.
IATA/FAA 3-letter FAA code, for airports located in Country "United States of America". 3-letter IATA code, for all other airports. Blank if not assigned.
ICAO 4-letter ICAO code. Blank if not assigned.
Latitude Decimal degrees, usually to six significant digits. Negative is South, positive is North.
Longitude    Decimal degrees, usually to six significant digits. Negative is West, positive is East.
Altitude In feet.
Timezone    Hours offset from UTC. Fractional hours are expressed as decimals, eg. India is 5.5.
DST Daylight savings time. One of E (Europe), A (US/Canada), S (South America), O (Australia), Z (New Zealand), N (None) or U (Unknown). See also: Help: Time
Tz database time zone
End of explanation
"""


airports.alt.describe()

airports.alt = airports.alt * 0.3048

airports.dtypes

"""
Explanation: Convert alt to m
End of explanation
"""


airports.isnull().sum(axis=0)

# we can create a new label whoch corresponds to not having data
airports.IATA.fillna("Blank", inplace=True)
airports.ICAO = airports.ICAO.fillna("Blank")

airports.isnull().sum(axis=0)

"""
Explanation: Check if we have nans.
End of explanation
"""


((airports.lat > 90) & (airports.lat < -90)).any()

((airports.lon > 180) & (airports.lon < -180)).any()

"""
Explanation: Let's check errors.
* Latitudes range from -90 to 90.
* Longitudes range from -180 to 180.
End of explanation
"""


airports.alt.describe()

"""
Explanation: We can chech outliers in the altitude
End of explanation
"""


qtls = airports.alt.quantile([.05,.5,.95],interpolation="higher")
qtls

# check how many of them are below the median
(airports.alt <= qtls[0.5]).sum()

#check how many of them are above of the median
(airports.alt >= qtls[0.5]).sum()

#check how many of them are below the .05 percentile
(airports.alt <= qtls[0.05]).sum()

#check how many of them are above the .95 percentile
(airports.alt >= qtls[0.95]).sum()

airports.shape[0]*.05

print("-> Check which airports are out of 5% range")
display(airports[(airports.alt < qtls[0.05])].head(10))

"""
Explanation: let's explore 5 and 95 percentiles
End of explanation
"""


print("-> Showing a sample of ten values")
airports.sample(n=10)

print("-> Showing the airports in higher positions")
airports.sort_values(by="alt",ascending=True)[:10]

"""
Explanation: Additionaly to what we have seen, we have extra functions to see how shaped and what values our data has.
* sample data: we can take a random sample of the obesrvations to avoid the ordering bias (if we head data, it can be sorted so some of the examples are ok, let's say the 100 first and the rest have some errors)
* sort data: to get obesrvations with higher or lower values
End of explanation
"""


airports.tz_db

airports["continent"] = airports.tz_db.str.split("/").str[0]
airports.continent.unique()

airports.continent.value_counts()

(airports.continent.value_counts()/airports.continent.value_counts().sum())*100

airports[airports.continent == "\\N"].shape

airports.continent = airports.continent.replace('\\N',"unknown")
airports.tz_db = airports.tz_db.replace('\\N',"unknown")

airports.continent.unique()

airports[airports.continent == "unknown"].head()

"""
Explanation: We can create new variables
End of explanation
"""


hem_select = lambda x: "South" if x < 0 else "North"
airports["hemisphere"] = airports.lat.apply(hem_select)

"""
Explanation: We can place hemisfere
End of explanation
"""


(airports.hemisphere.value_counts() / airports.shape[0]) * 100

(airports.continent.value_counts() / airports.shape[0]) * 100

((airports.country.value_counts() / airports.shape[0]) * 100).sample(10)

((airports.country.value_counts() / airports.shape[0]) * 100).head(10)

type(airports.country.value_counts())

"""
Explanation: We can calculate percentages.
End of explanation
"""


airports["alt_type"] = pd.cut(airports.alt,bins=3,labels=["low","med","high"])

airports.head()

"""
Explanation: Let's transformate alt into qualitative
End of explanation
"""


airp_group = airports.groupby(["continent","alt_type"])

"""
Explanation: Let's group data:
End of explanation
"""


airp_group.groups.keys()

"""
Explanation: The groups attribute is a dict whose keys are the computed unique groups and corresponding values being the axis labels belonging to each group. In the above example we have:
End of explanation
"""


airp_group.size()

airp_group["alt"].agg({"max":np.max,"min":np.min,"mean":np.mean}).head()

airports.alt.hist(bins=100)

"""
Explanation: Once the GroupBy object has been created, several methods are available to perform a computation on the grouped data.
End of explanation
"""


airp_group["alt"].sum().unstack()

"""
Explanation: Pandas has a handy .unstack() method—use it to convert the results into a more readable format and store that as a new variable
End of explanation
"""


airports.pivot_table(index="hemisphere",values="alt",aggfunc=np.mean)

airports.groupby("hemisphere").alt.mean()

"""
Explanation: Remember that we also saw how to pivot table
End of explanation
"""


my_df = pd.DataFrame(np.ones(100),columns=["y"])
my_df.head(10)

import matplotlib.pyplot as plt
%matplotlib inline
import matplotlib
matplotlib.style.use('ggplot')
plt.rcParams['figure.figsize'] = [10, 8]

my_df.plot()

my_df["z"] = my_df.y.cumsum()
my_df.plot()

my_df.y = my_df.z ** 2
my_df.plot()

my_df.z = np.log(my_df.y)
my_df.z.plot()

"""
Explanation: Visualizing data
One of the most useful tools for exploring data anf presenting results is through visual representations.
End of explanation
"""


airports.groupby("continent").size().plot.bar()

"""
Explanation: We can plot with different plot types:
    * ‘bar’ or ‘barh’ for bar plots
    * ‘hist’ for histogram
    * ‘box’ for boxplot
    * ‘kde’ or 'density' for density plots
    * ‘area’ for area plots
    * ‘scatter’ for scatter plots
    * ‘hexbin’ for hexagonal bin plots
    * ‘pie’ for pie plots
Bar
End of explanation
"""


airports.groupby("continent").alt.agg({"max":np.max,"min":np.min,"mean":np.mean}).plot(kind="bar")

airports.groupby("continent").alt.agg({"max":np.max,"min":np.min,"mean":np.mean}).plot(kind="bar",stacked=True)

airports.groupby("continent").alt.agg({"max":np.max,"min":np.min,"mean":np.mean}).plot(kind="barh",stacked=True)

"""
Explanation: Multiple Bars
End of explanation
"""


airports.alt.plot(kind="hist",bins=100)

airports.loc[:,["alt"]].plot(kind="hist")

airports.loc[:,["lat"]].plot(kind="hist",bins=100)

airports.loc[:,["lon"]].plot(kind="hist",bins=100)

"""
Explanation: Histogram
End of explanation
"""


airports.plot.box()

airports.alt.plot.box()

airports.pivot(columns="continent").alt.plot.box()

"""
Explanation: Box Plots
End of explanation
"""


sp_airp = airports[airports.country=="Spain"]
spain_alt = sp_airp.sort_values(by="alt").alt
spain_alt.index = range(spain_alt.size)
spain_alt.plot.area()

"""
Explanation: Area Plots
End of explanation
"""


airports.plot.scatter(y="lat",x="lon")

airports.plot.scatter(y="lat",x="lon",c="alt")

airports.plot.scatter(y="lat",x="lon",s=airports["alt"]/20)

"""
Explanation: Scatter Plot
End of explanation
"""


airports.plot.hexbin(x="lon",y="lat",C="alt",gridsize=20)

"""
Explanation: Hex Bins
End of explanation
"""


airports.alt.plot.kde()

airports.lat.plot.kde()

airports.lon.plot.kde()

"""
Explanation: Density Plot
End of explanation
"""


