"""
# Analysis of Airplane Crashes since 1908
"""
"""
## 1. Importing modules
"""
# This Python 3 environment comes with many helpful analytics libraries installed

import pandas as pd #linear algebra
import seaborn as sns #visualization tool
import matplotlib
from matplotlib import pyplot as plt
from scipy import stats as sts  #data processing, CSV file I/O (e.g. pd.read_csv)
import datetime as dt
from collections import Counter
import os

print(os.listdir("../input/"))
"""
## 2. Loading the data
"""
data = pd.read_csv("../input/Airplane_Crashes_and_Fatalities_Since_1908.csv")
"""
See first five columns to ensure our data file read correctly
"""
data.head()
"""
## 3. Data Info and Manipulation
"""
data.info()
data.isnull().any()
"""
As we can see from the outputs above, we have some NaN values. We will replace these values with 0 for the numeric values.
"""
data['Fatalities'].fillna(0, inplace = True)
data['Aboard'].fillna(0, inplace = True)
data['Ground'].fillna(0, inplace = True)
"""
Let's convert the 'Date' column to the appropriate format.
"""
data['Date'] = pd.to_datetime(data['Date'])
data['Date'] = data['Date'].dt.strftime("%d/%m/%Y")
data['Date'].head()
"""
For visualization, I create a new column called Year.
"""
data['Year'] = pd.DatetimeIndex(data['Date']).year
data['Year'].head()
"""
With data on number of fatalities and people aboard,I create a new variable with the number of people that survived the crash and call this variable 'Survived'.I also replace any NaN value with 0 on this column.
"""
data['Survived'] = data['Aboard'] - data['Fatalities']
data['Survived'].fillna(0, inplace = True)
"""
Now our dataframe looks like this.
"""
data.head()
"""
## 4. Visualizations
"""
matplotlib.rcParams['figure.figsize'] = (20, 10)
sns.set_context('talk')
sns.set_style('whitegrid')
sns.set_palette('tab20')
"""
### 4.1 Airplane Crashes per Year 
"""
"""
First we summarise to get the count of accidents per year
"""
total_crashes_year = data[['Year', 'Date']].groupby('Year').count()
total_crashes_year = total_crashes_year.reset_index()
total_crashes_year.columns = ['Year', 'Crashes']
"""
Line plot with Seaborn.
"""
sns.lineplot(x = 'Year', y = 'Crashes', data = total_crashes_year)
plt.title('Total Airplane Crashes per Year')
plt.xlabel('years')
plt.ylabel('number of crashes')
"""
We see that after 40's, there is a significant increase in airplane crashes. The highest peaks are between 1960 and 2000.
"""
"""
### 4.2 Death Toll per Year
"""
pcdeaths_year = data[['Year', 'Fatalities']].groupby('Year').sum()
pcdeaths_year.reset_index(inplace = True)
# Plot
sns.lineplot(x = 'Year', y = 'Fatalities', data = pcdeaths_year)
plt.title('Total Number of Fatalities by Air Plane Crashes per Year')
plt.xlabel('Fatalities')
plt.xlabel('Years')
"""
Here we can see the same pattern, the years that had the most accidents are also the ones with the most fatalities
"""
"""
### 4.3 People Aboard Airplanes per Year
"""
# summarise
abrd_per_year = data[['Year', 'Aboard']].groupby('Year').sum()
abrd_per_year = abrd_per_year.reset_index()
# plot
sns.lineplot(x = 'Year', y = 'Aboard', data = abrd_per_year)
plt.title('Total of People Aboard Airplanes per Year')
plt.xlabel('Years')
plt.ylabel('Count')
"""
From the 40's, the number of people aboard airplanes starts to increase. From 1960 to 2000 is where we have most people aboard, the same years with most plane crashes and fatalities.
"""
"""
### 4.4 Fatalities vs Survived vs Killed on Ground
"""
"""
Now we visualize how the number of fatalities compare with the number of survived and those who were killed on the ground.
"""
#summarise
FSG_per_year = data[['Year', 'Fatalities', 'Survived', 'Ground']].groupby('Year').sum()
FSG_per_year = FSG_per_year.reset_index()
#plot
sns.lineplot(x = 'Year', y = 'Fatalities', data = FSG_per_year, color = 'green')
sns.lineplot(x = 'Year', y = 'Survived', data = FSG_per_year, color = 'blue')
sns.lineplot(x = 'Year', y = 'Ground', data = FSG_per_year, color = 'red')
plt.legend(['Fatalities', 'Survival', 'Ground'])
plt.xlabel('Years')
plt.ylabel('Count')
plt.title('Fatalities vs Survived vs Killed on Ground per Year')
"""
### 4.5 Worst operators and dangerous locations

"""
oper_list = Counter(data['Operator']).most_common(10)
operators = []
crashes = []
for tpl in oper_list:
    if 'Military' not in tpl[0]:
        operators.append(tpl[0])
        crashes.append(tpl[1])
print('Top 10 the worst operators')
pd.DataFrame({'Count of crashes' : crashes}, index=operators)
loc_list = Counter(data['Location'].dropna()).most_common(10)
locs = []
crashes = []
for loc in loc_list:
    locs.append(loc[0])
    crashes.append(loc[1])
print('Top 10 the most dangerous locations')
pd.DataFrame({'Crashes in this location' : crashes}, index=locs)
"""
## 5. Text Clustering with K-Means
"""
"""
### 5.1 Importing needed modules
"""
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.cluster import KMeans
from sklearn.cluster import MiniBatchKMeans
from sklearn.metrics import adjusted_rand_score
from sklearn.preprocessing import LabelEncoder
from sklearn.preprocessing import MinMaxScaler
import seaborn as sns
import matplotlib.pyplot as plt
from sklearn.decomposition import PCA
"""
### 5.2 Data Preparation
"""
"""
In the 'Summary' column, we have NaN values as well, so we're going to create a new dataframe with the 'Summary' data and dropping all rows with NaN values.
"""
text_data = data['Summary'].dropna()
text_data = pd.DataFrame(text_data)
# for reproducibility
random_state = 0 
"""
KMeans normally works with numbers only: we need to have numbers.To get numbers, we do feature extraction.

The feature we’ll use is TF-IDF, a numerical statistic. This statistic uses term frequency and inverse document frequency.
The method TfidfVectorizer() implements the TF-IDF algorithm.
"""
documents = list(text_data['Summary'])
vectorizer = TfidfVectorizer(stop_words='english') # Stop words are like "a", "the", or "in" which don't have significant meaning
X = vectorizer.fit_transform(documents)
"""
## 5.3 Model Fitting
"""
"""
And now we fit the model. For this analysis, we'll be using the KMeans algorithm with 5 clusters.
"""
model = MiniBatchKMeans(n_clusters=5, random_state=random_state)
model.fit(X)
"""
What are the cluster center vectors?
"""
model.cluster_centers_
# predict cluster labels for new dataset
model.predict(X)

# to get cluster labels for the dataset used while
# training the model (used for models that does not
# support prediction on new dataset).
model.labels_
print ('Most Common Terms per Cluster:')

order_centroids = model.cluster_centers_.argsort()[:,::-1] #sort cluster centers by proximity to centroid
terms = vectorizer.get_feature_names()

for i in range(5):
    print("\n")
    print('Cluster %d:' % i)
    for j in order_centroids[i, :10]: #replace 10 with n words per cluster
        print ('%s' % terms[j]),
    print
"""
## 5.4 Visualization
To visualize, we’ll plot the features in a 2D space. As we know the dimension of features that we obtained from TfIdfVectorizer is quite large ( > 10,000), we need to reduce the dimension before we can plot. For this, we’ll ues PCA to transform our high dimensional features into 2 dimensions.
"""
# reduce the features to 2D
pca = PCA(n_components=2, random_state=random_state)
reduced_features = pca.fit_transform(X.toarray())

# reduce the cluster centers to 2D
reduced_cluster_centers = pca.transform(model.cluster_centers_)
plt.scatter(reduced_features[:,0], reduced_features[:,1], c=model.predict(X))
plt.scatter(reduced_cluster_centers[:, 0], reduced_cluster_centers[:,1], marker='x', s=150, c='b')
"""
## 5.5 Prediction
"""
print("\n")
print("Prediction")

Y = vectorizer.transform(["engine failure"])
prediction = model.predict(Y)
print(prediction)

Y = vectorizer.transform(["terrorism"])
prediction = model.predict(Y)
print(prediction)
 