import pandas as pd
import os as os
import preprocessing_helper as preprocessing_helper
import matplotlib as plt
% matplotlib inline

"""
Explanation: ----------------------------- AIRBNB CHALLENGE -----------------------------
Where will new guests book their first travel experience?
Malo Grisard, Guillaume Jaume, Cyril Pecoraro - EPFL - 15th of January 2017
<br>
<br>
Main Preprocessing:
Pipeline:
    1. Data exploration and cleaning
    2. Machine learning preprocessing
    3. Machine learning optimization
    4. Results
The purpose of this project is to predict which country a new user's first booking destination will be. We are given a list of users along with their demographics, web session records, and some summary statistics. All the users in this dataset are from the USA.
There are 12 possible outcomes of the destination country: 'US', 'FR', 'CA', 'GB', 'ES', 'IT', 'PT', 'NL','DE', 'AU', 'NDF' (no destination found), and 'other'.
In this notebook, we explored and cleaned the given data and try to show the most relevant features extracted.
The cleaned data are saved into separated files and analysed in the Machine Learning Notebook.
Important Remark: Due to size constraints, we were not able to load the file session.csv in the Git, you can upload it directly online from the Kaggle Competition here.
End of explanation
"""


filename = "train_users_2.csv"
folder = 'data'
fileAdress = os.path.join(folder, filename)
df = pd.read_csv(fileAdress)
df.head()

"""
Explanation: 1. Data exploration and cleaning
The dataset is composed by several files. First, we are going to explore each of them and clean some variables. For a complete explanation of each file, please see the file DATA.md.
1.1 file 'train_users_2.csv'
This file is the most important file in our dataset as it contains the users, information about them and the country of destination.
When a user has booked a travel through Airbnb, the destination country will be specified. Otherwise, 'NDF' will be indicated.
End of explanation
"""


df.isnull().any()

"""
Explanation: There are missing values in the columns : 

date_first_booking : users that never booked an airbnb apartment
gender : users that didn't wish to specify their gender
age : users that didn't wish to specify their age
first_affiliate_tracked : problem of missing data

We wil go each of these variable and take decisions regarding the missing values
End of explanation
"""


df = preprocessing_helper.cleanAge(df,'k')

"""
Explanation: Ages
There are 2 problems regarding ages in the dataset.
First, many users did not specify an age.
Also, some users specified their year of birth instead of age.
For the relevancy of the data we will keep users between the age of 15 and 100 years old, and those who specified their age.
For the others, we will naively assign a value of -1
End of explanation
"""


preprocessing_helper.plotAge(df)

"""
Explanation: The following graph presents the distribution of ages in the dataset. Also, the irrelevant ages are represented here, with their value of -1.
End of explanation
"""


df = preprocessing_helper.cleanGender(df)
preprocessing_helper.plotGender(df)

"""
Explanation: Gender
The following graph highlights the gender of Airbnb users. Note that around 45% did not specify their age.
End of explanation
"""


df = preprocessing_helper.cleanFirst_affiliate_tracked(df)

"""
Explanation: first_affiliate_tracked feature
Set the first marketing the user interacted with before the signing up to 'Untracked' if not specified.
End of explanation
"""


df = preprocessing_helper.cleanDate_First_booking(df)
preprocessing_helper.plotDate_First_booking_years(df)

"""
Explanation: Date_first_booking

This has a high similarity with the dates where accounts were created. Despite the high growth of airbnb bookings throughout the years, it is possible to see that the difference between the months increases over the years as each year parabol curve increases.
By studying each year independently, we could see that four peaks arise each month corresponding to a certain day in the week. The following plots will show the bookings distribution over the months and later over the week
The most prolific day for Airbnb counted 248 bookings
End of explanation
"""


preprocessing_helper.plotDate_First_booking_months(df)

"""
Explanation: It is possible to understand from this histogram that the bookings are pretty well spread over the year. Much less bookings are made during november and december and the months of May and June are the ones where users book the most. For these two months Airbnb counts more than 20000 bookings which corresponds to allmost a quarter of the bookings from our dataset.
End of explanation
"""


preprocessing_helper.plotDate_First_booking_weekdays(df)

"""
Explanation: As for the day where most accounts are created, it seems that tuesday and wednesdays are the days where people book the most appartments on Airbnb.
End of explanation
"""


filename = "cleaned_train_user.csv"
folder = 'cleaned_data'
fileAdress = os.path.join(folder, filename)
preprocessing_helper.saveFile(df, fileAdress)

"""
Explanation: Save cleaned and explored file
End of explanation
"""


# extract file 
filename = "test_users.csv"
folder = 'data'
fileAdress = os.path.join(folder, filename)
df = pd.read_csv(fileAdress)
# process file
df = preprocessing_helper.cleanAge(df,'k')
df = preprocessing_helper.cleanGender(df)
df = preprocessing_helper.cleanFirst_affiliate_tracked(df)
# save file 
filename = "cleaned_test_user.csv"
folder = 'cleaned_data'
fileAdress = os.path.join(folder, filename)
preprocessing_helper.saveFile(df, fileAdress)

"""
Explanation: 1.2 file 'test_user.csv'
This file has a similar structure than train_user_2.csv, so here, we will just do the cleaning process here.
End of explanation
"""


filename = "countries.csv"
folder = 'data'
fileAdress = os.path.join(folder, filename)
df = pd.read_csv(fileAdress)
df

df.describe()

"""
Explanation: 1.3 file 'countries.csv'
This file presents a summary of the countries presented in the dataset.
This is the signification:
- 'AU' = Australia
- 'ES' = Spain
- 'PT' = Portugal
- 'US' = USA
- 'FR' = France
- 'CA' = Canada
- 'GB' = Great Britain
- 'IT' = Italy
- 'NL' = Netherlands
- 'DE' = Germany
- 'NDF'= No destination found
All the variables are calculated wrt. the US and english. The levenshtein distance is an indication on how far is the language spoken in the destination country compared to english. All the other variables are general geographics elements. This file will not be used in our model as it does not give direct indications regarding the users.
End of explanation
"""


filename = "age_gender_bkts.csv"
folder = 'data'
fileAdress = os.path.join(folder, filename)
df = pd.read_csv(fileAdress)
df.head()

"""
Explanation: 1.4 file 'age_gender_bkts.csv'
This file presents demograhpic statistics about each country present in our dataset. This file will not be used in our model.
End of explanation
"""


df_country = df.groupby(['country_destination'],as_index=False).sum()
df_country

"""
Explanation: Population total per country
The following table shows the population in the country in 2015. These numbers correspond to data that can be found on the web.
End of explanation
"""


filename = "sessions.csv"
folder = 'data'
fileAdress = os.path.join(folder, filename)
df = pd.read_csv(fileAdress)
df.head()

"""
Explanation: 1.5 file 'sessions.csv'
This file keeps a track of each action made by each user (represented by their id). For each action (lookup, search etc...), the device type is saved so as the time spend for this action.
End of explanation
"""


df.isnull().any()

df = preprocessing_helper.cleanSubset(df, 'user_id') 

"""
Explanation: NaN users
As we can see, there are some missing user_id. Without a user_id, it is impossible to link them with the file train_user.csv. We will delete them as we cannot do anything with them.
End of explanation
"""


df['secs_elapsed'].fillna(-1, inplace = True)

"""
Explanation: Invalid session time
If a session time is NaN, there was probably an error during the session. We are not going to remove the rows correponding, because there are still some data interesting for the actions variable.
Instead, we are naively going to assign them a value of -1.
End of explanation
"""


df = preprocessing_helper.cleanAction(df)

"""
Explanation: Actions
Some action produce -unknown- for action_type and/or action_detail. Sometimes they produce NaN. We replace the NaN values with -unknown- for action_type,action_detail, action
End of explanation
"""


df.isnull().any()

"""
Explanation: As shown in the following, there are no more NaN values.
End of explanation
"""


# Get total number of action per user_id
data_session_number_action = preprocessing_helper.createActionFeature(df)

# Save to .csv file
filename = "total_action_user_id.csv"
folder = 'cleaned_data'
fileAdress = os.path.join(folder, filename)
preprocessing_helper.saveFile(data_session_number_action, fileAdress)

# Plot distribution total number of action per user_id
preprocessing_helper.plotActionFeature(data_session_number_action)

"""
Explanation: Total number of actions per user
From the session, we can compute the total number of actions per user. Intuitively, we can imagine that a user totalising few actions might be a user that does not book in the end. This value will be used as a new feature for the machine learning.
Note: The total number of actions is represented on a logarithmic basis.
End of explanation
"""


preprocessing_helper.plotHist(df['device_type'])

"""
Explanation: Device types
There are 14 possible devices. Most of the users however are distributed on three main devices.
End of explanation
"""


# Get Time spent on average per user_id
data_time_mean = preprocessing_helper.createAverageTimeFeature(df)

# Save to .csv file
data_time_mean = data_time_mean.rename(columns={'user_id': 'id'})
filename = "time_mean_user_id.csv"
folder = 'cleaned_data'
fileAdress = os.path.join(folder, filename)
preprocessing_helper.saveFile(data_time_mean, fileAdress)

# Plot distribution average time of session per user_id
preprocessing_helper.plotTimeFeature(data_time_mean['secs_elapsed'],'mean')

"""
Explanation: Time spent on average per user
The figure below shows the time spent on average per user. The following plot relates to the Total number of actions one with even clearer two gaussians.
We display only time > 20s.
This value will also be used as a feature for the machine learning.
End of explanation
"""


# Get Time spent in total per user_id
data_time_total = preprocessing_helper.createTotalTimeFeature(df)

# Save to .csv file
data_time_total = data_time_total.rename(columns={'user_id': 'id'})
filename = "time_total_user_id.csv"
folder = 'cleaned_data'
fileAdress = os.path.join(folder, filename)
preprocessing_helper.saveFile(data_time_total, fileAdress)

# Plot distribution total time of session per user_id
preprocessing_helper.plotTimeFeature(data_time_total['secs_elapsed'],'total')

"""
Explanation: Time spent in total per user
The figure below shows the total amount of time spent per user. 
We display only time > 20s
This feature is the 3rd and last one used for the machine learning from the file session. Intuitively, a long time spent leads to a booking and possibly further destinations.
End of explanation
"""


preprocessing_helper.plotTimeFeature(df['secs_elapsed'],'dist')

"""
Explanation: Distribution of time spent
This last graph shows the distribution of time spent in second per session. 
We display only time > 20s
End of explanation
"""


