# list containing the link(s) to the csv file(s)
data_links = ['https://storage.googleapis.com/tlc-trip-data/2015/yellow_tripdata_2015-06.csv']

filenames = []
for link in data_links:
    filenames.append(link.split('/')[-1])
    if not(os.path.isfile(filenames[-1])): # do not download file if it already exists
        urllib.urlretrieve(link, filename)

"""
Explanation: Getting and inspecting the data
Downloading the data. At the moment, only data from June 2015 is considered.
End of explanation
"""


df = pd.DataFrame()
for filename in filenames:
    df = df.append(pd.read_csv(filename), ignore_index=True)

df.head()

df.info()

df.describe()

"""
Explanation: Loading the data into a pandas data frame and look at it:
End of explanation
"""


df.groupby('payment_type').size().plot(kind='bar');

"""
Explanation: Note that some of the numerical features like tip amount and fare amount actually contain negative values. Those invalid values will be deleted in the next section.
Data cleaning
1) Retaining only trips paid by credit card
Only the credit card tips are recorded in the data set. Therefore, let's only retain trips with credit card payment. This might introduce some bias (as credit card payers may have a different tipping behaviour than others).
As seen below, most of the trips are anyway paid by credit card (label "1", followed by cash payment, label "2").
End of explanation
"""


np.sum((df.payment_type != 1) & (df.tip_amount != 0))

"""
Explanation: For some trips, people actually tipped with credit card, even though they did not pay with credit card:
End of explanation
"""


df = df[df.payment_type == 1]
df.drop('payment_type', axis=1, inplace=True)
df.shape

"""
Explanation: However, the number of those trips is negligible, so I ignore them here and only retain credit card trips. Then, the column "payment_type" can be removed:
End of explanation
"""


(df < 0).sum()

"""
Explanation: 2) Checking for unfeasible values in numerical features
As seen above, some of the numerical features contained negative values. Let's have a closer look...
End of explanation
"""


col_names = ['total_amount', 'improvement_surcharge', 'tip_amount', 'mta_tax', 'extra', 'fare_amount']

# this removes all rows where at least one value of the columns in col_names is < 0
rows_to_keep = (df[col_names] >= 0).sum(axis=1) == len(col_names)
print 'removing '+ str((~rows_to_keep).sum()) + ' rows...'
df = df[rows_to_keep]

(df[col_names] < 0).sum() # check if it worked

"""
Explanation: ...and remove the corresponding rows where negative values do not make any sense:
End of explanation
"""


ax = df.loc[sample(df.index, 30000)].plot(y='trip_distance',kind='hist', bins=200)
ax.set_xlim([0,25]);

"""
Explanation: 3) Deleting "invalid" trips
Inspecting trip distance
End of explanation
"""


rows_to_keep = df.trip_distance <= 50
print 'removing '+ str((~rows_to_keep).sum()) + ' rows...'
df = df[rows_to_keep]

"""
Explanation: Delete trips that are longer than 50 miles...
End of explanation
"""


rows_to_keep = df.trip_distance >= 0.1
print 'removing '+ str((~rows_to_keep).sum()) + ' rows...'
df = df[rows_to_keep]

"""
Explanation: ...and shorter than 0.1 miles:
End of explanation
"""


ax = df.loc[sample(df.index, 300000)].plot(y='fare_amount',kind='hist', bins=200)
ax.set_xlim([0,102]);

"""
Explanation: Inspecting trip fare
End of explanation
"""


rows_to_keep = df.fare_amount >= 1
print 'removing '+ str((~rows_to_keep).sum()) + ' rows...'
df = df[rows_to_keep]

"""
Explanation: There seem to be a decent amount of trips with a fixed rate of 50 USD (see spike above).
Now let's remove rows where the fare is below 1 USD:
End of explanation
"""


df.tpep_pickup_datetime = pd.to_datetime(df.tpep_pickup_datetime)
df.tpep_dropoff_datetime = pd.to_datetime(df.tpep_dropoff_datetime)

df['trip_duration'] = df.tpep_dropoff_datetime - df.tpep_pickup_datetime 

df['trip_duration_minutes'] = df.trip_duration.dt.seconds/60

ax = df.loc[sample(df.index, 300000)].plot(y='trip_duration_minutes', kind='hist', bins=500)
ax.set_xlim([0,150]);

"""
Explanation: Inspecting trip duration
End of explanation
"""


rows_to_keep = df.trip_duration_minutes>0.5
print 'removing '+ str((~rows_to_keep).sum()) + ' rows...'
df = df[rows_to_keep]

"""
Explanation: Remove trips that took less than half a minute...
End of explanation
"""


rows_to_keep = df.trip_duration_minutes<=2*60
print 'removing '+ str((~rows_to_keep).sum()) + ' rows...'
df = df[rows_to_keep]

"""
Explanation: ...as well as trips with a duration of more than 2 hours:
End of explanation
"""


df.plot(y='passenger_count', kind='hist', bins=30);

"""
Explanation: Inspecting passenger count
End of explanation
"""


rows_to_keep = df.passenger_count > 0
print 'removing '+ str((~rows_to_keep).sum()) + ' rows...'
df = df[rows_to_keep]

"""
Explanation: Remove trips with zero passenger count:
End of explanation
"""


rows_to_keep = df.passenger_count <= 6
print 'removing '+ str((~rows_to_keep).sum()) + ' rows...'
df = df[rows_to_keep]

"""
Explanation: Remove trips with a passenger count of more than 6:
End of explanation
"""


within_NY = (df.pickup_latitude > 40) & (df.pickup_latitude < 40.9) & \
            (df.pickup_longitude > -74.4) & (df.pickup_longitude < -73.4)

print 'removing '+ str((~within_NY).sum()) + ' rows...'
df = df[within_NY]

"""
Explanation: Removing invalid location coordinates
Remove trips that obviously did not start in NY:
End of explanation
"""


fig, ax = plt.subplots(figsize=(15, 10))
df.loc[sample(df.index, 200000)].plot(x='pickup_longitude', y='pickup_latitude', 
                                      kind='scatter', ax=ax, alpha=0.3, s=3)
ax.set_xlim([-74.2, -73.7])
ax.set_ylim([40.6, 40.9]);

"""
Explanation: Plot the pickup locations to check if they look good. Choose a random sample of all trips, since plotting all trips would take quite a while.
End of explanation
"""


subdf = df.loc[sample(df.index, 10000)] # subsample df 
data = subdf[['pickup_latitude', 'pickup_longitude']].values

mapa = folium.Map([40.7, -73.9], zoom_start=11, tiles='stamentoner') # create heatmap
mapa.add_children(plugins.HeatMap(data, min_opacity=0.005, max_zoom=18,
                                  max_val=0.01, radius=3, blur=3))
mapa

"""
Explanation: The above plot looks reasonable, you can clearly identify the geometry of New York. Let's plot a small subset of data points on a map. Next to central NY, one can identify small hotspots at the surrounding airports.
End of explanation
"""


fig, ax = plt.subplots(figsize=(12,4))
ax = df.loc[sample(df.index, 100000)].plot(y='tip_amount', kind='hist',bins=1500, ax=ax)
ax.set_xlim([0,10.5])
ax.set_xticks(np.arange(0, 11, 0.5));

"""
Explanation: Inspecting the tip
As the tip distribution below shows, people tend to tip whole numbers of dollars (see peaks at e.g. 1 and 2 dollars).
End of explanation
"""


# check if the fares and fees sum up to total_amount
print pd.concat([df.tip_amount + df.fare_amount + df.tolls_amount + \
           df.extra + df.mta_tax  + df.improvement_surcharge, \
           df.total_amount], axis=1).head()

# calculate tip percentage
df['total_fare'] = df.total_amount - df.tip_amount
df['tip_percentage'] = df.tip_amount / df.total_fare * 100

"""
Explanation: A useful metric for a taxi driver to compare tips is the percentage of tip given with respect to the total fare amount.
End of explanation
"""


data = df.loc[sample(df.index, 100000)].tip_percentage.values
plt.hist(data, np.arange(min(data)-0.5, max(data)+1.5)) 
plt.gca().set_xlim([0,35])
plt.gca().set_xticks(np.arange(0, 51, 5));
plt.legend(['tip_percentage']);

"""
Explanation: The tip percentage distribution below shows that people mostly seem to tip 0, 20, 25 or 30%.
End of explanation
"""


rows_to_keep = df.tip_percentage <= 100
print 'removing '+ str((~rows_to_keep).sum()) + ' rows...'
df = df[rows_to_keep]

df.tip_percentage.mean()

df.tip_percentage.median()

df.tip_percentage.mode()

df.tip_percentage.quantile(0.25)

# fig, ax = plt.subplots(figsize=(14,5))
# ax = df.loc[sample(df.index, 100000)].tip_percentage.plot(kind='hist',bins=2000, cumulative=True)
# ax.set_xlim([0,200])

"""
Explanation: Remove trips where a tip of more than 100% was recorded, regarding them as invalid outliers.
End of explanation
"""


fig, ax = plt.subplots(figsize=(12, 6))
for i in range(7):
    df[df.pickup_weekday==i].groupby('pickup_hour').mean().plot(y='tip_percentage', ax=ax)
plt.legend(['day ' + str(x) for x in range(7)])
ax.set_ylabel('average tip percentage')

"""
Explanation: Tip percentage by day of the week (Monday=0, Sunday=6). People tend to tip a little less on weekends (day 5-6).
End of explanation
"""


fig, ax = plt.subplots(figsize=(12, 6))
for i in range(7):
    df[df.pickup_weekday==i].groupby('pickup_hour').size().plot(ax=ax)
plt.legend(['day ' + str(x) for x in range(7)])
ax.set_ylabel('number of trips')

"""
Explanation: Let's look at the number of trips per hour and day:
End of explanation
"""


fig, ax = plt.subplots(figsize=(8,7))
df.boxplot('tip_percentage', by='passenger_count', showmeans=True, ax=ax)
ax.set_ylim([15,21])

"""
Explanation: The tip percentage does seem to depend too much on the number of passengers:
End of explanation
"""


df.to_pickle('df.pickle')

"""
Explanation: Save the cleaned data frame to a file:
End of explanation
"""


