%run helper_functions.py
%run filters.py #contain filtration code used in this notebook.
%run plotly_functions.py #wrapper around plot.ly
from datetime import date
from tabulate import tabulate
from collections import Counter
from IPython.display import Image
import math
import string
%matplotlib inline
plt.rcParams["figure.figsize"] = (15,20)
plt.rcParams["xtick.labelsize"] = 16
plt.rcParams["ytick.labelsize"] = 16
plt.rcParams["axes.labelsize"] = 20
plt.rcParams['legend.fontsize'] = 20
plt.style.use('fivethirtyeight')
pd.set_option('display.max_colwidth', -1)
import plotly.plotly as py
import plotly.graph_objs as go

"""
Explanation: Notebook 3.
This notebook will be concerned with EDA of our data.
Furthermore, this notebook will remove further duplicates that survived our first filtration technique in Notebook 2.
End of explanation
"""


df = unpickle_object("no_duplicates_df.pkl")

df.shape

df.head()

percentage_missing(df) # seems that only dates and times are missing for out data! And only 0.2%!

df[df["date"].isnull()].shape

df[df["time"].isnull()].shape

all(df[df["time"].isnull()].index == df[df["date"].isnull()].index) #perfect match for indicies!

df[df["date"].isnull()].iloc[-1, :]

"""
Explanation: In this notebook, we will try to handle missing data!
End of explanation
"""


df[df["date"].isnull()].head()

"""
Explanation: Our last entry in the dataframe has an index of 1049876
End of explanation
"""


1049876 - 1047747 # this is a range of 2129. Which is larger than the total dimensions missing.

#so we know that the missing values are not consecutive!

#these are the handles that have missing dates/times
list(set(df[df["date"].isnull()]['handle']))

"""
Explanation: Our first entry in the dataframe has an index of 1047747
End of explanation
"""


df.dropna(inplace=True)

df.shape

1286 + 610694 # looks like we dropped the correct rows!

"""
Explanation: Having sampled a large amount of the handles above, I found that most accounts were primarily bots or the accounts were suspended, meaning that they were formerly bots. As such, I will just drop all the rows that have missing data for date and time.
End of explanation
"""


df[df['date'] == date(1970,1,1)] #clearly bad rows of data!

to_drop = df[df['date'] == date(1970,1,1)].index

to_drop

df.loc[to_drop, :]

df.drop(to_drop, inplace=True)

df.shape

610694 - 7 # we dropped the right amount!

df.head()

"""
Explanation: We now have rows that have a date of 1970! They most have nonsensical tweets! We will drop these too!
End of explanation
"""


clean_df = filtration_1(df,"clean_tweet_V1", "clean_tweet_V2")

clean_df.head()

clean_df = filtration_2(clean_df, "clean_tweet_V2")

clean_df.head()

clean_df.shape

clean_df.drop_duplicates(subset="clean_tweet_V2", inplace=True)

clean_df.shape #lost around 80K rows! so many duplicates!

"""
Explanation: Let's now clean up our tweets further! This will ensure we dont have garbage hashtags or nonsensical words. This will be important for the lemmatization process later on!
In order to ensure that we have fully removed, duplciates, I will again drop duplicates based on clean_tweet_V2 column.
End of explanation
"""


clean_df.sort_values(by="handle").head(50) #as we can see from the sample, its all nonsense tweets

clean_df.sort_values(by='handle', inplace=True) #lets prep our dataframe for the cleaning process
clean_df.reset_index(inplace=True)
del clean_df['index']
clean_df.head()

clean_df.shape

to_drop = []
for index in clean_df.index:
    if clean_df.iloc[index, 1][0] == "_":#some users have _ at the start of their name
        continue
    if not clean_df.iloc[index, 1][0].isalnum():
        to_drop.append(index)
        

len(to_drop) #we need to drop 212 records

for handle in clean_df.iloc[to_drop, :]['handle']:
    print(handle)
    print()

clean_df.drop(to_drop, inplace=True)

clean_df.head()

clean_df.drop([190], inplace=True)

clean_df.shape

clean_df.sort_values(by=['date', 'time'], inplace=True)
clean_df.reset_index(inplace=True)
del clean_df['index']
clean_df.head()

clean_df_tweet_by_day_plot = bar_graph(clean_df.groupby("date").count()['tweet'].index, clean_df.groupby("date").count()['tweet'], "tweets by day", "Date", "No. Tweets", "tweets-by-day-V2")
clean_df_tweet_by_day_plot #plot looks pretty uniform!

Image("tweets_by_day.png")

diff_df = pd.DataFrame(clean_df.groupby("date").count()['tweet'].diff())
diff_df_plot = line_graph(list(diff_df.index)[2:], list(diff_df['tweet'])[2:],'Difference in tweets by day', 'Date', 'Difference', 'difference-bar-plot')
diff_df_plot # a couple of spikes!

Image("difference_by_day.png")

#Let's find how many unique hashtags we have over a 4 month period
set_of_all_hashtags = set()
counts = {}
for i in range(clean_df.shape[0]):
    temp_lst = clean_df.iloc[i, 6].split()
    hashtags = [x for x in temp_lst if x.startswith("#")]
    for tag in hashtags:
        if tag not in counts:
            counts[tag] = 1
        else:
            counts[tag] += 1
        set_of_all_hashtags.add(tag)
print("There are {} unique hashtags over the course of 4 months".format(len(set_of_all_hashtags)))

# let's visualize the top 50 hashtags
counts_lst = []
for k, v in counts.items():
    counts_lst.append((k, v))

top_50_hashtags = pd.DataFrame(sorted(counts_lst, key=lambda x: x[1], reverse=True)[:50])
top_50_hashtags.set_index(0, inplace=True)
top_50_hashtags.sort_values(by=1, inplace=True)
top_50_hashtags.index.names = ['hashtag']
top_50_hashtags = top_50_hashtags.rename(columns = {1:'count'})

top_50_hashtags_plot = horizontal_bar_graph(top_50_hashtags['count'], top_50_hashtags.index, 'Top 50 hashtags by count', 'Count', 'Hashtag','top-50-hashtags-barh')
top_50_hashtags_plot

Image("top_50_hashtags.png")

"""
Explanation: It seems that our handles column does not strictly contain the user names of twitter handles. Rather, they contain tweets! it is likely that this is the result of bots. As such, we will remove these entries from our dataset!
Also, if the handle contains the word bot, we will also remove it!
End of explanation
"""


stats = pd.DataFrame(clean_df.groupby("date")['tweet'].size().describe())
stats.drop(["count"], inplace=True)
stats = stats.rename(columns = {"tweet":'tweets_per_day'})
print(tabulate(stats, headers='keys', tablefmt='fancy_grid'))

"""
Explanation: Lets look at some stats at the day level:
End of explanation
"""


clean_df['clean_tweet_V2'] = clean_df['clean_tweet_V2'].apply(lambda x: x.replace("#",""))

clean_df.head()

"""
Explanation: Now that we have plotted the hashtags. There is no need to keep the hashtags in the corpus of a particular tweet. In fact, keeping the hashtag would serve to only confuse our sentiment calculations.
End of explanation
"""


hours = []

for index in clean_df.index:
    hours.append(clean_df.iloc[index, 2].hour)

clean_df.shape[0] == len(hours) #perfect

clean_df['hour_of_day'] = hours
clean_df = clean_df.set_value(clean_df[clean_df['hour_of_day'] == 0].index, "hour_of_day", 24)

clean_df.head()

clean_df_tweet_by_hour_plot = line_graph(clean_df.groupby("hour_of_day").count()['tweet'].index, clean_df.groupby("hour_of_day").count()['tweet'], "tweets by hour", "Hour", "No. Tweets", "tweets-by-hour")
clean_df_tweet_by_hour_plot #plot looks pretty uniform!

Image("tweets_by_hour.png")

number_of_tweets = []
number_of_users = []

for k, v in Counter(clean_df.groupby("handle").count()['tweet']).items():
    number_of_tweets.append(k)
    number_of_users.append(v)
scaled_num_users = list(map(lambda x: x/10e3, number_of_users))

tweets_per_user_plot = bar_graph(number_of_tweets, scaled_num_users, "Tweets per user", "Number of tweets per user", "Number of Users (10e3)", "tweet-per-user")
tweets_per_user_plot

Image("tweets_by_user.png") #the first bar has a value of 5.0471 x 10^3

"""
Explanation: While it is excellent that we have such a high level of granularity for our time column, it is not needed for our analysis. Rather, it would be excellent if we could place tweets into "hourly" buckets. This way, we can have analysis for both the day level and the hour level!
End of explanation
"""


clean_df.drop(["handle", "time", "tweet", "tuple_version_tweet", "clean_tweet_V1"], axis=1, inplace=True)

clean_df.head()

pickle_object(clean_df, "clean_df_NB3_Complete")

"""
Explanation: This concludes the exploration notebook! In the next notebook, we will gather some additional data and prepare our data for the modelling process!
As far as model building is concerned, we only need the dates, hours and clean_tweet_V2. Everything else is irrelevant. Let's go ahead and make these changes!
End of explanation
"""


