df_merged_meta.t_id.value_counts().head()

df_merged_meta[df_merged_meta.t_id == 700042121877835776][["topic_name"]]

df_merged_meta.t_id.value_counts()[df_merged_meta.t_id.value_counts() > 1]

df_merged_meta[df_merged_meta.t_id == 792354716521009152].T

df_merged_meta["is_controversial"] = df_merged_meta.topic_name.isin(CONTROVERSIAL_TOPICS)
df_merged_meta.is_controversial.value_counts()

"""
Explanation: Same tweet belongs to multiple datasets
End of explanation
"""


df_merged_meta.columns

df_mapped_cats = pd.read_csv("TID_URL_CATS.txt", sep="\t").assign(
    CATS=lambda x: x.CATS.apply(lambda k: k.split("|"))
)
df_mapped_cats.head()

URL_DICT = dict(zip(df_mapped_cats.URL.values, df_mapped_cats.CATS.values))
URL_DICT["http://TinyURL.com/NewYearCure"]

len(URL_DICT)

df_mapped_cats.TID.value_counts().head()

df_mapped_cats[df_mapped_cats.TID == 700152617033289728]

df_tweet_cat_counts = df_mapped_cats.groupby("TID")["CATS"].apply(lambda x: sum(x, []))
df_tweet_cat_counts.head()

df_tweet_cat_counts.reset_index().dtypes

df_merged_meta.shape

df_merged_meta.t_id.value_counts().head()

df_merged_meta_cats = df_merged_meta.merge(
    df_tweet_cat_counts.reset_index(), how="left", left_on="t_id", right_on="TID")

df_merged_meta_cats.columns

"""
Explanation: Merge URL types
End of explanation
"""


df_places = pd.read_csv("PARSED_STATES.final.txt", sep="\t")
df_places = df_places.rename(columns={
        "location": "u_location", "parse_manual": "u_state"
                                     })[["u_location", "u_state"]]
df_places.head()

df_merged_meta_cats = df_merged_meta_cats.merge(df_places, how="left", on="u_location")

df_merged_meta_cats.u_state.head()

df_merged_meta_cats.t_id.value_counts().head()

df_merged_meta_cats.to_hdf("FINAL_ANALYSIS_DATA.h5", "final_data")

"""
Explanation: Add location states
df_merged_meta_cats.u_location.value_counts().to_csv("USER_LOCATIONS.txt", sep="\t", encoding='utf-8')
! head USER_LOCATIONS.txt
! python process_user_locations.py ## RUN using python3 from command line
End of explanation
"""


