# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# # Topic Visualization
#
# Visualize topic distributions of Posts from enhanced Topic Model (topic_tokenizer2 and enhanced preprocessing).

# +
import os, sys, ast
sys.path.append("..")
from config import credentials
import dropbox

import numpy as np
import pandas as pd

import warnings
warnings.filterwarnings("ignore", category=DeprecationWarning)

import altair as alt
import matplotlib.pyplot as plt
# %matplotlib inline
# -

random_state = 23

# ## Loading

team_dbx = dropbox.DropboxTeam(credentials.dropbox_team_access_token)
team_root = team_dbx.with_path_root(dropbox.common.PathRoot.namespace_id(
    credentials.dropbox_team_namespace_id))
user_dbx = team_root.as_user(credentials.dropbox_team_member_id)

data_path = "/Data/CSVData"
model_path = "../models"
figures_path = "../reports/figures/"

# +
topics20_fpath = os.path.join(data_path, "topics", "final_topics_LDAn20.csv")
_, res = user_dbx.files_download(topics20_fpath)
topics20 = pd.read_csv(res.raw)

posts_fpath = os.path.join(data_path, "PolPosts.csv")
_, res = user_dbx.files_download(posts_fpath)
posts = pd.read_csv(res.raw)

print("Topics n20", topics20.shape)
print("All Posts", posts.shape)
# -

# ## Postprocess topics
#
# Join Posts with Topic tables and assign topics to duplicated texts.

# +
# Join
data = posts.set_index("textID").join(topics20.set_index("textID"), lsuffix="_posts", rsuffix="_topics")

# Assign topics to duplicated texts
data.topic = data.topic.astype("Int64")
data.topic = data.groupby("text_posts")["topic"].ffill()

# Cleansing
data["source"] = data.Site
data.drop(["text_topics", "Site"], axis=1, inplace=True)
data.dropna(subset=["text_posts", "topic"], inplace=True)
data.dateCreated = data.dateCreated.astype("datetime64[D]")

data.shape
# -

# ## Topic Interpretation

# +
topic_interpretation = {0: "-", 1: "-", 2: "Wohnen", 3: "Grüne",
                        4: "Steuern / Geld", 5: "ÖVP Mobilisierung", 6: "Parteien", 
                        7: "Prognosen", 8: "KPÖ / Linke Forderungen", 9: "Dank / TV-Runde",
                        10: "Links / Alternativ", 11: "ÖVP", 12: "-",
                        13: "SPÖ", 14: "Volksparteien", 15: "-", 16: "FPÖ",
                        17: "-", 18: "Wahlkampfslogans", 19: "-", 20: "Stenzel Rede bei IB"}

topic_color = {"Wohnen": "rosybrown", "Grüne": "green", "Steuern / Geld": "darkblue",
               "ÖVP Mobilisierung": "mediumturquoise", "Parteien": "gold",
               "Prognosen": "darkcyan", "KPÖ / Linke Forderungen": "darkred",
               "Dank / TV-Runde": "palegreen", "Links / Alternativ": "lime",
               "ÖVP": "turquoise", "SPÖ": "red", "Volksparteien": "indigo", "FPÖ": "blue",
               "Wahlkampfslogans": "orange", "Stenzel Rede bei IB": "peru"}


data["topic_interpretation"] = data.topic.map(topic_interpretation)
# -

# ## Exploration & Visualization

# ### Posts per topic

# +
p_data = data.topic.value_counts().to_frame("count").rename_axis("topic").reset_index()
p_data["topic_interpretation"] = p_data.topic.map(topic_interpretation)

chart = alt.Chart(p_data, title="Posts per Topic").mark_bar().encode(
    x=alt.X("topic:N", title="Topic"),
    y=alt.Y("count:Q", title="Number of posts"),
    opacity=alt.value(0.75),
    tooltip=[alt.Tooltip("topic_interpretation", title="Interpretation")]
)

#chart.save(os.path.join(figures_path, "topics", "LDAn20_posts_per_topic.html"))
chart


# -

# Top-20 Posts per Topic

def get_top_20(df, topic, topic_range):
    topic_subset = df[df.topic == topic].copy()
    topic_subset.topic_distribution = topic_subset.topic_distribution.apply(ast.literal_eval)
    flattend_subset = pd.concat([topic_subset,
                                pd.DataFrame(topic_subset.topic_distribution.values.tolist(),
                                             index=topic_subset.index, columns=topic_range)],
                               axis=1)
    return flattend_subset.sort_values(by=[topic], ascending=False).text.head(20)


# +
topics20_top20_fp = "../reports/topics_n20_top-20.txt"
topic_range = range(1, 21)

# with open(topics20_top20_fp, "w") as f:
#     for i in topic_range:
#         with pd.option_context("display.max_colwidth", -1):
#             f.write("Topic " + str(i) + ":\n" + get_top_20(topics20, i, topic_range).to_string(index=False) + "\n\n")
# -

# ### Interactions per topic

# Total interactions

grouped = data.groupby(["topic", "topic_interpretation"]).agg({"interactionCount": ["count", "sum", "mean"]})
pd.set_option("display.max_rows", None)
grouped

# +
grouped.columns = ["count", "sum", "mean"]
p_data = grouped.reset_index().round(2)
p_data["Topic"] = p_data.topic_interpretation
p_data["bar_label"] = ["Total interactions"] * p_data.shape[0]
p_data["mean_label"] = ["Mean interactions"] * p_data.shape[0]

bars = alt.Chart(p_data, title="Interactions per Topic").mark_bar().encode(
    x=alt.X("topic:N", title="Topic"),
    y=alt.Y("sum:Q",
            scale=alt.Scale(type="log", base=10),
            title="Total interactions (log)"),
    opacity=alt.value(0.75),
    tooltip=[alt.Tooltip("Topic", title="Topic"),
             alt.Tooltip("count", title="Posts"),
             alt.Tooltip("sum", title="Total interactions"),
             alt.Tooltip("mean", title="Mean interactions")]
)

mean_ticks = alt.Chart(p_data).mark_tick(color="red", thickness=2).encode(
    x=alt.X("topic:N", title="Topic"),
    y=alt.Y("mean:Q",
            scale=alt.Scale(type="log", base=10),
            title=""),
    color=alt.Color("mean_label",
                    scale=alt.Scale(range=["red"], domain=["Mean interactions"]), 
                    title="Legend"),
    opacity=alt.value(1)
)

chart = bars + mean_ticks
#chart.save(os.path.join(figures_path, "topics", "interactions_per_topic.html"))
chart
# -

# Interactions per source

grouped = data.groupby(["topic", "topic_interpretation", "source"]).agg({"interactionCount": ["count", "sum", "mean"]})
pd.set_option("display.max_rows", None)
grouped

# +
p_data = grouped.copy()
p_data["source"] = p_data.index.get_level_values("source")
p_data.columns = ["count", "sum", "mean", "Source"]
p_data.reset_index(inplace=True)
p_data["Topic"] = p_data.topic_interpretation

chart = alt.Chart(p_data, title="Interactions per Source and Topic").mark_bar().encode(
    alt.X("Source:N"),
    alt.Y("sum(sum):Q", 
          scale=alt.Scale(type="log", base=10),
          title="Interactions (log)"),
    column=alt.Column("topic:O", title="Topic"),
    color=alt.Color("Source", legend=None),
    opacity=alt.value(0.75),
    tooltip=["Topic"]
).properties(
    width=20
)

#chart.save(os.path.join(figures_path, "topics", "interactions_per_source_layered.html"))
chart

# +
p_data = grouped.copy().round(2)
p_data["source"] = p_data.index.get_level_values("source")
p_data.columns = ["count", "sum", "mean", "Source"]
p_data.reset_index(inplace=True)
p_data["Topic"] = p_data.topic_interpretation
p_data["order"] = p_data.source.map({"Facebook": 0, "Twitter": 1})

chart = alt.Chart(p_data, title="Interactions per Topic").mark_bar().encode(
    alt.X("topic:N"),
    alt.Y("sum(sum):Q", 
          title="Interactions"),
    color=alt.Color("Source"),
    order=alt.Order("order"),
    opacity=alt.value(0.75),
    tooltip=[alt.Tooltip("Topic", title="Topic"),
             alt.Tooltip("source", title="Source"),
             alt.Tooltip("count", title="Posts per source"),
             alt.Tooltip("sum", title="Total interactions per source"),
             alt.Tooltip("mean", title="Mean interactions per source")]
)

#chart.save(os.path.join(figures_path, "topics", "interactions_per_source_stacked.html"))
chart

# +
p_data = grouped.copy()
p_data["source"] = p_data.index.get_level_values("source")
p_data.columns = ["count", "sum", "mean", "Source"]
p_data.reset_index(inplace=True)
p_data["Topic"] = p_data.topic_interpretation

chart = alt.Chart(p_data, title="Interactions per Source and Topic").mark_bar().encode(
    alt.X("Source:N"),
    alt.Y("sum(mean):Q", 
          scale=alt.Scale(type="log", base=10),
          title="Mean interactions (log)"),
    column=alt.Column("topic:O", title="Topic"),
    color=alt.Color("Source", legend=None),
    opacity=alt.value(0.75),
    tooltip=["Topic"]
).properties(
    width=20
)

#chart.save(os.path.join(figures_path, "topics", "mean_interactions_per_source.html"))
chart
# -

# ### Topics Timeline
#
# Number of Posts per topic over monitoring period

grouped = data.groupby(["topic", "topic_interpretation", "dateCreated"]).agg({"interactionCount": ["count", "sum", "mean"]})
pd.set_option("display.max_rows", None)
#grouped

# +
grouped.columns = ["count", "sum", "mean"]
# drop not interpreted topics, will be merged to one uber-topic otherwise
grouped_cleaned = grouped.drop("-", level="topic_interpretation")
p_data = grouped_cleaned.reset_index().round(2)
p_data["Topic"] = p_data.topic_interpretation
p_data["color"] = p_data.topic_interpretation.map(topic_color) 

highlight = alt.selection(type="single", on="mouseover", fields=["Topic"],
                          nearest=True)

base = alt.Chart(p_data, title="Topics over Time").mark_line().encode(
    x=alt.X("monthdate(dateCreated):O", title="Date"),
    y=alt.Y("count:Q", title="# of Posts",
            scale=alt.Scale(domain=(0, 170))),
    #color=alt.Color("Topic:N", scale=alt.Scale(scheme="tableau20")),
    color=alt.Color("color", scale=None),
    tooltip=[alt.Tooltip("Topic", title="Topic"),
             alt.Tooltip("dateCreated", title="Day"),
             alt.Tooltip("count", title="Posts")]
)

points = base.mark_circle().encode(
    opacity=alt.value(0)
).add_selection(
    highlight
).properties(
    width=800,
    height=500
)

lines = base.mark_line().encode(
    size=alt.condition(~highlight, alt.value(0.5), alt.value(4))
)

chart = points + lines
#chart.save(os.path.join(figures_path, "topics", "topics_timeline.html"))
chart
# -



StarcoderdataJupyterScriptsDedupFiltered