Lvl up

dashboard.py

@@ -1,13 +1,16 @@
+import time
+import pandas as pd
 import streamlit as st
 from opendashboards.assets import io, inspect, metric, plot
 
-# dendrite time versus completion length
 # prompt-based completion score stats
 # instrospect specific RUN-UID-COMPLETION
+# cache individual file loads
 
 DEFAULT_PROJECT = "openvalidators"
 DEFAULT_FILTERS = {"tags": {"$in": ["1.0.0", "1.0.1", "1.0.2", "1.0.3", "1.0.4"]}}
 DEFAULT_SELECTED_RUNS = ['kt9bzxii']
+DEFAULT_SELECTED_HOTKEYS = None
 DEFAULT_SRC = 'followup'
 DEFAULT_COMPLETION_NTOP = 10
 DEFAULT_UID_NTOP = 10
@@ -31,59 +34,80 @@ st.title('Validator :red[Analysis] Dashboard :eyes:')
 st.markdown('#')
 st.markdown('#')
 
-# with st.sidebar:
-#     st.sidebar.header('Pages')
 
 with st.spinner(text=f'Checking wandb...'):
     df_runs = io.load_runs(project=DEFAULT_PROJECT, filters=DEFAULT_FILTERS, min_steps=10)
 
+
+### Wandb Runs ###
+with st.sidebar:
+
+    st.markdown('#')
+    st.sidebar.header(":violet[Select] Runs")
+
+    df_runs_subset = io.filter_dataframe(df_runs, demo_selection=df_runs.id.isin(DEFAULT_SELECTED_RUNS))
+    n_runs = len(df_runs_subset)
+
 metric.wandb(df_runs)
 
+
 # add vertical space
 st.markdown('#')
 st.markdown('#')
 
-tab1, tab2, tab3, tab4 = st.tabs(["Wandb Runs", "UID Health", "Completions", "Prompt-based scoring"])
-
+tab1, tab2, tab3, tab4 = st.tabs(["Raw Data", "UID Health", "Completions", "Prompt-based scoring"])
 
 ### Wandb Runs ###
 with tab1:
 
     st.markdown('#')
-    st.header(":violet[Wandb] Runs")
-
-    run_msg = st.info("Select a single run or compare multiple runs")
-    selected_runs = st.multiselect(f'Runs ({len(df_runs)})', df_runs.id, default=DEFAULT_SELECTED_RUNS, key='runs')
+    st.subheader(":violet[Run] Data")
+    with st.expander(f'Show :violet[raw] wandb data'):
+
+        filter_selected_checkbox = st.checkbox('Filter to selected runs', value=True)
+        df_to_show = df_runs_subset if filter_selected_checkbox else df_runs
+
+        # TODO: make this editable so that runs can be selected directly from the table
+        st.dataframe(
+            df_to_show.assign(
+                Selected=df_to_show.index.isin(df_runs_subset.index)
+            ).set_index('Selected').sort_index(ascending=False),#.style.highlight_max(subset=df_runs_subset.index, color='lightgreen', axis=1),
+            use_container_width=True,
+        )
 
-    # Load data if new runs selected
-    if not selected_runs:
-        # open a dialog to select runs
-        run_msg.error("Please select at least one run")
-        st.snow()
+    if n_runs:
+        df = io.load_data(df_runs_subset, load=True, save=True)
+        df_long = inspect.explode_data(df)
+        df_weights = inspect.weights(df)
+    else:
+        st.info(f'You must select at least one run to load data')
         st.stop()
 
-    df = io.load_data(df_runs.loc[df_runs.id.isin(selected_runs)], load=True, save=True)
-    df_long = inspect.explode_data(df)
-    df_weights = inspect.weights(df)
+    metric.runs(df_long)
 
-    metric.runs(df, df_long, selected_runs)
-
-    with st.expander(f'Show :violet[raw] data for {len(selected_runs)} selected runs'):
-        inspect.run_event_data(df_runs,df, selected_runs)
+    st.markdown('#')
+    st.subheader(":violet[Event] Data")
+    with st.expander(f'Show :violet[raw] event data for **{n_runs} selected runs**'):
+        raw_data_col1, raw_data_col2 = st.columns(2)
+        use_long_checkbox = raw_data_col1.checkbox('Use long format', value=True)
+        num_rows = raw_data_col2.slider('Number of rows:', min_value=1, max_value=100, value=10, key='num_rows')
+        st.dataframe(df_long.head(num_rows) if use_long_checkbox else df.head(num_rows),
+                     use_container_width=True)
+        
 
 
 ### UID Health ###
 with tab2:
 
     st.markdown('#')
-    st.header("UID :violet[Health]")
-    st.info(f"Showing UID health metrics for **{len(selected_runs)} selected runs**")
+    st.subheader("UID :violet[Health]")
+    st.info(f"Showing UID health metrics for **{n_runs} selected runs**")
 
     uid_src = st.radio('Select one:', ['followup', 'answer'], horizontal=True, key='uid_src')
 
     metric.uids(df_long, uid_src)
 
-    with st.expander(f'Show UID **{uid_src}** weights data for **{len(selected_runs)} selected runs**'):
+    with st.expander(f'Show UID **{uid_src}** weights data for **{n_runs} selected runs**'):
 
         uids = st.multiselect('UID:', sorted(df_long[f'{uid_src}_uids'].unique()), key='uid')
         st.markdown('#')
@@ -93,8 +117,8 @@ with tab2:
                 df_weights,
                 uids=uids,
         )
-        
-    with st.expander(f'Show UID **{uid_src}** leaderboard data for **{len(selected_runs)} selected runs**'):
+
+    with st.expander(f'Show UID **{uid_src}** leaderboard data for **{n_runs} selected runs**'):
 
         st.markdown('#')
         st.subheader(f"UID {uid_src.title()} :violet[Leaderboard]")
@@ -111,7 +135,7 @@ with tab2:
             )
 
 
-    with st.expander(f'Show UID **{uid_src}** diversity data for **{len(selected_runs)} selected runs**'):
+    with st.expander(f'Show UID **{uid_src}** diversity data for **{n_runs} selected runs**'):
 
         st.markdown('#')
         st.subheader(f"UID {uid_src.title()} :violet[Diversity]")
@@ -128,13 +152,14 @@ with tab3:
 
     msg_col1, msg_col2 = st.columns(2)
     completion_src = msg_col1.radio('Select one:', ['followup', 'answer'], horizontal=True, key='completion_src')
-    completion_info.info(f"Showing **{completion_src}** completions for **{len(selected_runs)} selected runs**")
-    
+    completion_info.info(f"Showing **{completion_src}** completions for **{n_runs} selected runs**")
+
     completion_ntop = msg_col2.slider('Top k:', min_value=1, max_value=50, value=DEFAULT_COMPLETION_NTOP, key='completion_ntop')
 
     completion_col = f'{completion_src}_completions'
     reward_col = f'{completion_src}_rewards'
     uid_col = f'{completion_src}_uids'
+    time_col = f'{completion_src}_times'
 
     completions = inspect.completions(df_long, completion_col)
 
@@ -148,7 +173,7 @@ with tab3:
         alias=True
     )
 
-    with st.expander(f'Show **{completion_src}** completion rewards data for **{len(selected_runs)} selected runs**'):
+    with st.expander(f'Show **{completion_src}** completion rewards data for **{n_runs} selected runs**'):
 
         st.markdown('#')
         st.subheader('Completion :violet[Rewards]')
@@ -166,10 +191,26 @@ with tab3:
         )
 
 
+    with st.expander(f'Show **{completion_src}** completion length data for **{n_runs} selected runs**'):
+
+        st.markdown('#')
+        st.subheader('Completion :violet[Length]')
+
+        words_checkbox = st.checkbox('Use words', value=True, key='words_checkbox')
+
+        plot.completion_length_time(
+            df,
+            completion_col=completion_col,
+            uid_col=uid_col,
+            time_col=time_col,
+            words=words_checkbox,
+        )
+
 ### Prompt-based scoring ###
 with tab4:
     # coming soon
     st.info('Prompt-based scoring coming soon')
+    st.snow()
 
     # st.dataframe(df_long_long.filter(regex=prompt_src).head())
 

opendashboards/assets/inspect.py

@@ -51,4 +51,7 @@ def run_event_data(df_runs, df, selected_runs):
                     column_config={
                         "url": st.column_config.LinkColumn("URL"),
                     }
-        )
+        )
+        
+def highlight_row(row, expr, color='lightgrey', bg_color='white'):
+    return [f'background-color:{color}' if expr else f'background-color:{bg_color}'] * len(row)

opendashboards/assets/io.py

@@ -5,6 +5,13 @@ import streamlit as st
 
 import  opendashboards.utils.utils as utils
 
+from pandas.api.types import (
+    is_categorical_dtype,
+    is_datetime64_any_dtype,
+    is_numeric_dtype,
+    is_object_dtype,
+)
+
 
 @st.cache_data
 def load_runs(project, filters, min_steps=10):
@@ -94,3 +101,92 @@ def load_data(selected_runs, load=True, save=False):
     return pd.concat(frames)
 
 
+def filter_dataframe(df: pd.DataFrame, demo_selection=None) -> pd.DataFrame:
+    """
+    Adds a UI on top of a dataframe to let viewers filter columns
+
+    Args:
+        df (pd.DataFrame): Original dataframe
+        demo_selection (pd.Index): Index of runs to select (if demo)
+
+    Returns:
+        pd.DataFrame: Filtered dataframe
+    """
+    filter_mode = st.sidebar.radio("Filter mode", ("Use demo", "Add filters"), index=0)
+
+    run_msg = st.info("Select a single wandb run or compare multiple runs")
+
+    if filter_mode == "Use demo":
+        df = df.loc[demo_selection]
+        run_msg.info(f"Selected {len(df)} runs")
+        return df
+    
+    df = df.copy()
+
+    # Try to convert datetimes into a standarrd format (datetime, no timezone)
+    for col in df.columns:
+        if is_object_dtype(df[col]):
+            try:
+                df[col] = pd.to_datetime(df[col])
+            except Exception:
+                pass
+
+        if is_datetime64_any_dtype(df[col]):
+            df[col] = df[col].dt.tz_localize(None)
+
+    modification_container = st.container()
+
+    with modification_container:
+        to_filter_columns = st.multiselect("Filter dataframe on", df.columns)
+        for column in to_filter_columns:
+            left, right = st.columns((1, 20))
+            # Treat columns with < 10 unique values as categorical
+            if is_categorical_dtype(df[column]) or df[column].nunique() < 10:
+                user_cat_input = right.multiselect(
+                    f"Values for {column}",
+                    df[column].unique(),
+                    default=list(df[column].unique()),
+                )
+                df = df[df[column].isin(user_cat_input)]
+            elif is_numeric_dtype(df[column]):
+                _min = float(df[column].min())
+                _max = float(df[column].max())
+                step = (_max - _min) / 100
+                user_num_input = right.slider(
+                    f"Values for {column}",
+                    min_value=_min,
+                    max_value=_max,
+                    value=(_min, _max),
+                    step=step,
+                )
+                df = df[df[column].between(*user_num_input)]
+            elif is_datetime64_any_dtype(df[column]):
+                user_date_input = right.date_input(
+                    f"Values for {column}",
+                    value=(
+                        df[column].min(),
+                        df[column].max(),
+                    ),
+                )
+                if len(user_date_input) == 2:
+                    user_date_input = tuple(map(pd.to_datetime, user_date_input))
+                    start_date, end_date = user_date_input
+                    df = df.loc[df[column].between(start_date, end_date)]
+            else:
+                user_text_input = right.text_input(
+                    f"Substring or regex in {column}",
+                )
+                if user_text_input:
+                    df = df[df[column].astype(str).str.contains(user_text_input)]
+
+
+    # Load data if new runs selected
+    if len(df):
+        run_msg.info(f"Selected {len(df)} runs")
+    else:
+        # open a dialog to select runs
+        run_msg.error("Please select at least one run")
+        # st.snow()
+        # st.stop()
+
+    return df

opendashboards/assets/metric.py

@@ -18,11 +18,11 @@ def wandb(df_runs):
 
 
 @st.cache_data
-def runs(df, df_long, selected_runs):
+def runs(df_long):
 
     col1, col2, col3 = st.columns(3)
-    col1.metric(label="Runs", value=len(selected_runs))
-    col1.metric(label="Events", value=df.shape[0]) #
+    col1.metric(label="Runs", value=df_long.run_id.nunique())
+    col1.metric(label="Events", value=df_long.shape[0]) 
     col2.metric(label="Followup UIDs", value=df_long.followup_uids.nunique())
     col2.metric(label="Answer UIDs", value=df_long.answer_uids.nunique())
     col3.metric(label="Followup Completions", value=df_long.followup_completions.nunique())

opendashboards/assets/plot.py

@@ -53,4 +53,16 @@ def weights(df, uids, ntop=10):
             ntop=ntop
         ),
         use_container_width=True
+    )
+
+def completion_length_time(df, completion_col, uid_col, time_col, words=False):
+    return st.plotly_chart(
+        plotting.plot_completion_length_time(
+            df,
+            uid_col=uid_col,
+            completion_col=completion_col,
+            time_col=time_col,
+            words=words
+        ),
+        use_container_width=True
     )

opendashboards/utils/plotting.py

@@ -97,8 +97,8 @@ def plot_uid_diversty(df: pd.DataFrame, remove_unsuccessful: bool = False) -> go
         merged,
         x="diversity_followup",
         y="diversity_answer",
-        opacity=0.3,
-        size="followup_completions_size",
+        opacity=0.35,
+        # size="followup_completions_size",
         color="reward_mean",
         hover_data=["UID"] + merged.columns.tolist(),
         marginal_x="histogram",
@@ -219,7 +219,7 @@ def plot_completion_rewards(
         labels={"rank": "Rank", reward_col: "Reward", time_col: ""},
         title=f"Rewards for {len(completions)} Messages",
         **plotly_config,
-        opacity=0.3,
+        opacity=0.35,
     )
 
 
@@ -258,12 +258,13 @@ def plot_leaderboard(
         labels={"x": f"{agg_col.title()}", "y": group_on, "color": ""},
         title=f"Leaderboard for {agg_col}, top {ntop} {group_on}",
         color_continuous_scale="BlueRed",
-        opacity=0.5,
+        opacity=0.35,
         hover_data=[rankings.index.astype(str)],
         **plotly_config,
     )
 
 
+
 def plot_dendrite_rates(
     df: pd.DataFrame, uid_col: str = "answer_uids", reward_col: str = "answer_rewards", ntop: int = 20, uids: List[int] = None
 ) -> go.Figure:
@@ -297,10 +298,38 @@ def plot_dendrite_rates(
         barmode="group",
         title="Dendrite Calls by UID",
         color_continuous_scale="Blues",
-        opacity=0.5,
+        opacity=0.35,
         **plotly_config,
     )
 
+def plot_completion_length_time(
+    df: pd.DataFrame,
+    uid_col: str = "answer_uids",
+    completion_col: str = "answer_completions",
+    time_col: str = "answer_times",
+    words: bool = False,    
+) -> go.Figure:
+    
+    df = df[[uid_col, completion_col, time_col]].explode(column=[uid_col, completion_col, time_col])
+    df["time"] = df[time_col].astype(float)
+    if words:
+        df["completion_length"] = df[completion_col].str.split().str.len()
+    else:
+        df["completion_length"] = df[completion_col].str.len()
+    
+    return px.scatter(
+        df,
+        x='completion_length',
+        y='time',
+        labels={"completion_length": f"Completion Length, {'Words' if words else 'Characters'}", "time": "Time (s)"},
+        title=f"Completion Length vs Time, {'Words' if words else 'Characters'}",
+        marginal_x="histogram",
+        marginal_y="histogram",
+        hover_data=[uid_col, completion_col],
+        opacity=0.35,
+        **plotly_config,
+    )
+            
 
 def plot_network_embedding(
     df: pd.DataFrame,
@@ -358,6 +387,6 @@ def plot_network_embedding(
         title=f"Graph for Top {ntop} Completion Similarities",
         color_continuous_scale="BlueRed",
         hover_data=["UID", "top_completions"],
-        opacity=0.5,
+        opacity=0.35,
         **plotly_config,
     )