# https://huggingface.co/docs/hub/en/spaces-github-actions import os import time import pandas as pd import streamlit as st from opendashboards.assets import io, inspect, metric, plot # prompt-based completion score stats # instrospect specific RUN-UID-COMPLETION # cache individual file loads # Hotkey # TODO: limit the historical lookup to something reasonable (e.g. 30 days) # TODO: Add sidebar for filters such as tags, hotkeys, etc. # TODO: Show trends for runs (versions, hotkeys, etc.). An area chart would be nice, a gantt chart would be better # TODO: Add a search bar for runs # TODO: Find a reason to make a pie chart (task distribution, maybe) # TODO: remove repetition plots (it's not really a thing any more) # TODO: MINER SKILLSET STAR CHART # TODO: Status codes for runs vs time (from analysis notebook) WANDB_PROJECT = "opentensor-dev/alpha-validators" PROJECT_URL = f'https://wandb.ai/{WANDB_PROJECT}/table?workspace=default' MAX_RECENT_RUNS = 300 DEFAULT_FILTERS = {}#{"tags": {"$in": [f'1.1.{i}' for i in range(10)]}} DEFAULT_SELECTED_HOTKEYS = None DEFAULT_TASK = 'qa' DEFAULT_COMPLETION_NTOP = 10 DEFAULT_UID_NTOP = 10 # Set app config st.set_page_config( page_title='Validator Dashboard', menu_items={ 'Report a bug': "https://github.com/opentensor/dashboards/issues", 'About': f""" This dashboard is part of the OpenTensor project. \n To see runs in wandb, go to: \n [Wandb Table](https://wandb.ai/{WANDB_PROJECT}/table?workspace=default) \n """ }, layout = "centered" ) st.title('Validator :red[Analysis] Dashboard :eyes:') # add vertical space st.markdown('#') st.markdown('#') with st.spinner(text=f'Checking wandb...'): df_runs = io.load_runs(project=WANDB_PROJECT, filters=DEFAULT_FILTERS, min_steps=10, max_recent=MAX_RECENT_RUNS) metric.wandb(df_runs) # add vertical space st.markdown('#') st.markdown('#') tab1, tab2, tab3, tab4 = st.tabs(["Run Data", "UID Health", "Completions", "Prompt-based scoring"]) ### Wandb Runs ### with tab1: st.markdown('#') st.subheader(":violet[Run] Data") # make multiselect for run_ids with label on same line run_ids = st.multiselect('Select one or more weights and biases run by id:', df_runs['run_id'], key='run_id', default=df_runs['run_id'][:3], help=f'Select one or more runs to analyze. You can find the raw data for these runs [here]({PROJECT_URL}).') n_runs = len(run_ids) df_runs_subset = df_runs[df_runs['run_id'].isin(run_ids)] with st.expander(f'Select from :violet[all] wandb runs'): edited_df = st.data_editor( df_runs.assign(Select=False).set_index('Select'), column_config={"Select": st.column_config.CheckboxColumn(required=True)}, disabled=df_runs.columns, use_container_width=True, ) if edited_df.index.any(): df_runs_subset = df_runs[edited_df.index==True] n_runs = len(df_runs_subset) if n_runs: df = io.load_data(df_runs_subset, load=True, save=True) df = inspect.clean_data(df) print(f'\nNans in columns: {df.isna().sum()}') df_long = inspect.explode_data(df) if 'rewards' in df_long: df_long['rewards'] = df_long['rewards'].astype(float) else: st.info(f'You must select at least one run to load data') st.stop() metric.runs(df_long) timeline_color = st.radio('Color by:', ['state', 'version', 'netuid'], key='timeline_color', horizontal=True) plot.timeline(df_runs, color=timeline_color) st.markdown('#') st.subheader(":violet[Event] Data") with st.expander(f'Show :violet[raw] event data for **{n_runs} selected runs**'): raw_data_col1, raw_data_col2 = st.columns(2) use_long_checkbox = raw_data_col1.checkbox('Use long format', value=True) num_rows = raw_data_col2.slider('Number of rows:', min_value=1, max_value=100, value=10, key='num_rows') st.dataframe(df_long.head(num_rows) if use_long_checkbox else df.head(num_rows), use_container_width=True) # step_types = ['all']+['augment','followup','answer']#list(df.name.unique()) step_types = ['all']+list(df.task.unique()) ### UID Health ### # TODO: Live time - time elapsed since moving_averaged_score for selected UID was 0 (lower bound so use >Time) # TODO: Weight - Most recent weight for selected UID (Add warning if weight is 0 or most recent timestamp is not current) with tab2: st.markdown('#') st.subheader("UID :violet[Health]") st.info(f"Showing UID health metrics for **{n_runs} selected runs**") uid_src = st.radio('Select task type:', step_types, horizontal=True, key='uid_src') df_uid = df_long[df_long.task.str.contains(uid_src)] if uid_src != 'all' else df_long metric.uids(df_uid, uid_src) uids = st.multiselect('UID:', sorted(df_uid['uids'].unique()), key='uid') with st.expander(f'Show UID health data for **{n_runs} selected runs** and **{len(uids)} selected UIDs**'): st.markdown('#') st.subheader(f"UID {uid_src.title()} :violet[Health]") agg_uid_checkbox = st.checkbox('Aggregate UIDs', value=True) if agg_uid_checkbox: metric.uids(df_uid, uid_src, uids) else: for uid in uids: st.caption(f'UID: {uid}') metric.uids(df_uid, uid_src, [uid]) st.subheader(f'Cumulative completion frequency') freq_col1, freq_col2 = st.columns(2) freq_ntop = freq_col1.slider('Number of Completions:', min_value=10, max_value=1000, value=100, key='freq_ntop') freq_rm_empty = freq_col2.checkbox('Remove empty (failed)', value=True, key='freq_rm_empty') freq_cumulative = freq_col2.checkbox('Cumulative', value=False, key='freq_cumulative') freq_normalize = freq_col2.checkbox('Normalize', value=True, key='freq_normalize') plot.uid_completion_counts(df_uid, uids=uids, src=uid_src, ntop=freq_ntop, rm_empty=freq_rm_empty, cumulative=freq_cumulative, normalize=freq_normalize) with st.expander(f'Show UID **{uid_src}** leaderboard data for **{n_runs} selected runs**'): st.markdown('#') st.subheader(f"UID {uid_src.title()} :violet[Leaderboard]") uid_col1, uid_col2 = st.columns(2) uid_ntop = uid_col1.slider('Number of UIDs:', min_value=1, max_value=50, value=DEFAULT_UID_NTOP, key='uid_ntop') uid_agg = uid_col2.selectbox('Aggregation:', ('mean','min','max','size','nunique'), key='uid_agg') plot.leaderboard( df_uid, ntop=uid_ntop, group_on='uids', agg_col='rewards', agg=uid_agg ) with st.expander(f'Show UID **{uid_src}** diversity data for **{n_runs} selected runs**'): st.markdown('#') st.subheader(f"UID {uid_src.title()} :violet[Diversity]") rm_failed = st.checkbox(f'Remove failed **{uid_src}** completions', value=True) plot.uid_diversty(df, rm_failed) ### Completions ### with tab3: st.markdown('#') st.subheader('Completion :violet[Leaderboard]') completion_info = st.empty() msg_col1, msg_col2 = st.columns(2) # completion_src = msg_col1.radio('Select one:', ['followup', 'answer'], horizontal=True, key='completion_src') completion_src = st.radio('Select task type:', step_types, horizontal=True, key='completion_src') df_comp = df_long[df_long.task.str.contains(completion_src)] if completion_src != 'all' else df_long completion_info.info(f"Showing **{completion_src}** completions for **{n_runs} selected runs**") completion_ntop = msg_col2.slider('Top k:', min_value=1, max_value=50, value=DEFAULT_COMPLETION_NTOP, key='completion_ntop') completions = inspect.completions(df_long, 'completions') # Get completions with highest average rewards plot.leaderboard( df_comp, ntop=completion_ntop, group_on='completions', agg_col='rewards', agg='mean', alias=True ) with st.expander(f'Show **{completion_src}** completion rewards data for **{n_runs} selected runs**'): st.markdown('#') st.subheader('Completion :violet[Rewards]') completion_select = st.multiselect('Completions:', completions.index, default=completions.index[:3].tolist()) # completion_regex = st.text_input('Completion regex:', value='', key='completion_regex') plot.completion_rewards( df_comp, completion_col='completions', reward_col='rewards', uid_col='uids', ntop=completion_ntop, completions=completion_select, ) # TODO: show the UIDs which have used the selected completions with st.expander(f'Show **{completion_src}** completion length data for **{n_runs} selected runs**'): st.markdown('#') st.subheader('Completion :violet[Length]') completion_length_radio = st.radio('Use: ', ['characters','words','sentences'], key='completion_length_radio') # Todo: use color to identify selected completions/ step names/ uids plot.completion_length_time( df_comp, completion_col='completions', uid_col='uids', time_col='timings', length_opt=completion_length_radio, ) ### Prompt-based scoring ### with tab4: # coming soon st.info('Prompt-based scoring coming soon') st.snow() # st.dataframe(df_long_long.filter(regex=prompt_src).head())