app.py

import abc
import gradio as gr
import os

from gen_table import *
from meta_data import *



with gr.Blocks(title="Open Agent Leaderboard") as demo:
    struct = load_results(OVERALL_MATH_SCORE_FILE)
    timestamp = struct['time']
    EVAL_TIME = format_timestamp(timestamp)
    results = struct['results']
    N_MODEL = len(results)
    N_DATA = len(results['IO'])
    DATASETS = list(results['IO'])
    DATASETS.remove('META')
    print(DATASETS)

    # Ensure overall_table is generated before defining llm_options
    check_box = BUILD_L1_DF(results, DEFAULT_MATH_BENCH)
    overall_table = generate_table(results, DEFAULT_MATH_BENCH)

    # Save the complete overall_table as a CSV file
    csv_path_overall = os.path.join(os.getcwd(), 'src/overall_results.csv')
    overall_table.to_csv(csv_path_overall, index=False)
    print(f"Overall results saved to {csv_path_overall}")

    # Extract all possible LLM options from overall_table
    llm_options = list(set(row.LLM for row in overall_table.itertuples() if hasattr(row, 'LLM')))
    
    gr.Markdown(LEADERBORAD_INTRODUCTION.format(EVAL_TIME))
    with gr.Tabs(elem_classes='tab-buttons') as tabs:
        with gr.Tab(label='🏅 Open Agent Overall Math Leaderboard'):
            gr.Markdown(LEADERBOARD_MD['MATH_MAIN'])
            # Move the definition of check_box and overall_table here
            # check_box = BUILD_L1_DF(results, DEFAULT_MATH_BENCH)
            # overall_table = generate_table(results, DEFAULT_MATH_BENCH)

            type_map = check_box['type_map']
            type_map['Rank'] = 'number'

            checkbox_group = gr.CheckboxGroup(
                choices=check_box['all'],
                value=check_box['required'],
                label='Evaluation Dimension',
                interactive=True,
            )

            # New CheckboxGroup component for selecting Algorithm and LLM
            algo_name = gr.CheckboxGroup(
                choices=ALGORITHMS,
                value=ALGORITHMS,
                label='Algorithm',
                interactive=True
            )

            llm_name = gr.CheckboxGroup(
                choices=llm_options,  # Use the extracted llm_options
                value=llm_options,
                label='LLM',
                interactive=True
            )

            initial_headers = ['Rank'] + check_box['essential'] + checkbox_group.value
            available_headers = [h for h in initial_headers if h in overall_table.columns]

            data_component = gr.components.DataFrame(
                value=overall_table[available_headers],  
                type='pandas',
                datatype=[type_map[x] for x in available_headers],
                interactive=False,
                wrap=True,
                visible=True)
            
            def filter_df(fields, algos, llms, *args):
                headers = ['Rank'] + check_box['essential'] + fields
                df = overall_table.copy()

                # Add filtering logic
                df['flag'] = df.apply(lambda row: (
                    row['Algorithm'] in algos and 
                    row['LLM'] in llms
                ), axis=1)
                
                df = df[df['flag']].copy()
                df.pop('flag')

                # Ensure all requested columns exist
                available_headers = [h for h in headers if h in df.columns]

                original_columns = df.columns.tolist()
                available_headers = sorted(available_headers, key=lambda x: original_columns.index(x))

                # If no columns are available, return an empty DataFrame with basic columns
                if not available_headers:
                    available_headers = ['Rank'] + check_box['essential']
                
                comp = gr.components.DataFrame(
                    value=df[available_headers],
                    type='pandas',
                    datatype=[type_map[x] for x in available_headers],
                    interactive=False,
                    wrap=True,
                    visible=True)

                return comp

            # Update change events to include new filtering conditions
            checkbox_group.change(
                fn=filter_df, 
                inputs=[checkbox_group, algo_name, llm_name], 
                outputs=data_component
            )

            algo_name.change(
                fn=filter_df,
                inputs=[checkbox_group, algo_name, llm_name],
                outputs=data_component
            )

            llm_name.change(
                fn=filter_df,
                inputs=[checkbox_group, algo_name, llm_name],
                outputs=data_component
            )

        with gr.Tab(label='🏅 Open Agent Detail Math Leaderboard'):
            gr.Markdown(LEADERBOARD_MD['MATH_DETAIL'])
            struct_detail = load_results(DETAIL_MATH_SCORE_FILE)
            timestamp = struct_detail['time']
            EVAL_TIME = format_timestamp(timestamp)
            results_detail = struct_detail['results']

            table, check_box = BUILD_L2_DF(results_detail, DEFAULT_MATH_BENCH)

            # Save the complete table as a CSV file
            csv_path_detail = os.path.join(os.getcwd(), 'src/detail_results.csv')
            table.to_csv(csv_path_detail, index=False)
            print(f"Detail results saved to {csv_path_detail}")

            type_map = check_box['type_map']
            type_map['Rank'] = 'number'

            checkbox_group = gr.CheckboxGroup(
                choices=check_box['all'],
                value=check_box['required'],
                label='Evaluation Dimension',
                interactive=True,
            )

            headers = ['Rank'] + checkbox_group.value
            with gr.Row():

                algo_name = gr.CheckboxGroup(
                        choices=ALGORITHMS,
                        value=ALGORITHMS,
                        label='Algorithm',
                        interactive=True
                    )

                dataset_name = gr.CheckboxGroup(
                        choices=DATASETS,
                        value=DATASETS,
                        label='Datasets',
                        interactive=True
                    )

            llm_name = gr.CheckboxGroup(
                    choices=check_box['LLM_options'],
                    value=check_box['LLM_options'],
                    label='LLM',
                    interactive=True
                )
                
            data_component = gr.components.DataFrame(
                value=table[headers],
                type='pandas',
                datatype=[type_map[x] for x in headers],
                interactive=False,
                wrap=True,
                visible=True)
            
            def filter_df2(fields, algos, datasets, llms):
                headers =  ['Rank'] + fields
                df = table.copy()
                
                # Filter data
                df['flag'] = df.apply(lambda row: (
                    row['Algorithm'] in algos and 
                    row['Dataset'] in datasets and 
                    row['LLM'] in llms
                ), axis=1)
                
                df = df[df['flag']].copy()
                df.pop('flag')
                
                # Group by dataset and calculate ranking within each group based on Score
                if 'Score' in df.columns:
                    # Create a temporary ranking column
                    df['Rank'] = df.groupby('Dataset')['Score'].rank(method='first', ascending=False)
                    
                    # Ensure ranking is integer
                    df['Rank'] = df['Rank'].astype(int)
                
                
                original_columns = df.columns.tolist()
                headers = sorted(headers, key=lambda x: original_columns.index(x))
                comp = gr.components.DataFrame(
                    value=df[headers],
                    type='pandas',
                    datatype=[type_map[x] for x in headers],
                    interactive=False,
                    wrap=True, 
                    visible=True)

                return comp

            # Add change events for all checkbox groups
            checkbox_group.change(
                fn=filter_df2, 
                inputs=[checkbox_group, algo_name, dataset_name, llm_name], 
                outputs=data_component
            )
            
            algo_name.change(
                fn=filter_df2,
                inputs=[checkbox_group, algo_name, dataset_name, llm_name],
                outputs=data_component
            )
            
            dataset_name.change(
                fn=filter_df2,
                inputs=[checkbox_group, algo_name, dataset_name, llm_name],
                outputs=data_component
            )
            
            llm_name.change(
                fn=filter_df2,
                inputs=[checkbox_group, algo_name, dataset_name, llm_name],
                outputs=data_component
            )


    with gr.Row():
        with gr.Accordion("📙 Citation", open=False):
            gr.Textbox(
                value=CITATION_BUTTON_TEXT, lines=7,
                label="Copy the BibTeX snippet to cite this source",
                elem_id="citation-button",
                show_copy_button=True,
            )


            
if __name__ == '__main__':
    demo.launch(server_name='0.0.0.0')