Update app.py

app.py

@@ -1,8 +1,36 @@
 import gradio as gr
+from gradio_leaderboard import Leaderboard
 import plotly.express as px
 from pathlib import Path
 import pandas as pd
 import numpy as np
+from langchain_openai import ChatOpenAI
+from langchain_experimental.agents.agent_toolkits import create_pandas_dataframe_agent
+from langchain.agents.agent_types import AgentType
+from langchain_google_genai import ChatGoogleGenerativeAI
+
+def explain_df(query, df):
+    agent = create_pandas_dataframe_agent(
+        # ChatOpenAI(
+        #     base_url="https://fmapi.swissai.cscs.ch",
+        #     temperature=0.01,
+        #     model="meta-llama/Llama-3.3-70B-Instruct"
+        # ),
+        ChatGoogleGenerativeAI(
+            model="gemini-1.5-pro",
+            temperature=0,
+            max_tokens=None,
+            timeout=None,
+            max_retries=2,
+        ),
+        df,
+        verbose=True,
+        allow_dangerous_code=True,
+    )
+    response = agent.invoke(query)
+    return response['output']
+
+    
 abs_path = Path(__file__).parent
 
 def parse_model_args(model_args):
@@ -34,47 +62,75 @@ def parse_model_precision(model_args):
 
 # Any pandas-compatible data
 df = pd.read_csv(str(abs_path / "eval_results.csv"))
+perf_df = pd.read_csv(str(abs_path / "perfbench_results.csv"))
 # take acc only
 df = df[df['metric'] == 'acc']
 # dedup
 df = df.drop_duplicates(subset=['model', 'task'])
 # pivot df, such that the column names are model,task,efficiency
 # but keep precision in its original place
-df = df.pivot(index='model', columns='task', values='value').reset_index()
+df['model_physical_size'] = df['model_physical_size'].apply(lambda x: x/1024/1024/1024)
+
+df = df.pivot(index=['model','hf_name','model_physical_size'], columns='task', values='value').reset_index()
 
 df['precision'] = df['model'].apply(lambda x: x.split(":")[-1])
 df['model'] = df['model'].apply(lambda x: x.split(":")[0])
-
-# average over all columns starting with 'task_'
 df['avg_acc'] = df.filter(like='task_').mean(axis=1)
-# keep 2 decimal points for avg_acc, and all tasks_
-# rename columns starting with 'task_' by removing 'task_'
+
 df = df.rename(columns=lambda x: x.replace('task_', ''))
 numeric_columns = df.select_dtypes(include=[np.number]).columns
+# remove physical size from numeric columns
+numeric_columns = numeric_columns.drop('model_physical_size')
 df[numeric_columns] = (df[numeric_columns]*100).round(2)
+df['model_physical_size'] = df['model_physical_size'].round(2)
+full_df = df.merge(perf_df, left_on='hf_name', right_on='hf_name', how='left')
 
 with gr.Blocks() as demo:
     gr.Markdown("""
     # 🥇 Efficient LLM Leaderboard
     """)
-    task_options = [col for col in df.columns if col not in ['model', 'precision']]
-
-    with gr.Row():
-        selected_tasks = gr.CheckboxGroup(choices=task_options, label="Select Tasks")
-    with gr.Row():
-        accuracy_plot = gr.Plot(label="Accuracy Plot")
-        data_table = gr.Dataframe(value=df, label="Result Table")
-
-    def update_outputs(selected_tasks):
-        if not selected_tasks:
-            return df[['model', 'precision']], None
-        filtered_df = df[['model', 'precision'] + selected_tasks]
-        melted_df = filtered_df.melt(id_vars=['model', 'precision'], var_name='task', value_name='accuracy')
-        fig = px.bar(melted_df, x='model', y='accuracy', color='precision', barmode='group', facet_col='task')
-        return filtered_df, fig
-
-    selected_tasks.change(fn=update_outputs, inputs=selected_tasks, outputs=[data_table, accuracy_plot])
-
-
+    with gr.Tabs():
+        with gr.TabItem("Leaderboard"):
+            # ...existing code...
+            task_options = [col for col in df.columns if col not in ['model','hf_name','model_physical_size', 'precision']]
+            with gr.Row():
+                selected_tasks = gr.CheckboxGroup(choices=task_options, label="Select Tasks")
+            with gr.Row():
+                accuracy_plot = gr.Plot(label="Accuracy Plot")
+                line_plot = gr.Plot(label="Average Accuracy vs Model Size")
+            with gr.Row():
+                throughput_line_plot = gr.Plot(label="Throughput vs Average Accuracy")
+                latency_line_plot = gr.Plot(label="Latency vs Average Accuracy")
+            with gr.Row():
+                data_table = gr.Dataframe(value=df, label="Result Table")
+            def update_outputs(selected_tasks):
+                if not selected_tasks:
+                    return df[['model', 'precision']], None, None
+                filtered_df = df[['model', 'precision', 'model_physical_size','hf_name'] + selected_tasks]
+                # average accuracy of selected tasks
+                filtered_df['avg_accuracy'] = filtered_df[selected_tasks].mean(axis=1)
+                bar_fig = px.bar(filtered_df, x='model', y='avg_accuracy', color='precision', barmode='group')
+                line_fig = px.line(filtered_df, x='model_physical_size', y='avg_accuracy', color='model', symbol='precision')
+                # set title of bar_fig
+                bar_fig.update_layout(title=f'tasks: {", ".join(selected_tasks)}')
+                line_fig.update_layout(title=f'tasks: {", ".join(selected_tasks)}')
+                with_perf_df = filtered_df.merge(perf_df, left_on='hf_name', right_on='hf_name', how='left')
+                throughput_line_fig = px.line(with_perf_df, x='output_throughput', y='avg_accuracy', color='model', symbol='precision')
+                latency_line_fig = px.line(with_perf_df, x="avg_e2e_latency", y='avg_accuracy', color='model', symbol='precision')
+                return with_perf_df, bar_fig, line_fig, throughput_line_fig, latency_line_fig
+            selected_tasks.change(
+                fn=update_outputs,
+                inputs=selected_tasks, 
+                outputs=[data_table, accuracy_plot, line_plot, throughput_line_plot, latency_line_plot]
+            )
+        with gr.TabItem("Find Model"):
+            query_input = gr.Textbox(label="Enter your query", placeholder="Enter your query here")
+            response_output = gr.Textbox(label="Response", interactive=False)
+            query_input.submit(
+                fn=lambda query: explain_df(query, df),
+                inputs=query_input,
+                outputs=response_output
+            )
+    
 if __name__ == "__main__":
-    demo.launch()
+    demo.launch(share=True)