crm_llm_leaderboard / src /populate.py
yibum's picture
update CRM Bias
4c0cc56
raw
history blame
2.91 kB
import os
import pandas as pd
def get_leaderboard_df_crm(
crm_results_path: str, accuracy_cols: list, cost_cols: list
) -> tuple[pd.DataFrame, pd.DataFrame]:
"""Creates a dataframe from all the individual experiment results"""
sf_finetuned_models = ["SF-TextBase 70B", "SF-TextBase 7B", "SF-TextSum"]
leaderboard_accuracy_df = pd.read_csv(os.path.join(crm_results_path, "hf_leaderboard_accuracy.csv"))
leaderboard_accuracy_df = leaderboard_accuracy_df[~leaderboard_accuracy_df["Model Name"].isin(sf_finetuned_models)]
# leaderboard_accuracy_df = leaderboard_accuracy_df.sort_values(
# by=[AutoEvalColumn.accuracy_metric_average.name], ascending=False
# )
leaderboard_accuracy_df = leaderboard_accuracy_df[accuracy_cols].round(decimals=2)
ref_df = leaderboard_accuracy_df[["Model Name", "LLM Provider"]].drop_duplicates()
leaderboard_cost_df = pd.read_csv(os.path.join(crm_results_path, "hf_leaderboard_latency_cost.csv"))
leaderboard_cost_df = leaderboard_cost_df[~leaderboard_cost_df["Model Name"].isin(sf_finetuned_models)]
leaderboard_cost_df = leaderboard_cost_df.join(ref_df.set_index("Model Name"), on="Model Name")
# leaderboard_cost_df["LLM Provider"] = leaderboard_cost_df["LLM Provider"].fillna("Google")
leaderboard_cost_df = leaderboard_cost_df[cost_cols].round(decimals=2)
leaderboard_ts_df = pd.read_csv(os.path.join(crm_results_path, "hf_leaderboard_ts.csv"))
leaderboard_ts__crm_bias_df = pd.read_csv(os.path.join(crm_results_path, "hf_leaderboard_crm_bias.csv"))
leaderboard_ts_df = leaderboard_ts_df[~leaderboard_ts_df["Model Name"].isin(sf_finetuned_models)]
leaderboard_ts_df = leaderboard_ts_df.join(ref_df.set_index("Model Name"), on="Model Name")
leaderboard_ts_df = leaderboard_ts_df.join(leaderboard_ts__crm_bias_df.set_index("Model Name"), on="Model Name")
# leaderboard_ts_df["LLM Provider"] = leaderboard_ts_df["LLM Provider"].fillna("Google")
privacy_cols = leaderboard_ts_df[
[
"Privacy Zero-Shot Match Avoidance",
"Privacy Zero-Shot Reveal Avoidance",
"Privacy Five-Shot Match Avoidance",
"Privacy Five-Shot Reveal Avoidance",
]
].apply(lambda x: x.str.rstrip("%").astype("float") / 100.0, axis=1)
leaderboard_ts_df["Privacy"] = privacy_cols.mean(axis=1).transform(lambda x: "{:,.2%}".format(x))
leaderboard_ts_df["Bias No CI"] = leaderboard_ts_df["CRM Bias"].transform(lambda x: x.split(" ")[0])
ts_cols = leaderboard_ts_df[
[
"Safety",
"Privacy",
"Truthfulness",
"Bias No CI",
]
].apply(lambda x: x.str.rstrip("%").astype("float") / 100.0, axis=1)
leaderboard_ts_df["Trust & Safety"] = ts_cols.mean(axis=1).transform(lambda x: "{:,.2%}".format(x))
return leaderboard_accuracy_df, leaderboard_cost_df, leaderboard_ts_df