Spaces:
Runtime error
Runtime error
File size: 3,872 Bytes
2d4b120 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 |
import requests
import pandas as pd
from tqdm.auto import tqdm
import streamlit as st
from huggingface_hub import HfApi, hf_hub_download
from huggingface_hub.repocard import metadata_load
def make_clickable(model_name):
link = "https://huggingface.co/" + model_name
return f'<a target="_blank" href="{link}">{model_name}</a>'
def get_model_ids():
api = HfApi()
# TODO: switch to hf-leaderboards for the final version.
models = api.list_models(filter="hf-asr-leaderboard")
model_ids = [x.modelId for x in models]
return model_ids
def get_metadata(model_id):
try:
readme_path = hf_hub_download(model_id, filename="README.md")
return metadata_load(readme_path)
except requests.exceptions.HTTPError:
# 404 README.md not found
return None
def parse_metric_value(value):
if isinstance(value, str):
"".join(value.split("%"))
try:
value = float(value)
except: # noqa: E722
value = None
elif isinstance(value, list):
if len(value) > 0:
value = value[0]
else:
value = None
value = round(value, 2) if value is not None else None
return value
def parse_metrics_rows(meta):
if "model-index" not in meta:
return None
for result in meta["model-index"][0]["results"]:
if "dataset" not in result or "metrics" not in result:
continue
dataset = result["dataset"]["type"]
if "args" not in result["dataset"]:
continue
row = {"dataset": dataset}
for metric in result["metrics"]:
type = metric["type"].lower().strip()
value = parse_metric_value(metric["value"])
if value is None:
continue
if type not in row or value < row[type]:
# overwrite the metric if the new value is lower (e.g. with LM)
row[type] = value
yield row
@st.cache(ttl=600)
def get_data():
data = []
model_ids = get_model_ids()
for model_id in tqdm(model_ids):
meta = get_metadata(model_id)
if meta is None:
continue
for row in parse_metrics_rows(meta):
if row is None:
continue
row["model_id"] = model_id
data.append(row)
return pd.DataFrame.from_records(data)
dataframe = get_data()
selectable_datasets = list(set(dataframe.dataset.tolist()))
st.markdown("# 🤗 Leaderboards")
dataset = st.sidebar.selectbox(
"Dataset",
selectable_datasets,
index=selectable_datasets.index("common_voice"),
)
dataset_df = dataframe[dataframe.dataset == dataset]
dataset_df = dataset_df.dropna(axis="columns", how="all")
metric = st.sidebar.selectbox(
"Metric",
list(filter(lambda column: column not in ("model_id", "dataset"), dataset_df.columns)),
)
dataset_df = dataset_df.filter(["model_id", metric])
dataset_df = dataset_df.dropna()
dataset_df = dataset_df.sort_values(by=metric, ascending=False)
st.markdown(
"Please click on the model's name to be redirected to its model card which includes documentation and examples on how to use it."
)
# display the model ranks
dataset_df = dataset_df.reset_index(drop=True)
dataset_df.index += 1
# turn the model ids into clickable links
dataset_df["model_id"] = dataset_df["model_id"].apply(make_clickable)
table_html = dataset_df.to_html(escape=False)
table_html = table_html.replace("<th>", '<th align="left">') # left-align the headers
st.write(table_html, unsafe_allow_html=True)
st.markdown(
"Want to beat the Leaderboard? Don't see your model here? Simply add the `hf-leaderboards` tag to your model card alongside your evaluation metrics. See [this commit](https://huggingface.co/facebook/wav2vec2-base-960h/commit/88338305603a4d8db25aca96e669beb5f7dc65cb) as an example."
)
|