Spaces:
Runtime error
Runtime error
from dataclasses import dataclass | |
from enum import Enum | |
class Task: | |
benchmark: str | |
metric: str | |
col_name: str | |
# Select your tasks here | |
# --------------------------------------------------- | |
class Tasks(Enum): | |
# task_key in the json file, metric_key in the json file, name to display in the leaderboard | |
task0 = Task("random_accuracy", "random_accuracy", "Accuracy (Random)") | |
task2 = Task("popular_accuracy", "popular_accuracy", "Accuracy (Popular)") | |
task1 = Task("adversarial_accuracy", "adversarial_accuracy", "Accuracy (Adversarial)") | |
task7 = Task("random_precision", "random_precision", "Precision (Random)") | |
task3 = Task("popular_precision", "popular_precision", "Precision (Popular)") | |
task11 = Task("adversarial_precision", "adversarial_precision", "Precision (Adversarial)") | |
task8 = Task("random_recall", "random_recall", "Recall (Random)") | |
task4 = Task("popular_recall", "popular_recall", "Recall (Popular)") | |
task12 = Task("adversarial_recall", "adversarial_recall", "Recall (Adversarial)") | |
task9 = Task("random_f1_score", "random_f1_score", "F1 Score (Random)") | |
task5 = Task("popular_f1_score", "popular_f1_score", "F1 Score (Popular)") | |
task13 = Task("adversarial_f1_score", "adversarial_f1_score", "F1 Score (Adversarial)") | |
task10 = Task("random_yes_percentage", "random_yes_percentage", "Yes Percent (Random)") | |
task6 = Task("popular_yes_percentage", "popular_yes_percentage", "Yes Percent (Popular)") | |
task14 = Task("adversarial_yes_percentage", "adversarial_yes_percentage", "Yes Percent (Adversarial)") | |
NUM_FEWSHOT = 0 # Change with your few shot | |
# --------------------------------------------------- | |
# Your leaderboard name | |
TITLE = """<h1 align="center" id="space-title">3D-POPE Leaderboard</h1>""" | |
# What does your leaderboard evaluate? | |
INTRODUCTION_TEXT = """ | |
#### This is the official leaderboard for the 3D Polling-based Object Probing Evaluation (3D-POPE) benchmark. | |
###### 3D-POPE is designed to assess a model's ability to accurately identify the presence or absence of objects in a given 3D scene. | |
""" | |
# Which evaluations are you running? how can people reproduce what you have? | |
LLM_BENCHMARKS_TEXT = f""" | |
## How it works | |
## Reproducibility | |
To reproduce our results, here is the commands you can run: | |
""" | |
EVALUATION_QUEUE_TEXT = """ | |
## Some good practices before submitting a model | |
### 1) Make sure you can load your model and tokenizer using AutoClasses: | |
```python | |
from transformers import AutoConfig, AutoModel, AutoTokenizer | |
config = AutoConfig.from_pretrained("your model name", revision=revision) | |
model = AutoModel.from_pretrained("your model name", revision=revision) | |
tokenizer = AutoTokenizer.from_pretrained("your model name", revision=revision) | |
``` | |
If this step fails, follow the error messages to debug your model before submitting it. It's likely your model has been improperly uploaded. | |
Note: make sure your model is public! | |
Note: if your model needs `use_remote_code=True`, we do not support this option yet but we are working on adding it, stay posted! | |
### 2) Convert your model weights to [safetensors](https://huggingface.co/docs/safetensors/index) | |
It's a new format for storing weights which is safer and faster to load and use. It will also allow us to add the number of parameters of your model to the `Extended Viewer`! | |
### 3) Make sure your model has an open license! | |
This is a leaderboard for Open LLMs, and we'd love for as many people as possible to know they can use your model 🤗 | |
### 4) Fill up your model card | |
When we add extra information about models to the leaderboard, it will be automatically taken from the model card | |
## In case of model failure | |
If your model is displayed in the `FAILED` category, its execution stopped. | |
Make sure you have followed the above steps first. | |
If everything is done, check you can launch the EleutherAIHarness on your model locally, using the above command without modifications (you can add `--limit` to limit the number of examples per task). | |
""" | |
CITATION_BUTTON_LABEL = "Copy the following snippet to cite these results" | |
CITATION_BUTTON_TEXT = r""" | |
@misc{yang20243dgrand, | |
title={3D-GRAND: Towards Better Grounding and Less Hallucination for 3D-LLMs}, | |
author={Jianing Yang and Xuweiyi Chen and Nikhil Madaan and Madhavan Iyengar and Shengyi Qian and David F. Fouhey and Joyce Chai}, | |
year={2024}, | |
eprint={2406.05132}, | |
archivePrefix={arXiv}, | |
primaryClass={cs.CV} | |
} | |
""" | |