remove backend from the front

README.md

@@ -4,7 +4,7 @@ emoji: 🥇
 colorFrom: green
 colorTo: indigo
 sdk: gradio
-sdk_version: 4.20.0
+sdk_version: 4.31.0
 app_file: app.py
 pinned: true
 license: apache-2.0

app.py

@@ -1,5 +1,3 @@
-import glob
-import json
 import logging
 import os
 
@@ -8,17 +6,22 @@ import numpy as np
 import pandas as pd
 import scipy.stats
 from apscheduler.schedulers.background import BackgroundScheduler
+from datasets import load_dataset
 from huggingface_hub import HfApi
 
-from src.backend import backend_routine
-from src.logging import configure_root_logger, setup_logger
+# Set up logging
+logger = logging.getLogger("app")
+logger.setLevel(logging.INFO)
+formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
+ch = logging.StreamHandler()
+ch.setFormatter(formatter)
+logger.addHandler(ch)
 
-configure_root_logger()
-logger = setup_logger(__name__)
+# Disable the absl logger (annoying)
 logging.getLogger("absl").setLevel(logging.WARNING)
 
 API = HfApi(token=os.environ.get("TOKEN"))
-RESULTS_REPO = "open-rl-leaderboard/results"
+RESULTS_REPO = "open-rl-leaderboard/results_v2"
 REFRESH_RATE = 5 * 60  # 5 minutes
 ALL_ENV_IDS = {
     "Atari": [
@@ -127,30 +130,11 @@ def iqm(x):
 
 def get_leaderboard_df():
     logger.info("Downloading results")
-    dir_path = API.snapshot_download(repo_id=RESULTS_REPO, repo_type="dataset")
-    pattern = os.path.join(dir_path, "**", "results_*.json")
-    filenames = glob.glob(pattern, recursive=True)
-
-    data = []
-    for filename in filenames:
-        try:
-            with open(filename) as fp:
-                report = json.load(fp)
-            if report["status"] == "DONE" and len(report["results"]) > 0:
-                user_id, model_id = report["config"]["model_id"].split("/")
-                row = {"user_id": user_id, "model_id": model_id, "model_sha": report["config"]["model_sha"]}
-                env_ids = list(report["results"].keys())
-                assert len(env_ids) == 1, "Only one environment supported for the moment"
-                row["env_id"] = env_ids[0]
-                row["iqm_episodic_return"] = iqm(report["results"][env_ids[0]]["episodic_returns"])
-                data.append(row)
-        except Exception as e:
-            logger.error(f"Error while processing {filename}: {e}")
-
-    df = pd.DataFrame(data)  # create DataFrame
-    df = df.fillna("")  # replace NaN values with empty strings
-    # Save to csv
-    df.to_csv("leaderboard.csv", index=False)
+    dataset = load_dataset(RESULTS_REPO, split="train")  # split is not important, but we need to use "train")
+    df = dataset.to_pandas()  # convert to pandas dataframe
+    df = df[df["status"] == "DONE"]  # keep only the models that are done
+    df["iqm_episodic_return"] = df["episodic_returns"].apply(iqm)
+    logger.debug("Results downloaded")
     return df
 
 
@@ -180,10 +164,10 @@ def refresh_video(df, env_id):
     if not env_df.empty:
         user_id = env_df.iloc[0]["user_id"]
         model_id = env_df.iloc[0]["model_id"]
-        model_sha = env_df.iloc[0]["model_sha"]
+        sha = env_df.iloc[0]["sha"]
         repo_id = f"{user_id}/{model_id}"
         try:
-            video_path = API.hf_hub_download(repo_id=repo_id, filename="replay.mp4", revision=model_sha, repo_type="model")
+            video_path = API.hf_hub_download(repo_id=repo_id, filename="replay.mp4", revision=sha, repo_type="model")
             return video_path
         except Exception as e:
             logger.error(f"Error while downloading video for {env_id}: {e}")
@@ -217,7 +201,8 @@ This leaderboard is quite empty... 😢
 Be the first to submit your model!
 Check the tab "🚀 Getting my agent evaluated"
 """
-    
+
+
 def refresh_num_models(df):
     return f"The leaderboard currently contains {len(df):,} models."
 
@@ -269,7 +254,7 @@ with gr.Blocks(css=css) as demo:
                         # If the env_id envs with "NoFrameskip-v4", we remove it to improve readability
                         tab_env_id = env_id[: -len("NoFrameskip-v4")] if env_id.endswith("NoFrameskip-v4") else env_id
                         with gr.TabItem(tab_env_id) as tab:
-                            logger.info(f"Creating tab for {env_id}")
+                            logger.debug(f"Creating tab for {env_id}")
                             with gr.Row(equal_height=False):
                                 with gr.Column(scale=3):
                                     gr_df = gr.components.Dataframe(
@@ -308,7 +293,6 @@ with gr.Blocks(css=css) as demo:
     demo.load(refresh, outputs=list(all_gr_dfs.values()) + list(all_gr_winners.values()) + [num_models_md])
 
 scheduler = BackgroundScheduler()
-scheduler.add_job(func=backend_routine, trigger="interval", seconds=REFRESH_RATE, max_instances=1)
 scheduler.add_job(func=update_globals, trigger="interval", seconds=REFRESH_RATE, max_instances=1)
 scheduler.start()
 

packages.txt

@@ -1,3 +0,0 @@
-swig
-libosmesa6-dev
-patchelf

requirements.txt

@@ -1,24 +1,7 @@
-APScheduler==3.10.1
-black==23.11.0
-click==8.1.3
-datasets==2.14.5
-gradio==4.20.0
-gradio_client
-gymnasium[all,accept-rom-license]==0.29.1
-huggingface-hub>=0.18.0
-matplotlib==3.7.1
-free-mujoco-py
-mujoco<=2.3.7
-numpy==1.24.2
-pandas==2.0.0
-python-dateutil==2.8.2
-requests==2.28.2
-rliable==1.0.8
-torch==2.2.2
-tqdm==4.65.0
-
-
-# Log Visualizer
-BeautifulSoup4==4.12.2
-lxml==4.9.3
-rich==13.3.4
+APScheduler==3.10.4
+datasets==2.19.1
+gradio==4.31.2
+huggingface-hub==0.23.0
+numpy==1.26.4
+pandas==2.2.2
+scipy==1.13.0

src/backend.py

@@ -1,90 +0,0 @@
-import json
-import os
-import random
-import re
-import tempfile
-
-from huggingface_hub import CommitOperationAdd, HfApi
-
-from src.evaluation import evaluate
-from src.logging import setup_logger
-
-logger = setup_logger(__name__)
-
-API = HfApi(token=os.environ.get("TOKEN"))
-RESULTS_REPO = "open-rl-leaderboard/results"
-
-
-def _backend_routine():
-    # List only the text classification models
-    rl_models = list(API.list_models(filter="reinforcement-learning"))
-    logger.info(f"Found {len(rl_models)} RL models")
-    compatible_models = []
-    for model in rl_models:
-        filenames = [sib.rfilename for sib in model.siblings]
-        if "agent.pt" in filenames:
-            compatible_models.append((model.modelId, model.sha))
-
-    logger.info(f"Found {len(compatible_models)} compatible models")
-
-    # Get the results
-    pattern = re.compile(r"^[^/]*/[^/]*/[^/]*results_[a-f0-9]+\.json$")
-    filenames = API.list_repo_files(RESULTS_REPO, repo_type="dataset")
-    filenames = [filename for filename in filenames if pattern.match(filename)]
-
-    evaluated_models = set()
-    for filename in filenames:
-        path = API.hf_hub_download(repo_id=RESULTS_REPO, filename=filename, repo_type="dataset")
-        with open(path) as fp:
-            report = json.load(fp)
-        evaluated_models.add((report["config"]["model_id"], report["config"]["model_sha"]))
-
-    # Find the models that are not associated with any results
-    pending_models = list(set(compatible_models) - evaluated_models)
-    logger.info(f"Found {len(pending_models)} pending models")
-
-    if len(pending_models) == 0:
-        return None
-
-    # Run an evaluation on the models
-    with tempfile.TemporaryDirectory() as tmp_dir:
-        commits = []
-        model_id, sha = random.choice(pending_models)
-        logger.info(f"Running evaluation on {model_id}")
-        report = {"config": {"model_id": model_id, "model_sha": sha}}
-        try:
-            evaluations = evaluate(model_id, revision=sha)
-        except Exception as e:
-            logger.error(f"Error evaluating {model_id}: {e}")
-            evaluations = None
-
-        if evaluations is not None:
-            report["results"] = evaluations
-            report["status"] = "DONE"
-        else:
-            report["status"] = "FAILED"
-
-        # Update the results
-        dumped = json.dumps(report, indent=2)
-        path_in_repo = f"{model_id}/results_{sha}.json"
-        local_path = os.path.join(tmp_dir, path_in_repo)
-        os.makedirs(os.path.dirname(local_path), exist_ok=True)
-        with open(local_path, "w") as f:
-            f.write(dumped)
-
-        commits.append(CommitOperationAdd(path_in_repo=path_in_repo, path_or_fileobj=local_path))
-
-        API.create_commit(
-            repo_id=RESULTS_REPO, commit_message="Add evaluation results", operations=commits, repo_type="dataset"
-        )
-
-
-def backend_routine():
-    try:
-        _backend_routine()
-    except Exception as e:
-        logger.error(f"{e.__class__.__name__}: {str(e)}")
-
-
-if __name__ == "__main__":
-    backend_routine()

src/css_html_js.py

@@ -1,20 +0,0 @@
-style_content = """
-pre, code {
-    background-color: #272822;
-}
-    .scrollable {
-        font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace;
-        height: 500px;
-        overflow: auto;
-    }
-    """
-dark_mode_gradio_js = """
-function refresh() {
-    const url = new URL(window.location);
-
-    if (url.searchParams.get('__theme') !== 'dark') {
-        url.searchParams.set('__theme', 'dark');
-        window.location.href = url.href;
-    }
-}
-"""

src/evaluation.py

@@ -1,365 +0,0 @@
-import fnmatch
-import os
-from typing import Dict, SupportsFloat
-
-import gymnasium as gym
-import numpy as np
-import torch
-from gymnasium import wrappers
-from huggingface_hub import HfApi
-from huggingface_hub.utils._errors import EntryNotFoundError
-
-from src.logging import setup_logger
-
-logger = setup_logger(__name__)
-
-API = HfApi(token=os.environ.get("TOKEN"))
-
-
-ALL_ENV_IDS = [
-    "AdventureNoFrameskip-v4",
-    "AirRaidNoFrameskip-v4",
-    "AlienNoFrameskip-v4",
-    "AmidarNoFrameskip-v4",
-    "AssaultNoFrameskip-v4",
-    "AsterixNoFrameskip-v4",
-    "AsteroidsNoFrameskip-v4",
-    "AtlantisNoFrameskip-v4",
-    "BankHeistNoFrameskip-v4",
-    "BattleZoneNoFrameskip-v4",
-    "BeamRiderNoFrameskip-v4",
-    "BerzerkNoFrameskip-v4",
-    "BowlingNoFrameskip-v4",
-    "BoxingNoFrameskip-v4",
-    "BreakoutNoFrameskip-v4",
-    "CarnivalNoFrameskip-v4",
-    "CentipedeNoFrameskip-v4",
-    "ChopperCommandNoFrameskip-v4",
-    "CrazyClimberNoFrameskip-v4",
-    "DefenderNoFrameskip-v4",
-    "DemonAttackNoFrameskip-v4",
-    "DoubleDunkNoFrameskip-v4",
-    "ElevatorActionNoFrameskip-v4",
-    "EnduroNoFrameskip-v4",
-    "FishingDerbyNoFrameskip-v4",
-    "FreewayNoFrameskip-v4",
-    "FrostbiteNoFrameskip-v4",
-    "GopherNoFrameskip-v4",
-    "GravitarNoFrameskip-v4",
-    "HeroNoFrameskip-v4",
-    "IceHockeyNoFrameskip-v4",
-    "JamesbondNoFrameskip-v4",
-    "JourneyEscapeNoFrameskip-v4",
-    "KangarooNoFrameskip-v4",
-    "KrullNoFrameskip-v4",
-    "KungFuMasterNoFrameskip-v4",
-    "MontezumaRevengeNoFrameskip-v4",
-    "MsPacmanNoFrameskip-v4",
-    "NameThisGameNoFrameskip-v4",
-    "PhoenixNoFrameskip-v4",
-    "PitfallNoFrameskip-v4",
-    "PongNoFrameskip-v4",
-    "PooyanNoFrameskip-v4",
-    "PrivateEyeNoFrameskip-v4",
-    "QbertNoFrameskip-v4",
-    "RiverraidNoFrameskip-v4",
-    "RoadRunnerNoFrameskip-v4",
-    "RobotankNoFrameskip-v4",
-    "SeaquestNoFrameskip-v4",
-    "SkiingNoFrameskip-v4",
-    "SolarisNoFrameskip-v4",
-    "SpaceInvadersNoFrameskip-v4",
-    "StarGunnerNoFrameskip-v4",
-    "TennisNoFrameskip-v4",
-    "TimePilotNoFrameskip-v4",
-    "TutankhamNoFrameskip-v4",
-    "UpNDownNoFrameskip-v4",
-    "VentureNoFrameskip-v4",
-    "VideoPinballNoFrameskip-v4",
-    "WizardOfWorNoFrameskip-v4",
-    "YarsRevengeNoFrameskip-v4",
-    "ZaxxonNoFrameskip-v4",
-    # Box2D
-    "BipedalWalker-v3",
-    "BipedalWalkerHardcore-v3",
-    "CarRacing-v2",
-    "LunarLander-v2",
-    "LunarLanderContinuous-v2",
-    # Toy text
-    "Blackjack-v1",
-    "CliffWalking-v0",
-    "FrozenLake-v1",
-    "FrozenLake8x8-v1",
-    # Classic control
-    "Acrobot-v1",
-    "CartPole-v1",
-    "MountainCar-v0",
-    "MountainCarContinuous-v0",
-    "Pendulum-v1",
-    # MuJoCo
-    "Ant-v4",
-    "HalfCheetah-v4",
-    "Hopper-v4",
-    "Humanoid-v4",
-    "HumanoidStandup-v4",
-    "InvertedDoublePendulum-v4",
-    "InvertedPendulum-v4",
-    "Pusher-v4",
-    "Reacher-v4",
-    "Swimmer-v4",
-    "Walker2d-v4",
-]
-
-NUM_EPISODES = 50
-
-
-class NoopResetEnv(gym.Wrapper[np.ndarray, int, np.ndarray, int]):
-    """
-    Sample initial states by taking random number of no-ops on reset.
-    No-op is assumed to be action 0.
-
-    :param env: Environment to wrap
-    :param noop_max: Maximum value of no-ops to run
-    """
-
-    def __init__(self, env: gym.Env, noop_max: int = 30) -> None:
-        super().__init__(env)
-        self.noop_max = noop_max
-        self.override_num_noops = None
-        self.noop_action = 0
-        assert env.unwrapped.get_action_meanings()[0] == "NOOP"  # type: ignore[attr-defined]
-
-    def reset(self, **kwargs):
-        self.env.reset(**kwargs)
-        if self.override_num_noops is not None:
-            noops = self.override_num_noops
-        else:
-            noops = self.unwrapped.np_random.integers(1, self.noop_max + 1)
-        assert noops > 0
-        obs = np.zeros(0)
-        info: Dict = {}
-        for _ in range(noops):
-            obs, _, terminated, truncated, info = self.env.step(self.noop_action)
-            if terminated or truncated:
-                obs, info = self.env.reset(**kwargs)
-        return obs, info
-
-
-class FireResetEnv(gym.Wrapper[np.ndarray, int, np.ndarray, int]):
-    """
-    Take action on reset for environments that are fixed until firing.
-
-    :param env: Environment to wrap
-    """
-
-    def __init__(self, env: gym.Env) -> None:
-        super().__init__(env)
-        assert env.unwrapped.get_action_meanings()[1] == "FIRE"  # type: ignore[attr-defined]
-        assert len(env.unwrapped.get_action_meanings()) >= 3  # type: ignore[attr-defined]
-
-    def reset(self, **kwargs):
-        self.env.reset(**kwargs)
-        obs, _, terminated, truncated, _ = self.env.step(1)
-        if terminated or truncated:
-            self.env.reset(**kwargs)
-        obs, _, terminated, truncated, _ = self.env.step(2)
-        if terminated or truncated:
-            self.env.reset(**kwargs)
-        return obs, {}
-
-
-class EpisodicLifeEnv(gym.Wrapper[np.ndarray, int, np.ndarray, int]):
-    """
-    Make end-of-life == end-of-episode, but only reset on true game over.
-    Done by DeepMind for the DQN and co. since it helps value estimation.
-
-    :param env: Environment to wrap
-    """
-
-    def __init__(self, env: gym.Env) -> None:
-        super().__init__(env)
-        self.lives = 0
-        self.was_real_done = True
-
-    def step(self, action: int):
-        obs, reward, terminated, truncated, info = self.env.step(action)
-        self.was_real_done = terminated or truncated
-        # check current lives, make loss of life terminal,
-        # then update lives to handle bonus lives
-        lives = self.env.unwrapped.ale.lives()  # type: ignore[attr-defined]
-        if 0 < lives < self.lives:
-            # for Qbert sometimes we stay in lives == 0 condition for a few frames
-            # so its important to keep lives > 0, so that we only reset once
-            # the environment advertises done.
-            terminated = True
-        self.lives = lives
-        return obs, reward, terminated, truncated, info
-
-    def reset(self, **kwargs):
-        """
-        Calls the Gym environment reset, only when lives are exhausted.
-        This way all states are still reachable even though lives are episodic,
-        and the learner need not know about any of this behind-the-scenes.
-
-        :param kwargs: Extra keywords passed to env.reset() call
-        :return: the first observation of the environment
-        """
-        if self.was_real_done:
-            obs, info = self.env.reset(**kwargs)
-        else:
-            # no-op step to advance from terminal/lost life state
-            obs, _, terminated, truncated, info = self.env.step(0)
-
-            # The no-op step can lead to a game over, so we need to check it again
-            # to see if we should reset the environment and avoid the
-            # monitor.py `RuntimeError: Tried to step environment that needs reset`
-            if terminated or truncated:
-                obs, info = self.env.reset(**kwargs)
-        self.lives = self.env.unwrapped.ale.lives()  # type: ignore[attr-defined]
-        return obs, info
-
-
-class MaxAndSkipEnv(gym.Wrapper[np.ndarray, int, np.ndarray, int]):
-    """
-    Return only every ``skip``-th frame (frameskipping)
-    and return the max between the two last frames.
-
-    :param env: Environment to wrap
-    :param skip: Number of ``skip``-th frame
-        The same action will be taken ``skip`` times.
-    """
-
-    def __init__(self, env: gym.Env, skip: int = 4) -> None:
-        super().__init__(env)
-        # most recent raw observations (for max pooling across time steps)
-        assert env.observation_space.dtype is not None, "No dtype specified for the observation space"
-        assert env.observation_space.shape is not None, "No shape defined for the observation space"
-        self._obs_buffer = np.zeros((2, *env.observation_space.shape), dtype=env.observation_space.dtype)
-        self._skip = skip
-
-    def step(self, action: int):
-        """
-        Step the environment with the given action
-        Repeat action, sum reward, and max over last observations.
-
-        :param action: the action
-        :return: observation, reward, terminated, truncated, information
-        """
-        total_reward = 0.0
-        terminated = truncated = False
-        for i in range(self._skip):
-            obs, reward, terminated, truncated, info = self.env.step(action)
-            done = terminated or truncated
-            if i == self._skip - 2:
-                self._obs_buffer[0] = obs
-            if i == self._skip - 1:
-                self._obs_buffer[1] = obs
-            total_reward += float(reward)
-            if done:
-                break
-        # Note that the observation on the done=True frame
-        # doesn't matter
-        max_frame = self._obs_buffer.max(axis=0)
-
-        return max_frame, total_reward, terminated, truncated, info
-
-
-class ClipRewardEnv(gym.RewardWrapper):
-    """
-    Clip the reward to {+1, 0, -1} by its sign.
-
-    :param env: Environment to wrap
-    """
-
-    def __init__(self, env: gym.Env) -> None:
-        super().__init__(env)
-
-    def reward(self, reward: SupportsFloat) -> float:
-        """
-        Bin reward to {+1, 0, -1} by its sign.
-
-        :param reward:
-        :return:
-        """
-        return np.sign(float(reward))
-
-
-def make(env_id):
-    def thunk():
-        env = gym.make(env_id)
-        env = wrappers.RecordEpisodeStatistics(env)
-        if "NoFrameskip" in env_id:
-            env = NoopResetEnv(env, noop_max=30)
-            env = MaxAndSkipEnv(env, skip=4)
-            env = EpisodicLifeEnv(env)
-            if "FIRE" in env.unwrapped.get_action_meanings():
-                env = FireResetEnv(env)
-            env = ClipRewardEnv(env)
-            env = wrappers.ResizeObservation(env, (84, 84))
-            env = wrappers.GrayScaleObservation(env)
-            env = wrappers.FrameStack(env, 4)
-        return env
-
-    return thunk
-
-
-def pattern_match(patterns, source_list):
-    if isinstance(patterns, str):
-        patterns = [patterns]
-
-    env_ids = set()
-    for pattern in patterns:
-        for matching in fnmatch.filter(source_list, pattern):
-            env_ids.add(matching)
-    return sorted(list(env_ids))
-
-
-def evaluate(model_id, revision):
-    tags = API.model_info(model_id, revision=revision).tags
-
-    # Extract the environment IDs from the tags (usually only one)
-    env_ids = pattern_match(tags, ALL_ENV_IDS)
-    logger.info(f"Selected environments: {env_ids}")
-
-    results = {}
-
-    # Check if the agent exists
-    try:
-        agent_path = API.hf_hub_download(repo_id=model_id, filename="agent.pt")
-    except EntryNotFoundError:
-        logger.error("Agent not found")
-        return None
-
-    # Check safety
-    security = next(iter(API.get_paths_info(model_id, "agent.pt", expand=True))).security
-    if security is None or "safe" not in security:
-        logger.warn("Agent safety not available")
-        # return None
-    elif not security["safe"]:
-        logger.error("Agent not safe")
-        return None
-
-    # Load the agent
-    try:
-        agent = torch.jit.load(agent_path).to("cuda")
-    except Exception as e:
-        logger.error(f"Error loading agent: {e}")
-        return None
-
-    # Evaluate the agent on the environments
-    for env_id in env_ids:
-        envs = gym.vector.SyncVectorEnv([make(env_id) for _ in range(1)])
-        observations, _ = envs.reset()
-        episodic_returns = []
-        while len(episodic_returns) < NUM_EPISODES:
-            actions = agent(torch.tensor(observations)).numpy()
-            observations, _, _, _, infos = envs.step(actions)
-            if "final_info" in infos:
-                for info in infos["final_info"]:
-                    if info is None or "episode" not in info:
-                        continue
-                    episodic_returns.append(float(info["episode"]["r"]))
-
-        results[env_id] = {"episodic_returns": episodic_returns}
-        logger.info(f"Environment {env_id}: {np.mean(episodic_returns)} ± {np.std(episodic_returns)}")
-    return results

src/logging.py

@@ -1,37 +0,0 @@
-from pathlib import Path
-
-proj_dir = Path(__file__).parents[1]
-
-log_file = proj_dir / "output.log"
-
-
-import logging
-
-
-def setup_logger(name: str):
-    logger = logging.getLogger(name)
-    logger.setLevel(logging.INFO)
-
-    formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
-
-    # Create a file handler to write logs to a file
-    file_handler = logging.FileHandler(log_file)
-    file_handler.setLevel(logging.INFO)
-    file_handler.setFormatter(formatter)
-    logger.addHandler(file_handler)
-
-    return logger
-
-
-def configure_root_logger():
-    # Configure the root logger
-    logging.basicConfig(level=logging.INFO)
-    root_logger = logging.getLogger()
-
-    formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
-
-    file_handler = logging.FileHandler(log_file)
-    file_handler.setLevel(logging.INFO)
-    file_handler.setFormatter(formatter)
-
-    root_logger.addHandler(file_handler)