Spaces:
Runtime error
Runtime error
Upload 5 files
Browse files- app.py +45 -0
- background_task.py +247 -0
- matchmaking.py +76 -0
- requirements.txt +5 -0
- utils.py +13 -0
app.py
ADDED
@@ -0,0 +1,45 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import gradio as gr
|
2 |
+
from huggingface_hub import HfApi
|
3 |
+
from matchmaking import *
|
4 |
+
from background_task import init_matchmaking, get_elo_data
|
5 |
+
from apscheduler.schedulers.background import BackgroundScheduler
|
6 |
+
from utils import *
|
7 |
+
|
8 |
+
matchmaking = Matchmaking()
|
9 |
+
api = HfApi()
|
10 |
+
|
11 |
+
# launch
|
12 |
+
scheduler = BackgroundScheduler()
|
13 |
+
scheduler.add_job(func=init_matchmaking, trigger="interval", seconds=300)
|
14 |
+
scheduler.start()
|
15 |
+
|
16 |
+
|
17 |
+
def update_elos():
|
18 |
+
matchmaking.read_history()
|
19 |
+
matchmaking.compute_elo()
|
20 |
+
matchmaking.save_elo_data()
|
21 |
+
|
22 |
+
|
23 |
+
with gr.Blocks() as block:
|
24 |
+
gr.Markdown(f"""
|
25 |
+
# 🏆 AI vs. AI SoccerTwos Leaderboard ⚽
|
26 |
+
|
27 |
+
In this leaderboard, you can find the ELO score and the rank of your trained model for the SoccerTwos environment.
|
28 |
+
|
29 |
+
If you want to know more about a model, just **copy the username and model and paste them into the search bar**.
|
30 |
+
|
31 |
+
👀 To visualize your agents competing check this demo: https://huggingface.co/spaces/unity/ML-Agents-SoccerTwos
|
32 |
+
|
33 |
+
🤖 For more information about this AI vs. AI challenge and to participate? [Check this](https://huggingface.co/deep-rl-course/unit7)
|
34 |
+
""")
|
35 |
+
with gr.Row():
|
36 |
+
output = gr.components.Dataframe(
|
37 |
+
value=get_elo_data,
|
38 |
+
headers=["Ranking 🏆", "User 🤗", "Model id 🤖", "ELO 🚀", "Games played 🎮"],
|
39 |
+
datatype=["number", "markdown", "markdown", "number", "number"]
|
40 |
+
)
|
41 |
+
with gr.Row():
|
42 |
+
refresh = gr.Button("Refresh")
|
43 |
+
refresh.click(get_elo_data, inputs=[], outputs=output)
|
44 |
+
|
45 |
+
block.launch()
|
background_task.py
ADDED
@@ -0,0 +1,247 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import os
|
2 |
+
import random
|
3 |
+
import subprocess
|
4 |
+
import pandas as pd
|
5 |
+
from datetime import datetime
|
6 |
+
from huggingface_hub import HfApi, Repository
|
7 |
+
from utils import *
|
8 |
+
|
9 |
+
DATASET_REPO_URL = "https://huggingface.co/datasets/huggingface-projects/bot-fight-data"
|
10 |
+
DATASET_TEMP_REPO_URL = "https://huggingface.co/datasets/huggingface-projects/temp-match-results"
|
11 |
+
FILTER_FILE = "https://huggingface.co/datasets/huggingface-projects/filter-bad-models/raw/main/bad_models.csv"
|
12 |
+
ELO_FILENAME = "soccer_elo.csv"
|
13 |
+
HISTORY_FILENAME = "soccer_history.csv"
|
14 |
+
TEMP_FILENAME = "results.csv"
|
15 |
+
ELO_DIR = "soccer_elo"
|
16 |
+
TEMP_DIR = "temp"
|
17 |
+
HF_TOKEN = os.environ.get("HF_TOKEN")
|
18 |
+
|
19 |
+
repo = Repository(
|
20 |
+
local_dir=ELO_DIR, clone_from=DATASET_REPO_URL, use_auth_token=HF_TOKEN
|
21 |
+
)
|
22 |
+
repo_temp = Repository(
|
23 |
+
local_dir=TEMP_DIR, clone_from=DATASET_TEMP_REPO_URL, use_auth_token=HF_TOKEN
|
24 |
+
)
|
25 |
+
|
26 |
+
api = HfApi()
|
27 |
+
os.chmod('./SoccerTows.x86_64', 0o755)
|
28 |
+
|
29 |
+
|
30 |
+
class Model:
|
31 |
+
"""
|
32 |
+
Class containing the info of a model.
|
33 |
+
|
34 |
+
:param name: Name of the model
|
35 |
+
:param elo: Elo rating of the model
|
36 |
+
:param games_played: Number of games played by the model (useful if we implement sigma uncertainty)
|
37 |
+
"""
|
38 |
+
|
39 |
+
def __init__(self, author, name, elo=1200, games_played=0):
|
40 |
+
self.author = author
|
41 |
+
self.name = name
|
42 |
+
self.elo = elo
|
43 |
+
self.games_played = games_played
|
44 |
+
|
45 |
+
|
46 |
+
class Matchmaking:
|
47 |
+
"""
|
48 |
+
Class managing the matchmaking between the models.
|
49 |
+
|
50 |
+
:param models: List of models
|
51 |
+
:param queue: Temporary list of models used for the matching process
|
52 |
+
:param k: Dev coefficient
|
53 |
+
:param max_diff: Maximum difference considered between two models' elo
|
54 |
+
:param matches: Dictionary containing the match history (to later upload as CSV)
|
55 |
+
"""
|
56 |
+
|
57 |
+
def __init__(self, models):
|
58 |
+
self.models = models
|
59 |
+
self.queue = self.models.copy()
|
60 |
+
self.k = 20
|
61 |
+
self.max_diff = 500
|
62 |
+
self.matches = {
|
63 |
+
"model1": [],
|
64 |
+
"model2": [],
|
65 |
+
"timestamp": [],
|
66 |
+
"result": [],
|
67 |
+
}
|
68 |
+
|
69 |
+
def run(self):
|
70 |
+
"""
|
71 |
+
Run the matchmaking process.
|
72 |
+
Add models to the queue, shuffle it, and match the models one by one to models with close ratings.
|
73 |
+
Compute the new elo for each model after each match and add the match to the match history.
|
74 |
+
"""
|
75 |
+
self.queue = self.models.copy()
|
76 |
+
random.shuffle(self.queue)
|
77 |
+
while len(self.queue) > 1:
|
78 |
+
print(f"Queue length: {len(self.queue)}")
|
79 |
+
model1 = self.queue.pop(0)
|
80 |
+
model2 = self.queue.pop(self.find_n_closest_indexes(model1, 10))
|
81 |
+
match(model1, model2)
|
82 |
+
self.load_results()
|
83 |
+
|
84 |
+
def load_results(self):
|
85 |
+
""" Load the match history from the hub. """
|
86 |
+
repo.git_pull()
|
87 |
+
results = pd.read_csv(
|
88 |
+
"https://huggingface.co/datasets/huggingface-projects/temp-match-results/raw/main/results.csv"
|
89 |
+
)
|
90 |
+
# while len(results) < len(self.matches["model1"]):
|
91 |
+
# time.sleep(60)
|
92 |
+
# results = pd.read_csv(
|
93 |
+
# "https://huggingface.co/datasets/huggingface-projects/temp-match-results/raw/main/results.csv"
|
94 |
+
# )
|
95 |
+
|
96 |
+
for i, row in results.iterrows():
|
97 |
+
model1 = row["model1"].split("/")
|
98 |
+
model2 = row["model2"].split("/")
|
99 |
+
model1 = self.find_model(model1[0], model1[1])
|
100 |
+
model2 = self.find_model(model2[0], model2[1])
|
101 |
+
result = row["result"]
|
102 |
+
if model1 is not None or model2 is not None:
|
103 |
+
self.compute_elo(model1, model2, row["result"])
|
104 |
+
self.matches["model1"].append(model1.author + "/" + model1.name)
|
105 |
+
self.matches["model2"].append(model2.author + "/" + model2.name)
|
106 |
+
self.matches["result"].append(result)
|
107 |
+
self.matches["timestamp"].append(row["timestamp"])
|
108 |
+
model1.games_played += 1
|
109 |
+
model2.games_played += 1
|
110 |
+
data_dict = {"model1": [], "model2": [], "timestamp": [], "result": []}
|
111 |
+
df = pd.DataFrame(data_dict)
|
112 |
+
print(df.head())
|
113 |
+
repo_temp.git_pull()
|
114 |
+
df.to_csv(os.path.join(TEMP_DIR, TEMP_FILENAME), index=False)
|
115 |
+
repo_temp.push_to_hub(commit_message="Reset results.csv")
|
116 |
+
|
117 |
+
def find_model(self, author, name):
|
118 |
+
""" Find a model in the models list. """
|
119 |
+
for model in self.models:
|
120 |
+
if model.author == author and model.name == name:
|
121 |
+
return model
|
122 |
+
return None
|
123 |
+
|
124 |
+
def compute_elo(self, model1, model2, result):
|
125 |
+
""" Compute the new elo for each model based on a match result. """
|
126 |
+
delta = model1.elo - model2.elo
|
127 |
+
win_probability = 1 / (1 + 10 ** (-delta / 500))
|
128 |
+
model1.elo += self.k * (result - win_probability)
|
129 |
+
model2.elo -= self.k * (result - win_probability)
|
130 |
+
|
131 |
+
def find_n_closest_indexes(self, model, n) -> int:
|
132 |
+
"""
|
133 |
+
Get a model index with a fairly close rating. If no model is found, return the last model in the queue.
|
134 |
+
We don't always pick the closest rating to add variety to the matchups.
|
135 |
+
|
136 |
+
:param model: Model to compare
|
137 |
+
:param n: Number of close models from which to pick a candidate
|
138 |
+
:return: id of the chosen candidate
|
139 |
+
"""
|
140 |
+
if len(self.queue) == 1:
|
141 |
+
return 0
|
142 |
+
indexes = []
|
143 |
+
closest_diffs = [9999999] * n
|
144 |
+
for i, m in enumerate(self.queue):
|
145 |
+
modelid1 = model.author + "/" + model.name
|
146 |
+
modelid2 = m.author + "/" + m.name
|
147 |
+
if modelid1 == modelid2:
|
148 |
+
continue
|
149 |
+
diff = abs(m.elo - model.elo)
|
150 |
+
if diff < max(closest_diffs):
|
151 |
+
closest_diffs.append(diff)
|
152 |
+
closest_diffs.sort()
|
153 |
+
closest_diffs.pop()
|
154 |
+
indexes.append(i)
|
155 |
+
random.shuffle(indexes)
|
156 |
+
return indexes[0]
|
157 |
+
|
158 |
+
def to_csv(self):
|
159 |
+
""" Save the match history as a CSV file to the hub. """
|
160 |
+
data_dict = {"rank": [], "author": [], "model": [], "elo": [], "games_played": []}
|
161 |
+
sorted_models = sorted(self.models, key=lambda x: x.elo, reverse=True)
|
162 |
+
for i, model in enumerate(sorted_models):
|
163 |
+
data_dict["rank"].append(i + 1)
|
164 |
+
data_dict["author"].append(model.author)
|
165 |
+
data_dict["model"].append(model.name)
|
166 |
+
data_dict["elo"].append(model.elo)
|
167 |
+
data_dict["games_played"].append(model.games_played)
|
168 |
+
df = pd.DataFrame(data_dict)
|
169 |
+
print(df.head())
|
170 |
+
repo.git_pull()
|
171 |
+
history = pd.read_csv(os.path.join(ELO_DIR, HISTORY_FILENAME))
|
172 |
+
new_history = pd.DataFrame(self.matches)
|
173 |
+
history = pd.concat([history, new_history])
|
174 |
+
history.to_csv(os.path.join(ELO_DIR, HISTORY_FILENAME), index=False)
|
175 |
+
df.to_csv(os.path.join(ELO_DIR, ELO_FILENAME), index=False)
|
176 |
+
repo.push_to_hub(commit_message="Update ELO")
|
177 |
+
|
178 |
+
|
179 |
+
def match(model1, model2):
|
180 |
+
"""
|
181 |
+
Simulate a match between two models using the Unity environment.
|
182 |
+
|
183 |
+
:param model1: First Model object
|
184 |
+
:param model2: Second Model object
|
185 |
+
:return: match result (0: model1 lost, 0.5: draw, 1: model1 won)
|
186 |
+
"""
|
187 |
+
model1_id = model1.author + "/" + model1.name
|
188 |
+
model2_id = model2.author + "/" + model2.name
|
189 |
+
print(f"Running {model1_id} against {model2_id}...")
|
190 |
+
subprocess.run(["./SoccerTows.x86_64", "-model1", model1_id, "-model2", model2_id, "-nographics", "-batchmode"])
|
191 |
+
print(f"Match {model1_id} against {model2_id} ended.")
|
192 |
+
|
193 |
+
|
194 |
+
def get_models_list(filter_bad_models) -> list:
|
195 |
+
"""
|
196 |
+
Get the list of models from the hub and the ELO file.
|
197 |
+
|
198 |
+
:return: list of Model objects
|
199 |
+
"""
|
200 |
+
models = []
|
201 |
+
models_ids = []
|
202 |
+
data = pd.read_csv(os.path.join(DATASET_REPO_URL, "resolve", "main", ELO_FILENAME))
|
203 |
+
models_on_hub = api.list_models(filter=["reinforcement-learning", "ml-agents", "ML-Agents-SoccerTwos", "onnx"])
|
204 |
+
for i, row in data.iterrows():
|
205 |
+
model_id = row["author"] + "/" + row["model"]
|
206 |
+
if model_id in filter_bad_models:
|
207 |
+
continue
|
208 |
+
models.append(Model(row["author"], row["model"], row["elo"], row["games_played"]))
|
209 |
+
models_ids.append(model_id)
|
210 |
+
for model in models_on_hub:
|
211 |
+
if model.modelId in filter_bad_models:
|
212 |
+
continue
|
213 |
+
author, name = model.modelId.split("/")[0], model.modelId.split("/")[1]
|
214 |
+
if model.modelId not in models_ids:
|
215 |
+
models.append(Model(author, name))
|
216 |
+
print("New model found: ", author, "-", name)
|
217 |
+
return models
|
218 |
+
|
219 |
+
|
220 |
+
def get_elo_data() -> pd.DataFrame:
|
221 |
+
"""
|
222 |
+
Get the ELO data from the hub for all the models that have played at least one game.
|
223 |
+
|
224 |
+
:return: ELO data as a pandas DataFrame
|
225 |
+
"""
|
226 |
+
repo.git_pull()
|
227 |
+
data = pd.read_csv(os.path.join(DATASET_REPO_URL, "resolve", "main", ELO_FILENAME))
|
228 |
+
|
229 |
+
return data
|
230 |
+
|
231 |
+
|
232 |
+
def init_matchmaking():
|
233 |
+
"""
|
234 |
+
Run the matchmaking algorithm and save the results to the hub.
|
235 |
+
|
236 |
+
1. Get the list of models from the hub and the ELO data
|
237 |
+
2. Match models together based on their ELO rating
|
238 |
+
3. Simulate the matches using Unity to get the match result
|
239 |
+
4. Compute the new ELO rating for each model
|
240 |
+
5. Save the results to the hub
|
241 |
+
"""
|
242 |
+
filter_bad_models = pd.read_csv(FILTER_FILE)["model"].tolist()
|
243 |
+
models = get_models_list(filter_bad_models)
|
244 |
+
matchmaking = Matchmaking(models)
|
245 |
+
matchmaking.run()
|
246 |
+
matchmaking.to_csv()
|
247 |
+
print("Matchmaking done --", datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f"))
|
matchmaking.py
ADDED
@@ -0,0 +1,76 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import random
|
2 |
+
import pandas as pd
|
3 |
+
import os
|
4 |
+
|
5 |
+
|
6 |
+
class Model:
|
7 |
+
"""
|
8 |
+
Class containing the info of a model.
|
9 |
+
|
10 |
+
:param name: Name of the model
|
11 |
+
:param elo: Elo rating of the model
|
12 |
+
:param games_played: Number of games played by the model (useful if we implement sigma uncertainty)
|
13 |
+
"""
|
14 |
+
def __init__(self, name, elo):
|
15 |
+
self.name = name
|
16 |
+
self.elo = elo
|
17 |
+
self.games_played = 0
|
18 |
+
|
19 |
+
|
20 |
+
class Matchmaking:
|
21 |
+
"""
|
22 |
+
Class managing the matchmaking between the models.
|
23 |
+
|
24 |
+
:param models: List of models
|
25 |
+
:param queue: Temporary list of models used for the matching process
|
26 |
+
:param k: Dev coefficient
|
27 |
+
:param max_diff: Maximum difference considered between two models' elo
|
28 |
+
:param matches: Dictionary containing the match history (to later upload as CSV)
|
29 |
+
"""
|
30 |
+
def __init__(self):
|
31 |
+
self.models = []
|
32 |
+
self.queue = []
|
33 |
+
self.start_elo = 1200
|
34 |
+
self.k = 20
|
35 |
+
self.max_diff = 500
|
36 |
+
self.matches = pd.DataFrame()
|
37 |
+
|
38 |
+
def read_history(self):
|
39 |
+
""" Read the match history from the CSV files, concat the Dataframes and sort them by datetime. """
|
40 |
+
path = "match_history"
|
41 |
+
files = os.listdir(path)
|
42 |
+
for file in files:
|
43 |
+
self.matches = pd.concat([self.matches, pd.read_csv(os.path.join(path, file))], ignore_index=True)
|
44 |
+
self.matches["datetime"] = pd.to_datetime(self.matches["datetime"], format="%Y-%m-%d %H:%M:%S.%f", errors="coerce")
|
45 |
+
self.matches = self.matches.dropna()
|
46 |
+
self.matches = self.matches.sort_values("datetime")
|
47 |
+
self.matches.reset_index(drop=True, inplace=True)
|
48 |
+
model_names = self.matches["model1"].unique()
|
49 |
+
self.models = [Model(name, self.start_elo) for name in model_names]
|
50 |
+
|
51 |
+
def compute_elo(self):
|
52 |
+
""" Compute the elo for each model after each match. """
|
53 |
+
for i, row in self.matches.iterrows():
|
54 |
+
model1 = self.get_model(row["model1"])
|
55 |
+
model2 = self.get_model(row["model2"])
|
56 |
+
result = row["result"]
|
57 |
+
delta = model1.elo - model2.elo
|
58 |
+
win_probability = 1 / (1 + 10 ** (-delta / 500))
|
59 |
+
model1.elo += self.k * (result - win_probability)
|
60 |
+
model2.elo -= self.k * (result - win_probability)
|
61 |
+
model1.games_played += 1
|
62 |
+
model2.games_played += 1
|
63 |
+
|
64 |
+
def save_elo_data(self):
|
65 |
+
""" Save the match history as a CSV file to the hub. """
|
66 |
+
df = pd.DataFrame(columns=['name', 'elo'])
|
67 |
+
for model in self.models:
|
68 |
+
df = pd.concat([df, pd.DataFrame([[model.name, model.elo]], columns=['name', 'elo'])])
|
69 |
+
df.to_csv('elo.csv', index=False)
|
70 |
+
|
71 |
+
def get_model(self, name):
|
72 |
+
""" Return the Model with the given name. """
|
73 |
+
for model in self.models:
|
74 |
+
if model.name == name:
|
75 |
+
return model
|
76 |
+
return None
|
requirements.txt
ADDED
@@ -0,0 +1,5 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
requests~=2.28.1
|
2 |
+
gradio~=3.14.0
|
3 |
+
pandas~=1.5.2
|
4 |
+
datasets~=2.8.0
|
5 |
+
APScheduler~=3.9.1.post1
|
utils.py
ADDED
@@ -0,0 +1,13 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
# Based on Omar Sanseviero work
|
2 |
+
# Make model clickable link
|
3 |
+
def make_clickable_model(model_name):
|
4 |
+
# remove user from model name
|
5 |
+
model_name_show = ' '.join(model_name.split('/')[1:])
|
6 |
+
|
7 |
+
link = "https://huggingface.co/" + model_name
|
8 |
+
return f'<a target="_blank" href="{link}">{model_name_show}</a>'
|
9 |
+
|
10 |
+
# Make user clickable link
|
11 |
+
def make_clickable_user(user_id):
|
12 |
+
link = "https://huggingface.co/" + user_id
|
13 |
+
return f'<a target="_blank" href="{link}">{user_id}</a>'
|