ThomasSimonini HF Staff commited on
Commit
7794cdb
·
verified ·
1 Parent(s): 10ac6b8

Upload 5 files

Browse files
Files changed (5) hide show
  1. app.py +45 -0
  2. background_task.py +247 -0
  3. matchmaking.py +76 -0
  4. requirements.txt +5 -0
  5. utils.py +13 -0
app.py ADDED
@@ -0,0 +1,45 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import gradio as gr
2
+ from huggingface_hub import HfApi
3
+ from matchmaking import *
4
+ from background_task import init_matchmaking, get_elo_data
5
+ from apscheduler.schedulers.background import BackgroundScheduler
6
+ from utils import *
7
+
8
+ matchmaking = Matchmaking()
9
+ api = HfApi()
10
+
11
+ # launch
12
+ scheduler = BackgroundScheduler()
13
+ scheduler.add_job(func=init_matchmaking, trigger="interval", seconds=300)
14
+ scheduler.start()
15
+
16
+
17
+ def update_elos():
18
+ matchmaking.read_history()
19
+ matchmaking.compute_elo()
20
+ matchmaking.save_elo_data()
21
+
22
+
23
+ with gr.Blocks() as block:
24
+ gr.Markdown(f"""
25
+ # 🏆 AI vs. AI SoccerTwos Leaderboard ⚽
26
+
27
+ In this leaderboard, you can find the ELO score and the rank of your trained model for the SoccerTwos environment.
28
+
29
+ If you want to know more about a model, just **copy the username and model and paste them into the search bar**.
30
+
31
+ 👀 To visualize your agents competing check this demo: https://huggingface.co/spaces/unity/ML-Agents-SoccerTwos
32
+
33
+ 🤖 For more information about this AI vs. AI challenge and to participate? [Check this](https://huggingface.co/deep-rl-course/unit7)
34
+ """)
35
+ with gr.Row():
36
+ output = gr.components.Dataframe(
37
+ value=get_elo_data,
38
+ headers=["Ranking 🏆", "User 🤗", "Model id 🤖", "ELO 🚀", "Games played 🎮"],
39
+ datatype=["number", "markdown", "markdown", "number", "number"]
40
+ )
41
+ with gr.Row():
42
+ refresh = gr.Button("Refresh")
43
+ refresh.click(get_elo_data, inputs=[], outputs=output)
44
+
45
+ block.launch()
background_task.py ADDED
@@ -0,0 +1,247 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ import random
3
+ import subprocess
4
+ import pandas as pd
5
+ from datetime import datetime
6
+ from huggingface_hub import HfApi, Repository
7
+ from utils import *
8
+
9
+ DATASET_REPO_URL = "https://huggingface.co/datasets/huggingface-projects/bot-fight-data"
10
+ DATASET_TEMP_REPO_URL = "https://huggingface.co/datasets/huggingface-projects/temp-match-results"
11
+ FILTER_FILE = "https://huggingface.co/datasets/huggingface-projects/filter-bad-models/raw/main/bad_models.csv"
12
+ ELO_FILENAME = "soccer_elo.csv"
13
+ HISTORY_FILENAME = "soccer_history.csv"
14
+ TEMP_FILENAME = "results.csv"
15
+ ELO_DIR = "soccer_elo"
16
+ TEMP_DIR = "temp"
17
+ HF_TOKEN = os.environ.get("HF_TOKEN")
18
+
19
+ repo = Repository(
20
+ local_dir=ELO_DIR, clone_from=DATASET_REPO_URL, use_auth_token=HF_TOKEN
21
+ )
22
+ repo_temp = Repository(
23
+ local_dir=TEMP_DIR, clone_from=DATASET_TEMP_REPO_URL, use_auth_token=HF_TOKEN
24
+ )
25
+
26
+ api = HfApi()
27
+ os.chmod('./SoccerTows.x86_64', 0o755)
28
+
29
+
30
+ class Model:
31
+ """
32
+ Class containing the info of a model.
33
+
34
+ :param name: Name of the model
35
+ :param elo: Elo rating of the model
36
+ :param games_played: Number of games played by the model (useful if we implement sigma uncertainty)
37
+ """
38
+
39
+ def __init__(self, author, name, elo=1200, games_played=0):
40
+ self.author = author
41
+ self.name = name
42
+ self.elo = elo
43
+ self.games_played = games_played
44
+
45
+
46
+ class Matchmaking:
47
+ """
48
+ Class managing the matchmaking between the models.
49
+
50
+ :param models: List of models
51
+ :param queue: Temporary list of models used for the matching process
52
+ :param k: Dev coefficient
53
+ :param max_diff: Maximum difference considered between two models' elo
54
+ :param matches: Dictionary containing the match history (to later upload as CSV)
55
+ """
56
+
57
+ def __init__(self, models):
58
+ self.models = models
59
+ self.queue = self.models.copy()
60
+ self.k = 20
61
+ self.max_diff = 500
62
+ self.matches = {
63
+ "model1": [],
64
+ "model2": [],
65
+ "timestamp": [],
66
+ "result": [],
67
+ }
68
+
69
+ def run(self):
70
+ """
71
+ Run the matchmaking process.
72
+ Add models to the queue, shuffle it, and match the models one by one to models with close ratings.
73
+ Compute the new elo for each model after each match and add the match to the match history.
74
+ """
75
+ self.queue = self.models.copy()
76
+ random.shuffle(self.queue)
77
+ while len(self.queue) > 1:
78
+ print(f"Queue length: {len(self.queue)}")
79
+ model1 = self.queue.pop(0)
80
+ model2 = self.queue.pop(self.find_n_closest_indexes(model1, 10))
81
+ match(model1, model2)
82
+ self.load_results()
83
+
84
+ def load_results(self):
85
+ """ Load the match history from the hub. """
86
+ repo.git_pull()
87
+ results = pd.read_csv(
88
+ "https://huggingface.co/datasets/huggingface-projects/temp-match-results/raw/main/results.csv"
89
+ )
90
+ # while len(results) < len(self.matches["model1"]):
91
+ # time.sleep(60)
92
+ # results = pd.read_csv(
93
+ # "https://huggingface.co/datasets/huggingface-projects/temp-match-results/raw/main/results.csv"
94
+ # )
95
+
96
+ for i, row in results.iterrows():
97
+ model1 = row["model1"].split("/")
98
+ model2 = row["model2"].split("/")
99
+ model1 = self.find_model(model1[0], model1[1])
100
+ model2 = self.find_model(model2[0], model2[1])
101
+ result = row["result"]
102
+ if model1 is not None or model2 is not None:
103
+ self.compute_elo(model1, model2, row["result"])
104
+ self.matches["model1"].append(model1.author + "/" + model1.name)
105
+ self.matches["model2"].append(model2.author + "/" + model2.name)
106
+ self.matches["result"].append(result)
107
+ self.matches["timestamp"].append(row["timestamp"])
108
+ model1.games_played += 1
109
+ model2.games_played += 1
110
+ data_dict = {"model1": [], "model2": [], "timestamp": [], "result": []}
111
+ df = pd.DataFrame(data_dict)
112
+ print(df.head())
113
+ repo_temp.git_pull()
114
+ df.to_csv(os.path.join(TEMP_DIR, TEMP_FILENAME), index=False)
115
+ repo_temp.push_to_hub(commit_message="Reset results.csv")
116
+
117
+ def find_model(self, author, name):
118
+ """ Find a model in the models list. """
119
+ for model in self.models:
120
+ if model.author == author and model.name == name:
121
+ return model
122
+ return None
123
+
124
+ def compute_elo(self, model1, model2, result):
125
+ """ Compute the new elo for each model based on a match result. """
126
+ delta = model1.elo - model2.elo
127
+ win_probability = 1 / (1 + 10 ** (-delta / 500))
128
+ model1.elo += self.k * (result - win_probability)
129
+ model2.elo -= self.k * (result - win_probability)
130
+
131
+ def find_n_closest_indexes(self, model, n) -> int:
132
+ """
133
+ Get a model index with a fairly close rating. If no model is found, return the last model in the queue.
134
+ We don't always pick the closest rating to add variety to the matchups.
135
+
136
+ :param model: Model to compare
137
+ :param n: Number of close models from which to pick a candidate
138
+ :return: id of the chosen candidate
139
+ """
140
+ if len(self.queue) == 1:
141
+ return 0
142
+ indexes = []
143
+ closest_diffs = [9999999] * n
144
+ for i, m in enumerate(self.queue):
145
+ modelid1 = model.author + "/" + model.name
146
+ modelid2 = m.author + "/" + m.name
147
+ if modelid1 == modelid2:
148
+ continue
149
+ diff = abs(m.elo - model.elo)
150
+ if diff < max(closest_diffs):
151
+ closest_diffs.append(diff)
152
+ closest_diffs.sort()
153
+ closest_diffs.pop()
154
+ indexes.append(i)
155
+ random.shuffle(indexes)
156
+ return indexes[0]
157
+
158
+ def to_csv(self):
159
+ """ Save the match history as a CSV file to the hub. """
160
+ data_dict = {"rank": [], "author": [], "model": [], "elo": [], "games_played": []}
161
+ sorted_models = sorted(self.models, key=lambda x: x.elo, reverse=True)
162
+ for i, model in enumerate(sorted_models):
163
+ data_dict["rank"].append(i + 1)
164
+ data_dict["author"].append(model.author)
165
+ data_dict["model"].append(model.name)
166
+ data_dict["elo"].append(model.elo)
167
+ data_dict["games_played"].append(model.games_played)
168
+ df = pd.DataFrame(data_dict)
169
+ print(df.head())
170
+ repo.git_pull()
171
+ history = pd.read_csv(os.path.join(ELO_DIR, HISTORY_FILENAME))
172
+ new_history = pd.DataFrame(self.matches)
173
+ history = pd.concat([history, new_history])
174
+ history.to_csv(os.path.join(ELO_DIR, HISTORY_FILENAME), index=False)
175
+ df.to_csv(os.path.join(ELO_DIR, ELO_FILENAME), index=False)
176
+ repo.push_to_hub(commit_message="Update ELO")
177
+
178
+
179
+ def match(model1, model2):
180
+ """
181
+ Simulate a match between two models using the Unity environment.
182
+
183
+ :param model1: First Model object
184
+ :param model2: Second Model object
185
+ :return: match result (0: model1 lost, 0.5: draw, 1: model1 won)
186
+ """
187
+ model1_id = model1.author + "/" + model1.name
188
+ model2_id = model2.author + "/" + model2.name
189
+ print(f"Running {model1_id} against {model2_id}...")
190
+ subprocess.run(["./SoccerTows.x86_64", "-model1", model1_id, "-model2", model2_id, "-nographics", "-batchmode"])
191
+ print(f"Match {model1_id} against {model2_id} ended.")
192
+
193
+
194
+ def get_models_list(filter_bad_models) -> list:
195
+ """
196
+ Get the list of models from the hub and the ELO file.
197
+
198
+ :return: list of Model objects
199
+ """
200
+ models = []
201
+ models_ids = []
202
+ data = pd.read_csv(os.path.join(DATASET_REPO_URL, "resolve", "main", ELO_FILENAME))
203
+ models_on_hub = api.list_models(filter=["reinforcement-learning", "ml-agents", "ML-Agents-SoccerTwos", "onnx"])
204
+ for i, row in data.iterrows():
205
+ model_id = row["author"] + "/" + row["model"]
206
+ if model_id in filter_bad_models:
207
+ continue
208
+ models.append(Model(row["author"], row["model"], row["elo"], row["games_played"]))
209
+ models_ids.append(model_id)
210
+ for model in models_on_hub:
211
+ if model.modelId in filter_bad_models:
212
+ continue
213
+ author, name = model.modelId.split("/")[0], model.modelId.split("/")[1]
214
+ if model.modelId not in models_ids:
215
+ models.append(Model(author, name))
216
+ print("New model found: ", author, "-", name)
217
+ return models
218
+
219
+
220
+ def get_elo_data() -> pd.DataFrame:
221
+ """
222
+ Get the ELO data from the hub for all the models that have played at least one game.
223
+
224
+ :return: ELO data as a pandas DataFrame
225
+ """
226
+ repo.git_pull()
227
+ data = pd.read_csv(os.path.join(DATASET_REPO_URL, "resolve", "main", ELO_FILENAME))
228
+
229
+ return data
230
+
231
+
232
+ def init_matchmaking():
233
+ """
234
+ Run the matchmaking algorithm and save the results to the hub.
235
+
236
+ 1. Get the list of models from the hub and the ELO data
237
+ 2. Match models together based on their ELO rating
238
+ 3. Simulate the matches using Unity to get the match result
239
+ 4. Compute the new ELO rating for each model
240
+ 5. Save the results to the hub
241
+ """
242
+ filter_bad_models = pd.read_csv(FILTER_FILE)["model"].tolist()
243
+ models = get_models_list(filter_bad_models)
244
+ matchmaking = Matchmaking(models)
245
+ matchmaking.run()
246
+ matchmaking.to_csv()
247
+ print("Matchmaking done --", datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f"))
matchmaking.py ADDED
@@ -0,0 +1,76 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import random
2
+ import pandas as pd
3
+ import os
4
+
5
+
6
+ class Model:
7
+ """
8
+ Class containing the info of a model.
9
+
10
+ :param name: Name of the model
11
+ :param elo: Elo rating of the model
12
+ :param games_played: Number of games played by the model (useful if we implement sigma uncertainty)
13
+ """
14
+ def __init__(self, name, elo):
15
+ self.name = name
16
+ self.elo = elo
17
+ self.games_played = 0
18
+
19
+
20
+ class Matchmaking:
21
+ """
22
+ Class managing the matchmaking between the models.
23
+
24
+ :param models: List of models
25
+ :param queue: Temporary list of models used for the matching process
26
+ :param k: Dev coefficient
27
+ :param max_diff: Maximum difference considered between two models' elo
28
+ :param matches: Dictionary containing the match history (to later upload as CSV)
29
+ """
30
+ def __init__(self):
31
+ self.models = []
32
+ self.queue = []
33
+ self.start_elo = 1200
34
+ self.k = 20
35
+ self.max_diff = 500
36
+ self.matches = pd.DataFrame()
37
+
38
+ def read_history(self):
39
+ """ Read the match history from the CSV files, concat the Dataframes and sort them by datetime. """
40
+ path = "match_history"
41
+ files = os.listdir(path)
42
+ for file in files:
43
+ self.matches = pd.concat([self.matches, pd.read_csv(os.path.join(path, file))], ignore_index=True)
44
+ self.matches["datetime"] = pd.to_datetime(self.matches["datetime"], format="%Y-%m-%d %H:%M:%S.%f", errors="coerce")
45
+ self.matches = self.matches.dropna()
46
+ self.matches = self.matches.sort_values("datetime")
47
+ self.matches.reset_index(drop=True, inplace=True)
48
+ model_names = self.matches["model1"].unique()
49
+ self.models = [Model(name, self.start_elo) for name in model_names]
50
+
51
+ def compute_elo(self):
52
+ """ Compute the elo for each model after each match. """
53
+ for i, row in self.matches.iterrows():
54
+ model1 = self.get_model(row["model1"])
55
+ model2 = self.get_model(row["model2"])
56
+ result = row["result"]
57
+ delta = model1.elo - model2.elo
58
+ win_probability = 1 / (1 + 10 ** (-delta / 500))
59
+ model1.elo += self.k * (result - win_probability)
60
+ model2.elo -= self.k * (result - win_probability)
61
+ model1.games_played += 1
62
+ model2.games_played += 1
63
+
64
+ def save_elo_data(self):
65
+ """ Save the match history as a CSV file to the hub. """
66
+ df = pd.DataFrame(columns=['name', 'elo'])
67
+ for model in self.models:
68
+ df = pd.concat([df, pd.DataFrame([[model.name, model.elo]], columns=['name', 'elo'])])
69
+ df.to_csv('elo.csv', index=False)
70
+
71
+ def get_model(self, name):
72
+ """ Return the Model with the given name. """
73
+ for model in self.models:
74
+ if model.name == name:
75
+ return model
76
+ return None
requirements.txt ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ requests~=2.28.1
2
+ gradio~=3.14.0
3
+ pandas~=1.5.2
4
+ datasets~=2.8.0
5
+ APScheduler~=3.9.1.post1
utils.py ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Based on Omar Sanseviero work
2
+ # Make model clickable link
3
+ def make_clickable_model(model_name):
4
+ # remove user from model name
5
+ model_name_show = ' '.join(model_name.split('/')[1:])
6
+
7
+ link = "https://huggingface.co/" + model_name
8
+ return f'<a target="_blank" href="{link}">{model_name_show}</a>'
9
+
10
+ # Make user clickable link
11
+ def make_clickable_user(user_id):
12
+ link = "https://huggingface.co/" + user_id
13
+ return f'<a target="_blank" href="{link}">{user_id}</a>'