tfrere commited on
Commit
4759fe1
·
1 Parent(s): 5af2380

update model handling in benchmark generation

Browse files
backend/main.py CHANGED
@@ -3,6 +3,7 @@ from fastapi.middleware.cors import CORSMiddleware
3
  import os
4
  from dotenv import load_dotenv
5
  from routes import routers, session_files, active_tasks, benchmark
 
6
 
7
  # Load environment variables from .env file
8
  load_dotenv()
@@ -32,6 +33,15 @@ app.add_middleware(
32
  async def startup_event():
33
  print("Application startup")
34
  print(f"Initial session_files: {session_files}")
 
 
 
 
 
 
 
 
 
35
 
36
  # Enregistrer toutes les routes
37
  for router in routers:
 
3
  import os
4
  from dotenv import load_dotenv
5
  from routes import routers, session_files, active_tasks, benchmark
6
+ from tasks.get_available_model_provider import test_models
7
 
8
  # Load environment variables from .env file
9
  load_dotenv()
 
33
  async def startup_event():
34
  print("Application startup")
35
  print(f"Initial session_files: {session_files}")
36
+
37
+ # Tester les modèles au démarrage et afficher les résultats
38
+ print("===== Testing model availability at startup =====")
39
+ test_results = test_models(verbose=True)
40
+ print("===== Model testing completed =====")
41
+ if test_results["working_model"]:
42
+ print(f"✅ Found working model: {test_results['working_model']} with provider: {test_results['provider']}")
43
+ else:
44
+ print("❌ WARNING: No working models found. The application might not function correctly!")
45
 
46
  # Enregistrer toutes les routes
47
  for router in routers:
backend/tasks/get_available_model_provider.py CHANGED
@@ -3,7 +3,7 @@ import logging
3
  import json
4
  from huggingface_hub import model_info, InferenceClient
5
  from dotenv import load_dotenv
6
- from config.models_config import PREFERRED_PROVIDERS
7
 
8
  # Load environment variables once at the module level
9
  load_dotenv()
@@ -30,6 +30,9 @@ def test_provider(model_name: str, provider: str, verbose: bool = False) -> bool
30
  """
31
 
32
  try:
 
 
 
33
  # Get HF token from environment
34
  hf_token = os.environ.get("HF_TOKEN")
35
  if not hf_token:
@@ -177,32 +180,93 @@ def get_available_model_provider(model_name, verbose=False):
177
  logger.error(f"Error in get_available_model_provider: {str(e)}")
178
  return None
179
 
180
- if __name__ == "__main__":
181
-
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
182
  models = [
183
  "Qwen/QwQ-32B",
184
  "Qwen/Qwen2.5-72B-Instruct",
 
 
185
  "meta-llama/Llama-3.3-70B-Instruct",
186
  "deepseek-ai/DeepSeek-R1-Distill-Llama-70B",
187
  "mistralai/Mistral-Small-24B-Instruct-2501",
188
- "meta-llama/Llama-3.1-8B-Instruct",
189
- "Qwen/Qwen2.5-32B-Instruct"
190
  ]
191
-
192
- providers = []
193
- unavailable_models = []
194
-
195
  for model in models:
196
- provider = get_available_model_provider(model, verbose=True)
 
197
  if provider:
198
- providers.append((model, provider))
199
  else:
200
- unavailable_models.append(model)
201
-
202
- for model, provider in providers:
203
- print(f"Model: {model}, Provider: {provider}")
 
 
 
 
 
204
 
205
- if unavailable_models:
206
- print(f"Models with no available providers: {', '.join(unavailable_models)}")
207
 
208
- print(f"Total Providers {len(providers)}: {providers}")
 
 
 
 
 
3
  import json
4
  from huggingface_hub import model_info, InferenceClient
5
  from dotenv import load_dotenv
6
+ from config.models_config import PREFERRED_PROVIDERS, DEFAULT_BENCHMARK_MODEL, ALTERNATIVE_BENCHMARK_MODELS
7
 
8
  # Load environment variables once at the module level
9
  load_dotenv()
 
30
  """
31
 
32
  try:
33
+
34
+ load_dotenv()
35
+
36
  # Get HF token from environment
37
  hf_token = os.environ.get("HF_TOKEN")
38
  if not hf_token:
 
180
  logger.error(f"Error in get_available_model_provider: {str(e)}")
181
  return None
182
 
183
+ def test_models(verbose=True):
184
+ """
185
+ Test le modèle par défaut et les modèles alternatifs, puis retourne un résumé des résultats.
186
+
187
+ Args:
188
+ verbose: Afficher les logs détaillés
189
+
190
+ Returns:
191
+ Un dictionnaire avec les résultats des tests
192
+ """
193
+ results = {
194
+ "default_model": None,
195
+ "working_model": None,
196
+ "provider": None,
197
+ "all_models": {},
198
+ "available_models": [],
199
+ "unavailable_models": []
200
+ }
201
+
202
+ if verbose:
203
+ print(f"Testing main default model: {DEFAULT_BENCHMARK_MODEL}")
204
+
205
+ # Test du modèle par défaut
206
+ provider = get_available_model_provider(DEFAULT_BENCHMARK_MODEL, verbose=verbose)
207
+
208
+ if provider:
209
+ if verbose:
210
+ print(f"\n✅ SUCCESS: Found provider for default model {DEFAULT_BENCHMARK_MODEL}: {provider}")
211
+ results["default_model"] = DEFAULT_BENCHMARK_MODEL
212
+ results["working_model"] = DEFAULT_BENCHMARK_MODEL
213
+ results["provider"] = provider
214
+ else:
215
+ if verbose:
216
+ print(f"\n❌ DEFAULT MODEL FAILED: No provider found for {DEFAULT_BENCHMARK_MODEL}")
217
+ print("Trying alternative models...")
218
+
219
+ # Essayer les modèles alternatifs
220
+ for alt_model in ALTERNATIVE_BENCHMARK_MODELS:
221
+ if verbose:
222
+ print(f"\nTrying alternative model: {alt_model}")
223
+ alt_provider = get_available_model_provider(alt_model, verbose=verbose)
224
+ if alt_provider:
225
+ if verbose:
226
+ print(f"\n✅ SUCCESS: Found provider for alternative model {alt_model}: {alt_provider}")
227
+ results["working_model"] = alt_model
228
+ results["provider"] = alt_provider
229
+ break
230
+ elif verbose:
231
+ print(f"❌ Failed to find provider for alternative model: {alt_model}")
232
+ else:
233
+ if verbose:
234
+ print("\n❌ ALL MODELS FAILED: No provider found for any model")
235
+
236
+ # Tester tous les modèles pour avoir une vue d'ensemble
237
  models = [
238
  "Qwen/QwQ-32B",
239
  "Qwen/Qwen2.5-72B-Instruct",
240
+ "Qwen/Qwen2.5-32B-Instruct",
241
+ "meta-llama/Llama-3.1-8B-Instruct",
242
  "meta-llama/Llama-3.3-70B-Instruct",
243
  "deepseek-ai/DeepSeek-R1-Distill-Llama-70B",
244
  "mistralai/Mistral-Small-24B-Instruct-2501",
 
 
245
  ]
246
+
247
+ if verbose:
248
+ print("\n=== Testing all available models ===")
249
+
250
  for model in models:
251
+ provider = get_available_model_provider(model, verbose)
252
+ results["all_models"][model] = provider
253
  if provider:
254
+ results["available_models"].append((model, provider))
255
  else:
256
+ results["unavailable_models"].append(model)
257
+
258
+ if verbose:
259
+ print("\n=== Results Summary ===")
260
+ for model, provider in results["available_models"]:
261
+ print(f"Model: {model}, Provider: {provider}")
262
+
263
+ if results["unavailable_models"]:
264
+ print(f"Models with no available providers: {', '.join(results['unavailable_models'])}")
265
 
266
+ print(f"Total Available Models: {len(results['available_models'])}")
 
267
 
268
+ return results
269
+
270
+ if __name__ == "__main__":
271
+ # Exécuter le test si le script est lancé directement
272
+ test_results = test_models(verbose=True)