tfrere commited on
Commit
8695aa8
·
1 Parent(s): c2b7f1b

normalize config file and update default benchmark model

Browse files
backend/config/models_config.py CHANGED
@@ -18,17 +18,22 @@ DEFAULT_EVALUATION_MODELS = [
18
  "mistralai/Mistral-Small-24B-Instruct-2501",
19
  ]
20
 
 
 
 
 
 
21
  # Required model for create_bench_config_file.py (only one default model)
22
- DEFAULT_MODEL = "deepseek-ai/DeepSeek-R1-Distill-Llama-70B"
23
 
24
  # Models by roles for benchmark configuration
25
  # All roles use the default model except chunking
26
- MODEL_ROLES = {
27
- "ingestion": [DEFAULT_MODEL],
28
- "summarization": [DEFAULT_MODEL],
29
  "chunking": ["intfloat/multilingual-e5-large-instruct"],
30
- "single_shot_question_generation": [DEFAULT_MODEL],
31
- "multi_hop_question_generation": [DEFAULT_MODEL],
32
  }
33
 
34
  # Default evaluation timeout (in seconds)
 
18
  "mistralai/Mistral-Small-24B-Instruct-2501",
19
  ]
20
 
21
+ # "Qwen/Qwen2.5-72B-Instruct"
22
+ # "meta-llama/Llama-3.1-8B-Instruct"
23
+ # "Qwen/Qwen2.5-32B-Instruct",
24
+ # "deepseek-ai/DeepSeek-R1-Distill-Llama-70B",
25
+
26
  # Required model for create_bench_config_file.py (only one default model)
27
+ DEFAULT_BENCHMARK_MODEL = "Qwen/Qwen2.5-32B-Instruct"
28
 
29
  # Models by roles for benchmark configuration
30
  # All roles use the default model except chunking
31
+ BENCHMARK_MODEL_ROLES = {
32
+ "ingestion": [DEFAULT_BENCHMARK_MODEL],
33
+ "summarization": [DEFAULT_BENCHMARK_MODEL],
34
  "chunking": ["intfloat/multilingual-e5-large-instruct"],
35
+ "single_shot_question_generation": [DEFAULT_BENCHMARK_MODEL],
36
+ "multi_hop_question_generation": [DEFAULT_BENCHMARK_MODEL],
37
  }
38
 
39
  # Default evaluation timeout (in seconds)
backend/tasks/create_bench_config_file.py CHANGED
@@ -15,8 +15,8 @@ from huggingface_hub import HfApi
15
 
16
  from tasks.get_available_model_provider import get_available_model_provider
17
  from config.models_config import (
18
- DEFAULT_MODEL,
19
- MODEL_ROLES,
20
  DEFAULT_BENCHMARK_TIMEOUT,
21
  )
22
 
@@ -124,15 +124,15 @@ class CreateBenchConfigTask:
124
  raise RuntimeError("HF_TOKEN environment variable is not defined")
125
 
126
  # Get provider for the default model
127
- provider = self.get_model_provider(DEFAULT_MODEL)
128
  if not provider:
129
- error_msg = f"Required model not available: {DEFAULT_MODEL}. Cannot proceed with benchmark."
130
  self._add_log(f"[ERROR] {error_msg}")
131
  raise RuntimeError(error_msg)
132
 
133
  # Create model configuration
134
  model_list = [{
135
- "model_name": DEFAULT_MODEL,
136
  "provider": provider,
137
  "api_key": "$HF_TOKEN",
138
  "max_concurrent_requests": 32,
@@ -156,7 +156,7 @@ class CreateBenchConfigTask:
156
  },
157
  "model_list": model_list,
158
 
159
- "model_roles": MODEL_ROLES,
160
  "pipeline": {
161
  "ingestion": {
162
  "source_documents_dir": f"uploaded_files/{self.session_uid}/uploaded_files/",
 
15
 
16
  from tasks.get_available_model_provider import get_available_model_provider
17
  from config.models_config import (
18
+ DEFAULT_BENCHMARK_MODEL,
19
+ BENCHMARK_MODEL_ROLES,
20
  DEFAULT_BENCHMARK_TIMEOUT,
21
  )
22
 
 
124
  raise RuntimeError("HF_TOKEN environment variable is not defined")
125
 
126
  # Get provider for the default model
127
+ provider = self.get_model_provider(DEFAULT_BENCHMARK_MODEL)
128
  if not provider:
129
+ error_msg = f"Required model not available: {DEFAULT_BENCHMARK_MODEL}. Cannot proceed with benchmark."
130
  self._add_log(f"[ERROR] {error_msg}")
131
  raise RuntimeError(error_msg)
132
 
133
  # Create model configuration
134
  model_list = [{
135
+ "model_name": DEFAULT_BENCHMARK_MODEL,
136
  "provider": provider,
137
  "api_key": "$HF_TOKEN",
138
  "max_concurrent_requests": 32,
 
156
  },
157
  "model_list": model_list,
158
 
159
+ "model_roles": BENCHMARK_MODEL_ROLES,
160
  "pipeline": {
161
  "ingestion": {
162
  "source_documents_dir": f"uploaded_files/{self.session_uid}/uploaded_files/",