Spaces:
Running
on
CPU Upgrade
Running
on
CPU Upgrade
normalize config file and update default benchmark model
Browse files
backend/config/models_config.py
CHANGED
@@ -18,17 +18,22 @@ DEFAULT_EVALUATION_MODELS = [
|
|
18 |
"mistralai/Mistral-Small-24B-Instruct-2501",
|
19 |
]
|
20 |
|
|
|
|
|
|
|
|
|
|
|
21 |
# Required model for create_bench_config_file.py (only one default model)
|
22 |
-
|
23 |
|
24 |
# Models by roles for benchmark configuration
|
25 |
# All roles use the default model except chunking
|
26 |
-
|
27 |
-
"ingestion": [
|
28 |
-
"summarization": [
|
29 |
"chunking": ["intfloat/multilingual-e5-large-instruct"],
|
30 |
-
"single_shot_question_generation": [
|
31 |
-
"multi_hop_question_generation": [
|
32 |
}
|
33 |
|
34 |
# Default evaluation timeout (in seconds)
|
|
|
18 |
"mistralai/Mistral-Small-24B-Instruct-2501",
|
19 |
]
|
20 |
|
21 |
+
# "Qwen/Qwen2.5-72B-Instruct"
|
22 |
+
# "meta-llama/Llama-3.1-8B-Instruct"
|
23 |
+
# "Qwen/Qwen2.5-32B-Instruct",
|
24 |
+
# "deepseek-ai/DeepSeek-R1-Distill-Llama-70B",
|
25 |
+
|
26 |
# Required model for create_bench_config_file.py (only one default model)
|
27 |
+
DEFAULT_BENCHMARK_MODEL = "Qwen/Qwen2.5-32B-Instruct"
|
28 |
|
29 |
# Models by roles for benchmark configuration
|
30 |
# All roles use the default model except chunking
|
31 |
+
BENCHMARK_MODEL_ROLES = {
|
32 |
+
"ingestion": [DEFAULT_BENCHMARK_MODEL],
|
33 |
+
"summarization": [DEFAULT_BENCHMARK_MODEL],
|
34 |
"chunking": ["intfloat/multilingual-e5-large-instruct"],
|
35 |
+
"single_shot_question_generation": [DEFAULT_BENCHMARK_MODEL],
|
36 |
+
"multi_hop_question_generation": [DEFAULT_BENCHMARK_MODEL],
|
37 |
}
|
38 |
|
39 |
# Default evaluation timeout (in seconds)
|
backend/tasks/create_bench_config_file.py
CHANGED
@@ -15,8 +15,8 @@ from huggingface_hub import HfApi
|
|
15 |
|
16 |
from tasks.get_available_model_provider import get_available_model_provider
|
17 |
from config.models_config import (
|
18 |
-
|
19 |
-
|
20 |
DEFAULT_BENCHMARK_TIMEOUT,
|
21 |
)
|
22 |
|
@@ -124,15 +124,15 @@ class CreateBenchConfigTask:
|
|
124 |
raise RuntimeError("HF_TOKEN environment variable is not defined")
|
125 |
|
126 |
# Get provider for the default model
|
127 |
-
provider = self.get_model_provider(
|
128 |
if not provider:
|
129 |
-
error_msg = f"Required model not available: {
|
130 |
self._add_log(f"[ERROR] {error_msg}")
|
131 |
raise RuntimeError(error_msg)
|
132 |
|
133 |
# Create model configuration
|
134 |
model_list = [{
|
135 |
-
"model_name":
|
136 |
"provider": provider,
|
137 |
"api_key": "$HF_TOKEN",
|
138 |
"max_concurrent_requests": 32,
|
@@ -156,7 +156,7 @@ class CreateBenchConfigTask:
|
|
156 |
},
|
157 |
"model_list": model_list,
|
158 |
|
159 |
-
"model_roles":
|
160 |
"pipeline": {
|
161 |
"ingestion": {
|
162 |
"source_documents_dir": f"uploaded_files/{self.session_uid}/uploaded_files/",
|
|
|
15 |
|
16 |
from tasks.get_available_model_provider import get_available_model_provider
|
17 |
from config.models_config import (
|
18 |
+
DEFAULT_BENCHMARK_MODEL,
|
19 |
+
BENCHMARK_MODEL_ROLES,
|
20 |
DEFAULT_BENCHMARK_TIMEOUT,
|
21 |
)
|
22 |
|
|
|
124 |
raise RuntimeError("HF_TOKEN environment variable is not defined")
|
125 |
|
126 |
# Get provider for the default model
|
127 |
+
provider = self.get_model_provider(DEFAULT_BENCHMARK_MODEL)
|
128 |
if not provider:
|
129 |
+
error_msg = f"Required model not available: {DEFAULT_BENCHMARK_MODEL}. Cannot proceed with benchmark."
|
130 |
self._add_log(f"[ERROR] {error_msg}")
|
131 |
raise RuntimeError(error_msg)
|
132 |
|
133 |
# Create model configuration
|
134 |
model_list = [{
|
135 |
+
"model_name": DEFAULT_BENCHMARK_MODEL,
|
136 |
"provider": provider,
|
137 |
"api_key": "$HF_TOKEN",
|
138 |
"max_concurrent_requests": 32,
|
|
|
156 |
},
|
157 |
"model_list": model_list,
|
158 |
|
159 |
+
"model_roles": BENCHMARK_MODEL_ROLES,
|
160 |
"pipeline": {
|
161 |
"ingestion": {
|
162 |
"source_documents_dir": f"uploaded_files/{self.session_uid}/uploaded_files/",
|