fix minor error
Browse files
src/know_lang_bot/evaluation/chatbot_evaluation.py
CHANGED
@@ -5,7 +5,6 @@ from pydantic_ai import Agent
|
|
5 |
from know_lang_bot.config import AppConfig
|
6 |
from know_lang_bot.utils.model_provider import create_pydantic_model
|
7 |
from know_lang_bot.chat_bot.chat_graph import ChatResult, process_chat
|
8 |
-
import json
|
9 |
import asyncio
|
10 |
|
11 |
class EvalMetric(str, Enum):
|
@@ -104,7 +103,7 @@ Format your response as JSON:
|
|
104 |
}
|
105 |
|
106 |
total_score = sum(
|
107 |
-
metrics[metric] * weights[metric] * case.difficulty
|
108 |
for metric in EvalMetric
|
109 |
)
|
110 |
|
@@ -112,7 +111,7 @@ Format your response as JSON:
|
|
112 |
case=case,
|
113 |
metrics=metrics,
|
114 |
total_score=total_score,
|
115 |
-
feedback=metrics
|
116 |
)
|
117 |
|
118 |
async def evaluate_batch(
|
@@ -354,9 +353,13 @@ async def main():
|
|
354 |
collection = chromadb.PersistentClient(path=str(config.db.persist_directory)).get_collection(name=config.db.collection_name)
|
355 |
|
356 |
for case in TRANSFORMER_TEST_CASES:
|
357 |
-
|
358 |
-
|
359 |
-
|
|
|
|
|
|
|
|
|
360 |
|
361 |
break
|
362 |
|
|
|
5 |
from know_lang_bot.config import AppConfig
|
6 |
from know_lang_bot.utils.model_provider import create_pydantic_model
|
7 |
from know_lang_bot.chat_bot.chat_graph import ChatResult, process_chat
|
|
|
8 |
import asyncio
|
9 |
|
10 |
class EvalMetric(str, Enum):
|
|
|
103 |
}
|
104 |
|
105 |
total_score = sum(
|
106 |
+
metrics.model_dump()[metric] * weights[metric] * case.difficulty
|
107 |
for metric in EvalMetric
|
108 |
)
|
109 |
|
|
|
111 |
case=case,
|
112 |
metrics=metrics,
|
113 |
total_score=total_score,
|
114 |
+
feedback=metrics.feedback,
|
115 |
)
|
116 |
|
117 |
async def evaluate_batch(
|
|
|
353 |
collection = chromadb.PersistentClient(path=str(config.db.persist_directory)).get_collection(name=config.db.collection_name)
|
354 |
|
355 |
for case in TRANSFORMER_TEST_CASES:
|
356 |
+
try:
|
357 |
+
chat_result = await process_chat(question=case.question, collection=collection, config=config)
|
358 |
+
result = await evaluator.evaluate_single(case, chat_result)
|
359 |
+
console.print(Pretty(result.model_dump()))
|
360 |
+
|
361 |
+
except Exception as e:
|
362 |
+
console.print_exception()
|
363 |
|
364 |
break
|
365 |
|