Update cfg.py

cfg.py

@@ -4,7 +4,10 @@ from buster.formatters.documents import DocumentsFormatterJSON
 from buster.formatters.prompts import PromptFormatter
 from buster.retriever import DeepLakeRetriever, Retriever
 from buster.tokenizers import GPTTokenizer
-from buster.validators import QuestionAnswerValidator, Validator
+from buster.validators import Validator
+from buster.llm_utils import get_openai_embedding_constructor
+
+embedding_fn = get_openai_embedding_constructor(client_kwargs=client_kwargs)
 
 buster_cfg = BusterConfig(
     retriever_cfg={
@@ -57,16 +60,18 @@ buster_cfg = BusterConfig(
             "Now answer the following question:\n"
         ),
     },
-    validator_cfg={
-        "unknown_response_templates": [
-            "I'm sorry, but I am an AI language model trained to assist with questions related to AI. I cannot answer that question as it is not relevant to the library or its usage. Is there anything else I can assist you with?",
-        ],
-        "unknown_threshold": 0.85,
-        "embedding_model": "text-embedding-ada-002",
-        "use_reranking": True,
-        "invalid_question_response": "This question does not seem relevant to my current knowledge. If you think this is a mistake, you can modify the question validation prompt.",
-        "check_question_prompt": """You are an chatbot answering questions about the Mila cluster, a compute cluster used by Mila students.
 
+       validator_cfg={
+        "question_validator_cfg": {
+            "invalid_question_response": "This question does not seem relevant to my current knowledge.",
+            "completion_kwargs": {
+                "model": "gpt-3.5-turbo",
+                "stream": False,
+                "temperature": 0,
+            },
+            "client_kwargs": client_kwargs,
+            "check_question_prompt": """You are an chatbot answering questions about the Mila cluster, a compute cluster used by Mila students.
+    
 Your job is to determine wether or not a question is valid, and should be answered. The question is valid if it is related to cluster usage and anything related to AI.
 A user will submit a question. Respond 'true' if it is valid, respond 'false' if it is invalid.
 
@@ -82,11 +87,24 @@ Q: What is the meaning of life?
 false
 
 A user will submit a question. Respond 'true' if it is valid, respond 'false' if it is invalid.""",
-        "completion_kwargs": {
-            "model": "gpt-3.5-turbo",
-            "stream": False,
-            "temperature": 0,
         },
+        "answer_validator_cfg": {
+            "unknown_response_templates": [
+                "I'm sorry, but I am an AI language model trained to assist with questions related to AI. I cannot answer that question as it is not relevant to the library or its usage. Is there anything else I can assist you with?",
+            ],
+            "unknown_threshold": 0.85,
+            "embedding_fn": embedding_fn,
+        },
+        "documents_validator_cfg": {
+            "completion_kwargs": {
+                "model": "gpt-3.5-turbo",
+                "stream": False,
+                "temperature": 0,
+            },
+            "client_kwargs": client_kwargs,
+        },
+        "use_reranking": True,
+        "validate_documents": False,
     },
 )
 
@@ -105,7 +123,7 @@ def setup_buster(buster_cfg: BusterConfig):
         ),
         **buster_cfg.documents_answerer_cfg,
     )
-    validator: Validator = QuestionAnswerValidator(**buster_cfg.validator_cfg)
+    validator: Validator = Validator(**buster_cfg.validator_cfg)
     buster: Buster = Buster(
         retriever=retriever, document_answerer=document_answerer, validator=validator
     )