Upload 7 files

generator/compute_metrics.py

@@ -1,84 +1,84 @@
-import json
-import logging
-
-def compute_metrics(attributes, total_sentences):
-    # Extract relevant information from attributes
-    all_relevant_sentence_keys = attributes.get("all_relevant_sentence_keys", [])
-    all_utilized_sentence_keys = attributes.get("all_utilized_sentence_keys", [])
-    sentence_support_information = attributes.get("sentence_support_information", [])
-
-    # Compute Context Relevance
-    context_relevance = len(all_relevant_sentence_keys) / total_sentences if total_sentences else 0
-    
-    # Compute Context Utilization
-    context_utilization = len(all_utilized_sentence_keys) / total_sentences if total_sentences else 0
-    
-    # Compute Completeness score
-    Ri = set(all_relevant_sentence_keys)
-    Ui = set(all_utilized_sentence_keys)
-
-    completeness_score = len(Ri & Ui) / len(Ri) if len(Ri) else 0
-
-    # Compute Adherence
-    adherence = all(info.get("fully_supported", False) for info in sentence_support_information)
-    #adherence = 1 if all(info.get("fully_supported", False) for info in sentence_support_information) else 0
-    
-    return {
-        "Context Relevance": context_relevance,
-        "Context Utilization": context_utilization,
-        "Completeness Score": completeness_score,
-        "Adherence": adherence
-    }
-
-def get_metrics(attributes, total_sentences):
-    if attributes.content:
-        try:
-            result_content = attributes.content  # Access the content attribute
-            # Extract the JSON part from the result_content
-            json_start = result_content.find("{")
-            json_end = result_content.rfind("}") + 1
-            json_str = result_content[json_start:json_end]
-            result_json = json.loads(json_str)
-            # Compute metrics using the extracted attributes
-            metrics = compute_metrics(result_json, total_sentences)
-            logging.info(metrics)
-            
-            return metrics        
-        except json.JSONDecodeError as e:
-            logging.error(f"JSONDecodeError: {e}")
-
-def get_attributes_text(attributes):
-        try:
-            result_content = attributes.content  # Access the content attribute
-            # Extract the JSON part from the result_content
-            json_start = result_content.find("{")
-            json_end = result_content.rfind("}") + 1
-            json_str = result_content[json_start:json_end]
-            result_json = json.loads(json_str)
-            
-            # Extract the required fields from json
-            relevance_explanation = result_json.get("relevance_explanation", "N/A")
-            all_relevant_sentence_keys = result_json.get("all_relevant_sentence_keys", [])
-            overall_supported_explanation = result_json.get("overall_supported_explanation", "N/A")
-            overall_supported = result_json.get("overall_supported", "N/A")
-            sentence_support_information = result_json.get("sentence_support_information", [])
-            all_utilized_sentence_keys = result_json.get("all_utilized_sentence_keys", [])
-
-            # Format the metrics for display
-            attributes_text = "Attributes:\n"
-            attributes_text = f"### Relevance Explanation:\n{relevance_explanation}\n\n"
-            attributes_text += f"### All Relevant Sentence Keys:\n{', '.join(all_relevant_sentence_keys)}\n\n"
-            attributes_text += f"### Overall Supported Explanation:\n{overall_supported_explanation}\n\n"
-            attributes_text += f"### Overall Supported:\n{overall_supported}\n\n"
-            attributes_text += "### Sentence Support Information:\n"
-            for info in sentence_support_information:
-                attributes_text += f"- Response Sentence Key: {info.get('response_sentence_key', 'N/A')}\n"
-                attributes_text += f"  Explanation: {info.get('explanation', 'N/A')}\n"
-                attributes_text += f"  Supporting Sentence Keys: {', '.join(info.get('supporting_sentence_keys', []))}\n"
-                attributes_text += f"  Fully Supported: {info.get('fully_supported', 'N/A')}\n"
-            attributes_text += f"\n### All Utilized Sentence Keys:\n{', '.join(all_utilized_sentence_keys)}"
-
-            return attributes_text
-        except Exception as e:
-            logging.error(f"Error extracting attributes: {e}")
+import json
+import logging
+
+def compute_metrics(attributes, total_sentences):
+    # Extract relevant information from attributes
+    all_relevant_sentence_keys = attributes.get("all_relevant_sentence_keys", [])
+    all_utilized_sentence_keys = attributes.get("all_utilized_sentence_keys", [])
+    sentence_support_information = attributes.get("sentence_support_information", [])
+
+    # Compute Context Relevance
+    context_relevance = len(all_relevant_sentence_keys) / total_sentences if total_sentences else 0
+    
+    # Compute Context Utilization
+    context_utilization = len(all_utilized_sentence_keys) / total_sentences if total_sentences else 0
+    
+    # Compute Completeness score
+    Ri = set(all_relevant_sentence_keys)
+    Ui = set(all_utilized_sentence_keys)
+
+    completeness_score = len(Ri & Ui) / len(Ri) if len(Ri) else 0
+
+    # Compute Adherence
+    adherence = all(info.get("fully_supported", False) for info in sentence_support_information)
+    #adherence = 1 if all(info.get("fully_supported", False) for info in sentence_support_information) else 0
+    
+    return {
+        "Context Relevance": context_relevance,
+        "Context Utilization": context_utilization,
+        "Completeness Score": completeness_score,
+        "Adherence": adherence
+    }
+
+def get_metrics(attributes, total_sentences):
+    if attributes.content:
+        try:
+            result_content = attributes.content  # Access the content attribute
+            # Extract the JSON part from the result_content
+            json_start = result_content.find("{")
+            json_end = result_content.rfind("}") + 1
+            json_str = result_content[json_start:json_end]
+            result_json = json.loads(json_str)
+            # Compute metrics using the extracted attributes
+            metrics = compute_metrics(result_json, total_sentences)
+            logging.info(metrics)
+            
+            return metrics        
+        except json.JSONDecodeError as e:
+            logging.error(f"JSONDecodeError: {e}")
+
+def get_attributes_text(attributes):
+        try:
+            result_content = attributes.content  # Access the content attribute
+            # Extract the JSON part from the result_content
+            json_start = result_content.find("{")
+            json_end = result_content.rfind("}") + 1
+            json_str = result_content[json_start:json_end]
+            result_json = json.loads(json_str)
+            
+            # Extract the required fields from json
+            relevance_explanation = result_json.get("relevance_explanation", "N/A")
+            all_relevant_sentence_keys = result_json.get("all_relevant_sentence_keys", [])
+            overall_supported_explanation = result_json.get("overall_supported_explanation", "N/A")
+            overall_supported = result_json.get("overall_supported", "N/A")
+            sentence_support_information = result_json.get("sentence_support_information", [])
+            all_utilized_sentence_keys = result_json.get("all_utilized_sentence_keys", [])
+
+            # Format the metrics for display
+            attributes_text = "Attributes:\n"
+            attributes_text = f"### Relevance Explanation:\n{relevance_explanation}\n\n"
+            attributes_text += f"### All Relevant Sentence Keys:\n{', '.join(all_relevant_sentence_keys)}\n\n"
+            attributes_text += f"### Overall Supported Explanation:\n{overall_supported_explanation}\n\n"
+            attributes_text += f"### Overall Supported:\n{overall_supported}\n\n"
+            attributes_text += "### Sentence Support Information:\n"
+            for info in sentence_support_information:
+                attributes_text += f"- Response Sentence Key: {info.get('response_sentence_key', 'N/A')}\n"
+                attributes_text += f"  Explanation: {info.get('explanation', 'N/A')}\n"
+                attributes_text += f"  Supporting Sentence Keys: {', '.join(info.get('supporting_sentence_keys', []))}\n"
+                attributes_text += f"  Fully Supported: {info.get('fully_supported', 'N/A')}\n"
+            attributes_text += f"\n### All Utilized Sentence Keys:\n{', '.join(all_utilized_sentence_keys)}"
+
+            return attributes_text
+        except Exception as e:
+            logging.error(f"Error extracting attributes: {e}")
             return f"An error occurred while extracting attributes: {e}"

generator/compute_rmse_auc_roc_metrics.py

@@ -1,74 +1,74 @@
-
-from sklearn.metrics import roc_auc_score, root_mean_squared_error
-from generator.generate_metrics import generate_metrics, retrieve_and_generate_response
-import logging
-
-def compute_rmse_auc_roc_metrics(gen_llm, val_llm, dataset, vector_store, num_question):
-
-     # Lists to accumulate ground truths and predictions for AUC-ROC computation
-    all_ground_truth_relevance = []
-    all_predicted_relevance = []
-
-    all_ground_truth_utilization = []
-    all_predicted_utilization = []
-
-    all_ground_truth_adherence = []
-    all_predicted_adherence = []
-
-    # For each question in dataset get the metrics
-    for i, document in enumerate(dataset):
-        # Extract ground truth metrics from dataset
-        ground_truth_relevance = dataset[i]['relevance_score']
-        ground_truth_utilization = dataset[i]['utilization_score']
-        ground_truth_adherence = 1 if dataset[i]['adherence_score'] else 0
-        
-        query = document['question']
-        logging.info(f'Query number: {i + 1}')
-        # Call the generate_metrics for each query
-        response, source_docs = retrieve_and_generate_response(gen_llm, vector_store, query)
-        attributes, metrics = generate_metrics(val_llm, response, source_docs, query, 25)
-        
-        # Extract predicted metrics (ensure these are continuous if possible)
-        predicted_relevance = metrics.get('Context Relevance', 0) if metrics else 0
-        predicted_utilization = metrics.get('Context Utilization', 0) if metrics else 0
-        predicted_adherence = 1 if metrics.get('Adherence', False) else 0
-        
-        # === Handle Continuous Inputs for RMSE ===
-        all_ground_truth_relevance.append(ground_truth_relevance)
-        all_predicted_relevance.append(predicted_relevance)
-        all_ground_truth_utilization.append(ground_truth_utilization)
-        all_predicted_utilization.append(predicted_utilization)
-        
-        all_ground_truth_adherence.append(ground_truth_adherence)
-        all_predicted_adherence.append(predicted_adherence)
-
-        if i == num_question:
-          break
-    
-    # === Compute RMSE & AUC-ROC for the Entire Dataset ===
-    try:
-        logging.info(f"All Ground Truth Relevance: {all_ground_truth_relevance}")
-        logging.info(f"All Predicted Relevance: {all_predicted_relevance}")
-        relevance_rmse = root_mean_squared_error(all_ground_truth_relevance, all_predicted_relevance)
-    except ValueError:
-        relevance_rmse = None
-
-    try:
-        logging.info(f"All Ground Truth Utilization: {all_ground_truth_utilization}")
-        logging.info(f"All Predicted Utilization: {all_predicted_utilization}")
-        utilization_rmse = root_mean_squared_error(all_ground_truth_utilization, all_predicted_utilization)
-    except ValueError:
-        utilization_rmse = None
-
-    try:
-        logging.info(f"All Ground Truth Adherence: {all_ground_truth_adherence}")
-        logging.info(f"All Predicted Adherence: {all_predicted_adherence}")
-        adherence_auc = roc_auc_score(all_ground_truth_adherence, all_predicted_adherence)
-    except ValueError:
-        adherence_auc = None   
-
-    logging.info(f"Relevance RMSE score: {relevance_rmse}")
-    logging.info(f"Utilization RMSE score: {utilization_rmse}")
-    logging.info(f"Overall Adherence AUC-ROC: {adherence_auc}")
-
-    return relevance_rmse, utilization_rmse, adherence_auc
+
+from sklearn.metrics import roc_auc_score, root_mean_squared_error
+from generator.generate_metrics import generate_metrics, retrieve_and_generate_response
+import logging
+
+def compute_rmse_auc_roc_metrics(gen_llm, val_llm, dataset, vector_store, num_question):
+
+     # Lists to accumulate ground truths and predictions for AUC-ROC computation
+    all_ground_truth_relevance = []
+    all_predicted_relevance = []
+
+    all_ground_truth_utilization = []
+    all_predicted_utilization = []
+
+    all_ground_truth_adherence = []
+    all_predicted_adherence = []
+
+    # For each question in dataset get the metrics
+    for i, document in enumerate(dataset):
+        # Extract ground truth metrics from dataset
+        ground_truth_relevance = dataset[i]['relevance_score']
+        ground_truth_utilization = dataset[i]['utilization_score']
+        ground_truth_adherence = 1 if dataset[i]['adherence_score'] else 0
+        
+        query = document['question']
+        logging.info(f"Query number: {i + 1}")
+        # Call the generate_metrics for each query
+        response, source_docs = retrieve_and_generate_response(gen_llm, vector_store, query)
+        attributes, metrics = generate_metrics(val_llm, response, source_docs, query, 25)
+        
+        # Extract predicted metrics (ensure these are continuous if possible)
+        predicted_relevance = metrics.get('Context Relevance', 0) if metrics else 0
+        predicted_utilization = metrics.get('Context Utilization', 0) if metrics else 0
+        predicted_adherence = 1 if metrics.get('Adherence', False) else 0
+        
+        # === Handle Continuous Inputs for RMSE ===
+        all_ground_truth_relevance.append(ground_truth_relevance)
+        all_predicted_relevance.append(predicted_relevance)
+        all_ground_truth_utilization.append(ground_truth_utilization)
+        all_predicted_utilization.append(predicted_utilization)
+        
+        all_ground_truth_adherence.append(ground_truth_adherence)
+        all_predicted_adherence.append(predicted_adherence)
+
+        if i == num_question:
+          break
+    
+    # === Compute RMSE & AUC-ROC for the Entire Dataset ===
+    try:
+        logging.info(f"All Ground Truth Relevance: {all_ground_truth_relevance}")
+        logging.info(f"All Predicted Relevance: {all_predicted_relevance}")
+        relevance_rmse = root_mean_squared_error(all_ground_truth_relevance, all_predicted_relevance)
+    except ValueError:
+        relevance_rmse = None
+
+    try:
+        logging.info(f"All Ground Truth Utilization: {all_ground_truth_utilization}")
+        logging.info(f"All Predicted Utilization: {all_predicted_utilization}")
+        utilization_rmse = root_mean_squared_error(all_ground_truth_utilization, all_predicted_utilization)
+    except ValueError:
+        utilization_rmse = None
+
+    try:
+        logging.info(f"All Ground Truth Adherence: {all_ground_truth_adherence}")
+        logging.info(f"All Predicted Adherence: {all_predicted_adherence}")
+        adherence_auc = roc_auc_score(all_ground_truth_adherence, all_predicted_adherence)
+    except ValueError:
+        adherence_auc = None   
+
+    logging.info(f"Relevance RMSE score: {relevance_rmse}")
+    logging.info(f"Utilization RMSE score: {utilization_rmse}")
+    logging.info(f"Overall Adherence AUC-ROC: {adherence_auc}")
+
+    return relevance_rmse, utilization_rmse, adherence_auc

generator/create_prompt.py

@@ -1,104 +1,104 @@
-from langchain.docstore.document import Document
-
-def create_prompt(documents, question, response):
-    prompt = f""" I asked someone to answer a question based on one or more documents. Your task is to review their response and assess whether or not each sentence in that response is supported by text in the documents. If supported, identify which sentences in the documents provide that support. Additionally, identify which documents contain useful information for answering the question, and which documents the answer was sourced from.
-
-    Here are the documents, each of which is split into sentences. Alongside each sentence is associated key, such as '0a.' or '0b.' that you can use to refer to it:
-    {documents}
-
-    The question was:
-    {question}
-
-    Here is their response, split into sentences with associated keys:
-    {response}
-
-    Provide a JSON response with an answer , relevance_explanation, all_relevant_sentence_keys, overall_supported_explanation, overall_supported, sentence_support_information(this can be a json inside a json and with these fields response_sentence_key, explanation, supporting_sentence_keys, fully_supported) and all_utilized_sentence_keys below are the definitions
-
-    You must respond with a JSON object matching this schema:
-
-    {{
-      "answer": "string",
-      "relevance_explanation": "string",
-      "all_relevant_sentence_keys": ["string"],
-      "overall_supported_explanation": "string",
-      "overall_supported": "boolean",
-      "sentence_support_information": [
-        {{
-          "response_sentence_key": "string",
-          "explanation": "string",
-          "supporting_sentence_keys": ["string"],
-          "fully_supported": "boolean"
-        }},
-      ],
-      "all_utilized_sentence_keys": ["string"]
-    }}
-
-        The relevance_explanation field is a string explaining which documents
-        contain useful information for answering the question. Provide a step-by-step
-        breakdown of information provided in the documents and how it is useful for
-        answering the question.
-
-        The all_relevant_sentence_keys field is a list of all document sentences keys
-        (e.g. '0a') that are relevant to the question. Include every sentence that is
-        useful and relevant to the question, even if it was not used in the response,
-        or if only parts of the sentence are useful. Ignore the provided response when
-        making this judgement and base your judgement solely on the provided documents
-        and question. Omit sentences that, if removed from the document, would not
-        impact someone's ability to answer the question.
-        
-        The overall_supported_explanation field is a string explaining why the response
-        *as a whole* is or is not supported by the documents. In this field, provide a
-        step-by-step breakdown of the claims made in the response and the support (or
-        lack thereof) for those claims in the documents. Begin by assessing each claim
-        separately, one by one; don’t make any remarks about the response as a whole
-        until you have assessed all the claims in isolation.
-        
-        The overall_supported field is a boolean indicating whether the response as a
-        whole is supported by the documents. This value should reflect the conclusion
-        you drew at the end of your step-by-step breakdown in overall_supported_explanation.
-        In the sentence_support_information field, provide information about the support
-        *for each sentence* in the response.
-        The sentence_support_information field is a list of objects, one for each sentence
-        in the response. Each object MUST have the following fields:
-        - response_sentence_key: a string identifying the sentence in the response.
-        This key is the same as the one used in the response above.
-        16
-        - explanation: a string explaining why the sentence is or is not supported by the
-        documents.
-        - supporting_sentence_keys: keys (e.g. '0a') of sentences from the documents that
-        support the response sentence. If the sentence is not supported, this list MUST
-        be empty. If the sentence is supported, this list MUST contain one or more keys.
-        In special cases where the sentence is supported, but not by any specific sentence,
-        you can use the string "supported_without_sentence" to indicate that the sentence
-        is generally supported by the documents. Consider cases where the sentence is
-        expressing inability to answer the question due to lack of relevant information in
-        the provided contex as "supported_without_sentence". In cases where the sentence
-        is making a general statement (e.g. outlining the steps to produce an answer, or
-        summarizing previously stated sentences, or a transition sentence), use the
-        sting "general".In cases where the sentence is correctly stating a well-known fact,
-        like a mathematical formula, use the string "well_known_fact". In cases where the
-        sentence is performing numerical reasoning (e.g. addition, multiplication), use
-        the string "numerical_reasoning".
-        - fully_supported: a boolean indicating whether the sentence is fully supported by
-        the documents.
-        - This value should reflect the conclusion you drew at the end of your step-by-step
-        breakdown in explanation.
-        - If supporting_sentence_keys is an empty list, then fully_supported must be false.
-        - Otherwise, use fully_supported to clarify whether everything in the response
-        sentence is fully supported by the document text indicated in supporting_sentence_keys
-        (fully_supported = true), or whether the sentence is only partially or incompletely
-        supported by that document text (fully_supported = false).
-        The all_utilized_sentence_keys field is a list of all sentences keys (e.g. ’0a’) that
-        were used to construct the answer. Include every sentence that either directly supported
-        the answer, or was implicitly used to construct the answer, even if it was not used
-        in its entirety. Omit sentences that were not used, and could have been removed from
-        the documents without affecting the answer.
-        You must respond with a valid JSON string. Use escapes for quotes, e.g. ‘\\"‘, and
-        newlines, e.g. ‘\\n‘. Do not write anything before or after the JSON string. Do not
-        wrap the JSON string in backticks like ‘‘‘ or ‘‘‘json.
-        As a reminder: your task is to review the response and assess which documents contain
-        useful information pertaining to the question, and how each sentence in the response
-        is supported by the text in the documents.
-
-    """
+from langchain.docstore.document import Document
+
+def create_prompt(documents, question, response):
+    prompt = f""" I asked someone to answer a question based on one or more documents. Your task is to review their response and assess whether or not each sentence in that response is supported by text in the documents. If supported, identify which sentences in the documents provide that support. Additionally, identify which documents contain useful information for answering the question, and which documents the answer was sourced from.
+
+    Here are the documents, each of which is split into sentences. Alongside each sentence is associated key, such as '0a.' or '0b.' that you can use to refer to it:
+    {documents}
+
+    The question was:
+    {question}
+
+    Here is their response, split into sentences with associated keys:
+    {response}
+
+    Provide a JSON response with an answer , relevance_explanation, all_relevant_sentence_keys, overall_supported_explanation, overall_supported, sentence_support_information(this can be a json inside a json and with these fields response_sentence_key, explanation, supporting_sentence_keys, fully_supported) and all_utilized_sentence_keys below are the definitions
+
+    You must respond with a JSON object matching this schema:
+
+    {{
+      "answer": "string",
+      "relevance_explanation": "string",
+      "all_relevant_sentence_keys": ["string"],
+      "overall_supported_explanation": "string",
+      "overall_supported": "boolean",
+      "sentence_support_information": [
+        {{
+          "response_sentence_key": "string",
+          "explanation": "string",
+          "supporting_sentence_keys": ["string"],
+          "fully_supported": "boolean"
+        }},
+      ],
+      "all_utilized_sentence_keys": ["string"]
+    }}
+
+        The relevance_explanation field is a string explaining which documents
+        contain useful information for answering the question. Provide a step-by-step
+        breakdown of information provided in the documents and how it is useful for
+        answering the question.
+
+        The all_relevant_sentence_keys field is a list of all document sentences keys
+        (e.g. '0a') that are relevant to the question. Include every sentence that is
+        useful and relevant to the question, even if it was not used in the response,
+        or if only parts of the sentence are useful. Ignore the provided response when
+        making this judgement and base your judgement solely on the provided documents
+        and question. Omit sentences that, if removed from the document, would not
+        impact someone's ability to answer the question.
+        
+        The overall_supported_explanation field is a string explaining why the response
+        *as a whole* is or is not supported by the documents. In this field, provide a
+        step-by-step breakdown of the claims made in the response and the support (or
+        lack thereof) for those claims in the documents. Begin by assessing each claim
+        separately, one by one; don’t make any remarks about the response as a whole
+        until you have assessed all the claims in isolation.
+        
+        The overall_supported field is a boolean indicating whether the response as a
+        whole is supported by the documents. This value should reflect the conclusion
+        you drew at the end of your step-by-step breakdown in overall_supported_explanation.
+        In the sentence_support_information field, provide information about the support
+        *for each sentence* in the response.
+        The sentence_support_information field is a list of objects, one for each sentence
+        in the response. Each object MUST have the following fields:
+        - response_sentence_key: a string identifying the sentence in the response.
+        This key is the same as the one used in the response above.
+        16
+        - explanation: a string explaining why the sentence is or is not supported by the
+        documents.
+        - supporting_sentence_keys: keys (e.g. '0a') of sentences from the documents that
+        support the response sentence. If the sentence is not supported, this list MUST
+        be empty. If the sentence is supported, this list MUST contain one or more keys.
+        In special cases where the sentence is supported, but not by any specific sentence,
+        you can use the string "supported_without_sentence" to indicate that the sentence
+        is generally supported by the documents. Consider cases where the sentence is
+        expressing inability to answer the question due to lack of relevant information in
+        the provided contex as "supported_without_sentence". In cases where the sentence
+        is making a general statement (e.g. outlining the steps to produce an answer, or
+        summarizing previously stated sentences, or a transition sentence), use the
+        sting "general".In cases where the sentence is correctly stating a well-known fact,
+        like a mathematical formula, use the string "well_known_fact". In cases where the
+        sentence is performing numerical reasoning (e.g. addition, multiplication), use
+        the string "numerical_reasoning".
+        - fully_supported: a boolean indicating whether the sentence is fully supported by
+        the documents.
+        - This value should reflect the conclusion you drew at the end of your step-by-step
+        breakdown in explanation.
+        - If supporting_sentence_keys is an empty list, then fully_supported must be false.
+        - Otherwise, use fully_supported to clarify whether everything in the response
+        sentence is fully supported by the document text indicated in supporting_sentence_keys
+        (fully_supported = true), or whether the sentence is only partially or incompletely
+        supported by that document text (fully_supported = false).
+        The all_utilized_sentence_keys field is a list of all sentences keys (e.g. ’0a’) that
+        were used to construct the answer. Include every sentence that either directly supported
+        the answer, or was implicitly used to construct the answer, even if it was not used
+        in its entirety. Omit sentences that were not used, and could have been removed from
+        the documents without affecting the answer.
+        You must respond with a valid JSON string. Use escapes for quotes, e.g. ‘\\"‘, and
+        newlines, e.g. ‘\\n‘. Do not write anything before or after the JSON string. Do not
+        wrap the JSON string in backticks like ‘‘‘ or ‘‘‘json.
+        As a reminder: your task is to review the response and assess which documents contain
+        useful information pertaining to the question, and how each sentence in the response
+        is supported by the text in the documents.
+
+    """
     return prompt

generator/extract_attributes.py

@@ -1,26 +1,26 @@
-from generator.create_prompt import create_prompt
-from generator.document_utils import apply_sentence_keys_documents, apply_sentence_keys_response
-
-# Function to extract attributes
-def extract_attributes(val_llm, question, relevant_docs, response):
-    # Format documents into a string by accessing the `page_content` attribute of each Document
-    #formatted_documents = "\n".join([f"Doc {i+1}: {doc.page_content}" for i, doc in enumerate(relevant_docs)])
-    formatted_documents = apply_sentence_keys_documents(relevant_docs)
-    formatted_responses = apply_sentence_keys_response(response)
-
-    #print(f"Formatted documents : {formatted_documents}")
-    # Print the number of sentences in each document
-    '''for i, doc in enumerate(formatted_documents):
-        num_sentences = len(doc)
-        print(f"Document {i} has {num_sentences} sentences.")'''
-
-    # Calculate the total number of sentences from formatted_documents
-    total_sentences = sum(len(doc) for doc in formatted_documents)
-    #print(f"Total number of sentences {total_sentences}")
-    
-    attribute_prompt = create_prompt(formatted_documents, question, formatted_responses)
-
-    # Instead of using BaseMessage, pass the formatted prompt directly to invoke
-    result = val_llm.invoke(attribute_prompt)
-
+from generator.create_prompt import create_prompt
+from generator.document_utils import apply_sentence_keys_documents, apply_sentence_keys_response
+
+# Function to extract attributes
+def extract_attributes(val_llm, question, relevant_docs, response):
+    # Format documents into a string by accessing the `page_content` attribute of each Document
+    #formatted_documents = "\n".join([f"Doc {i+1}: {doc.page_content}" for i, doc in enumerate(relevant_docs)])
+    formatted_documents = apply_sentence_keys_documents(relevant_docs)
+    formatted_responses = apply_sentence_keys_response(response)
+
+    #print(f"Formatted documents : {formatted_documents}")
+    # Print the number of sentences in each document
+    '''for i, doc in enumerate(formatted_documents):
+        num_sentences = len(doc)
+        print(f"Document {i} has {num_sentences} sentences.")'''
+
+    # Calculate the total number of sentences from formatted_documents
+    total_sentences = sum(len(doc) for doc in formatted_documents)
+    #print(f"Total number of sentences {total_sentences}")
+    
+    attribute_prompt = create_prompt(formatted_documents, question, formatted_responses)
+
+    # Instead of using BaseMessage, pass the formatted prompt directly to invoke
+    result = val_llm.invoke(attribute_prompt)
+
     return result, total_sentences

generator/generate_metrics.py

@@ -6,7 +6,7 @@ from generator.compute_metrics import get_metrics
 from generator.extract_attributes import extract_attributes
 
 def retrieve_and_generate_response(gen_llm, vector_store, query):
-    logging.info(f'Query: {query}')
+    logging.info(f"Query: {query}")
     
     # Step 1: Retrieve relevant documents for given query
     relevant_docs = retrieve_top_k_documents(vector_store, query, top_k=5)

generator/generate_response.py

@@ -1,18 +1,18 @@
-from langchain.chains import RetrievalQA
-
-def generate_response(llm, vector_store, question, relevant_docs):
-    # Create a retrieval-based question-answering chain using the relevant documents
-    qa_chain = RetrievalQA.from_chain_type(
-        llm=llm,
-        retriever=vector_store.as_retriever(),
-        return_source_documents=True
-    )
-    try:
-        result = qa_chain.invoke(question, documents=relevant_docs)
-        response = result['result']
-        source_docs = result['source_documents']
-        return response, source_docs
-    except Exception as e:
-        print(f"Error during QA chain invocation: {e}")
-        raise e
+from langchain.chains import RetrievalQA
+
+def generate_response(llm, vector_store, question, relevant_docs):
+    # Create a retrieval-based question-answering chain using the relevant documents
+    qa_chain = RetrievalQA.from_chain_type(
+        llm=llm,
+        retriever=vector_store.as_retriever(),
+        return_source_documents=True
+    )
+    try:
+        result = qa_chain.invoke(question, documents=relevant_docs)
+        response = result['result']
+        source_docs = result['source_documents']
+        return response, source_docs
+    except Exception as e:
+        print(f"Error during QA chain invocation: {e}")
+        raise e