Organise prompts

prompts.py

@@ -0,0 +1,94 @@
+# Default values for compatible mode
+DEFAULT_EVAL_CRITERIA = """Does the model provide relevant and useful responses to the user's needs or questions?"""
+
+DEFAULT_SCORE_1 = "The model's responses are irrelevant or unhelpful to the user's needs or queries."
+DEFAULT_SCORE_2 = "The model sometimes provides helpful information, but often fails to address the user's actual needs or questions."
+DEFAULT_SCORE_3 = "The model generally provides helpful responses that address the user's needs, though it may occasionally miss the mark."
+DEFAULT_SCORE_4 = "The model regularly provides helpful responses that are well-aligned with the user's inquiries, with only rare inaccuracies."
+DEFAULT_SCORE_5 = "The model consistently offers highly relevant and useful responses that perfectly cater to the user's needs and inquiries."
+
+# Default Eval Prompt
+DEFAULT_EVAL_PROMPT = """Does the model provide relevant and useful responses to the user's needs or questions?
+
+Scoring Rubric:
+Score 1: The model's responses are irrelevant or unhelpful to the user's needs or queries.
+Score 2: The model sometimes provides helpful information, but often fails to address the user's actual needs or questions.
+Score 3: The model generally provides helpful responses that address the user's needs, though it may occasionally miss the mark.
+Score 4: The model regularly provides helpful responses that are well-aligned with the user's inquiries, with only rare inaccuracies.
+Score 5: The model consistently offers highly relevant and useful responses that perfectly cater to the user's needs and inquiries.
+
+[User Query]: {{input}}
+
+[AI Response]: {{response}}"""
+
+# Split the eval prompt into editable and fixed parts
+DEFAULT_EVAL_PROMPT_EDITABLE = """Does the model provide relevant and useful responses to the user's needs or questions?
+
+Scoring Rubric:
+Score 1: The model's responses are irrelevant or unhelpful to the user's needs or queries.
+Score 2: The model sometimes provides helpful information, but often fails to address the user's actual needs or questions.
+Score 3: The model generally provides helpful responses that address the user's needs, though it may occasionally miss the mark.
+Score 4: The model regularly provides helpful responses that are well-aligned with the user's inquiries, with only rare inaccuracies.
+Score 5: The model consistently offers highly relevant and useful responses that perfectly cater to the user's needs and inquiries."""
+
+# Fixed suffix that will always be appended
+FIXED_EVAL_SUFFIX = """
+[User Query]: {{input}}
+
+[AI Response]: {{response}}"""
+
+# Define the Prometheus prompt used by default (without reference)
+PROMETHEUS_PROMPT = """###Task Description:
+An instruction (might include an Input inside it) and a response to evaluate are given.
+1. Write a detailed feedback that assesses the quality of the response strictly based on the given score rubric, not evaluating in general.
+2. After writing the feedback, write a score that is an integer between 1 and 5.
+3. The output format should look as follows: "Feedback: (write a feedback for criteria) [RESULT] (an integer number between 1 and 5)"
+4. Please do not generate any other openings, closings, or explanations.
+
+###The instruction to evaluate:
+{human_input}
+
+###Response to evaluate:
+{ai_response}
+
+###Score Rubrics:
+[{eval_criteria}]
+Score 1: {score1_desc}
+Score 2: {score2_desc}
+Score 3: {score3_desc}
+Score 4: {score4_desc}
+Score 5: {score5_desc}
+
+###Feedback:
+"""
+
+# Define the Prometheus prompt with reference response
+PROMETHEUS_PROMPT_WITH_REFERENCE = """###Task Description:
+An instruction (might include an Input inside it), a response to evaluate, a reference answer that gets a score of 5, and a score rubric representing an evaluation criteria are given.
+1. Write a detailed feedback that assesses the quality of the response strictly based on the given score rubric, not evaluating in general.
+2. After writing the feedback, write a score that is an integer between 1 and 5.
+3. The output format should look as follows: "Feedback: (write a feedback for criteria) [RESULT] (an integer number between 1 and 5)"
+4. Please do not generate any other openings, closings, or explanations.
+
+###The instruction to evaluate:
+{human_input}
+
+###Response to evaluate:
+{ai_response}
+
+###Reference Answer (Score 5):
+{ground_truth_input}
+
+###Score Rubrics:
+[{eval_criteria}]
+Score 1: {score1_desc}
+Score 2: {score2_desc}
+Score 3: {score3_desc}
+Score 4: {score4_desc}
+Score 5: {score5_desc}
+
+###Feedback:
+"""
+
+# Judge system prompt for non-Prometheus models
+JUDGE_SYSTEM_PROMPT = """Please act as an impartial judge and evaluate based on the user's instruction. Your output format should strictly adhere to JSON as follows: {"feedback": "<write feedback>", "result": <numerical score>}. Ensure the output is valid JSON, without additional formatting or explanations."""