TTsamurai
commited on
Commit
·
6306fdf
1
Parent(s):
3745407
separate the feedback based on the structure of fields
Browse files
app.py
CHANGED
@@ -54,8 +54,11 @@ SYSTEM_INSTRUECTION_PREFERENCE_ELICITATION_FILE = os.path.join(
|
|
54 |
SUMMARIZATION_PROMPT_FILE = os.path.join(STATIC_FILE, "txt/system_summarization_user_preference_elicitation.txt")
|
55 |
|
56 |
uuid_this_session = str(uuid.uuid4())
|
57 |
-
|
58 |
-
|
|
|
|
|
|
|
59 |
feedback_folder.mkdir(parents=True, exist_ok=True) # Ensure the directory exists
|
60 |
|
61 |
scheduler = CommitScheduler(
|
@@ -69,7 +72,7 @@ scheduler = CommitScheduler(
|
|
69 |
|
70 |
|
71 |
# Function to save user feedback
|
72 |
-
def save_feedback(user_id: str, uuid: str, type: str, value) -> None:
|
73 |
"""
|
74 |
Append input/outputs and user feedback to a JSON Lines file using a thread lock to avoid concurrent writes from different users.
|
75 |
"""
|
@@ -245,7 +248,7 @@ def add_user_profile_to_system_instruction(
|
|
245 |
summ, _ = generate_response_debugging(summarization_instruction)
|
246 |
user_preference_elicitation_data["summary_history"] = summ
|
247 |
# log_action(user_id, "Prompt", "Preference Elicitation Summarization", summ)
|
248 |
-
save_feedback(user_id, uuid_this_session, "Preference_Elicitation_Summarization", {"summarization": summ})
|
249 |
# print(f"Preference Summary:{summ}")
|
250 |
system_instruction += f"\nPrevious Conversations with the Customer about the User Profile: {user_preference_elicitation_data['summary_history']}\n"
|
251 |
else:
|
@@ -668,13 +671,14 @@ def create_demo():
|
|
668 |
|
669 |
# Log the user message and response
|
670 |
save_feedback(
|
671 |
-
user_id, uuid_this_session, "Interaction", {"type": tab_name, "role": "user", "content": message}
|
672 |
)
|
673 |
save_feedback(
|
674 |
user_id,
|
675 |
uuid_this_session,
|
676 |
"Interaction",
|
677 |
{"type": tab_name, "role": "assistant", "content": outputs_text},
|
|
|
678 |
)
|
679 |
# log_action(user_id, tab_name, "User Message", message)
|
680 |
# log_action(user_id, tab_name, "Response", outputs_text)
|
@@ -687,12 +691,15 @@ def create_demo():
|
|
687 |
uuid_this_session,
|
688 |
"Interaction",
|
689 |
{"type": "User_Elicitation", "role": "user", "content": message},
|
|
|
|
|
690 |
)
|
691 |
save_feedback(
|
692 |
user_id,
|
693 |
uuid_this_session,
|
694 |
"Interaction",
|
695 |
{"type": "User_Elicitation", "role": "assistant", "content": outputs_text},
|
|
|
696 |
)
|
697 |
# log_action(user_id, "User_Elicitation", "User Message", message)
|
698 |
# log_action(user_id, "User_Elicitation", "Response", outputs_text)
|
@@ -745,13 +752,14 @@ def create_demo():
|
|
745 |
print(f"Tab: {tab_name}\nHistory: {history}")
|
746 |
# Log the user message and response
|
747 |
save_feedback(
|
748 |
-
user_id, uuid_this_session, "Interaction", {"type": tab_name, "role": "user", "content": first_message}
|
749 |
)
|
750 |
save_feedback(
|
751 |
user_id,
|
752 |
uuid_this_session,
|
753 |
"Interaction",
|
754 |
{"type": tab_name, "role": "assistant", "content": outputs_text},
|
|
|
755 |
)
|
756 |
# log_action(user_id, tab_name, "User Message", first_message)
|
757 |
# log_action(user_id, tab_name, "Response", outputs_text)
|
@@ -764,12 +772,14 @@ def create_demo():
|
|
764 |
uuid_this_session,
|
765 |
"Interaction",
|
766 |
{"type": "User_Elicitation", "role": "user", "content": first_message},
|
|
|
767 |
)
|
768 |
save_feedback(
|
769 |
user_id,
|
770 |
uuid_this_session,
|
771 |
"Interaction",
|
772 |
{"type": "User_Elicitation", "role": "assistant", "content": outputs_text},
|
|
|
773 |
)
|
774 |
# log_action(user_id, "User_Elicitation", "User Message", first_message)
|
775 |
# log_action(user_id, "User_Elicitation", "Response", outputs_text)
|
@@ -823,13 +833,14 @@ def create_demo():
|
|
823 |
history = huggingface_to_gradio_message(history)
|
824 |
if tab_name is not None:
|
825 |
save_feedback(
|
826 |
-
user_id, uuid_this_session, "Interaction", {"type": tab_name, "role": "user", "content": message}
|
827 |
)
|
828 |
save_feedback(
|
829 |
user_id,
|
830 |
uuid_this_session,
|
831 |
"Interaction",
|
832 |
{"type": tab_name, "role": "assistant", "content": outputs_text},
|
|
|
833 |
)
|
834 |
# log_action(user_id, tab_name, "Show More of the Advisor’s Answer", "User continued the conversation")
|
835 |
# log_action(user_id, tab_name, "Response", outputs_text)
|
@@ -843,12 +854,14 @@ def create_demo():
|
|
843 |
uuid_this_session,
|
844 |
"Interaction",
|
845 |
{"type": "User_Elicitation", "role": "user", "content": message},
|
|
|
846 |
)
|
847 |
save_feedback(
|
848 |
user_id,
|
849 |
uuid_this_session,
|
850 |
"Interaction",
|
851 |
{"type": "User_Elicitation", "role": "assistant", "content": outputs_text},
|
|
|
852 |
)
|
853 |
# log_action(user_id, "User_Elicitation", "Response", outputs_text)
|
854 |
tab_data["history"] = history
|
@@ -860,7 +873,7 @@ def create_demo():
|
|
860 |
|
861 |
# dropdown, readon_button, multi-evaluator
|
862 |
print(f"Tab: {tab_name}\nEvaluation: {evals}")
|
863 |
-
save_feedback(user_id, uuid_this_session, "Round_Evaluation", evals)
|
864 |
# log_action(user_id, tab_name, "Round Evaluation", "Following")
|
865 |
# for key, value in evals.items():
|
866 |
# log_action(user_id, tab_name, key, value)
|
@@ -913,6 +926,7 @@ def create_demo():
|
|
913 |
fourth_comp: ranking_fourth_comp,
|
914 |
fifth_comp: ranking_fifth_comp,
|
915 |
},
|
|
|
916 |
)
|
917 |
# log_action(user_id, "Final_Ranking", first_comp, ranking_first_comp)
|
918 |
# log_action(user_id, "Final_Ranking", second_comp, ranking_second_comp)
|
|
|
54 |
SUMMARIZATION_PROMPT_FILE = os.path.join(STATIC_FILE, "txt/system_summarization_user_preference_elicitation.txt")
|
55 |
|
56 |
uuid_this_session = str(uuid.uuid4())
|
57 |
+
feedback_file_interaction = Path("user_feedback/") / f"data_{uuid_this_session}.json"
|
58 |
+
feedback_file_evaluation_and_summarization = (
|
59 |
+
Path("user_feedback/") / f"evaluation_and_summarization_{uuid_this_session}.json"
|
60 |
+
)
|
61 |
+
feedback_folder = feedback_file_interaction.parent
|
62 |
feedback_folder.mkdir(parents=True, exist_ok=True) # Ensure the directory exists
|
63 |
|
64 |
scheduler = CommitScheduler(
|
|
|
72 |
|
73 |
|
74 |
# Function to save user feedback
|
75 |
+
def save_feedback(user_id: str, uuid: str, type: str, value, feedback_file) -> None:
|
76 |
"""
|
77 |
Append input/outputs and user feedback to a JSON Lines file using a thread lock to avoid concurrent writes from different users.
|
78 |
"""
|
|
|
248 |
summ, _ = generate_response_debugging(summarization_instruction)
|
249 |
user_preference_elicitation_data["summary_history"] = summ
|
250 |
# log_action(user_id, "Prompt", "Preference Elicitation Summarization", summ)
|
251 |
+
save_feedback(user_id, uuid_this_session, "Preference_Elicitation_Summarization", {"summarization": summ}, feedback_file_evaluation_and_summarization)
|
252 |
# print(f"Preference Summary:{summ}")
|
253 |
system_instruction += f"\nPrevious Conversations with the Customer about the User Profile: {user_preference_elicitation_data['summary_history']}\n"
|
254 |
else:
|
|
|
671 |
|
672 |
# Log the user message and response
|
673 |
save_feedback(
|
674 |
+
user_id, uuid_this_session, "Interaction", {"type": tab_name, "role": "user", "content": message, feedback_file_interaction}
|
675 |
)
|
676 |
save_feedback(
|
677 |
user_id,
|
678 |
uuid_this_session,
|
679 |
"Interaction",
|
680 |
{"type": tab_name, "role": "assistant", "content": outputs_text},
|
681 |
+
feedback_file_interaction
|
682 |
)
|
683 |
# log_action(user_id, tab_name, "User Message", message)
|
684 |
# log_action(user_id, tab_name, "Response", outputs_text)
|
|
|
691 |
uuid_this_session,
|
692 |
"Interaction",
|
693 |
{"type": "User_Elicitation", "role": "user", "content": message},
|
694 |
+
feedback_file_interaction
|
695 |
+
|
696 |
)
|
697 |
save_feedback(
|
698 |
user_id,
|
699 |
uuid_this_session,
|
700 |
"Interaction",
|
701 |
{"type": "User_Elicitation", "role": "assistant", "content": outputs_text},
|
702 |
+
feedback_file_interaction
|
703 |
)
|
704 |
# log_action(user_id, "User_Elicitation", "User Message", message)
|
705 |
# log_action(user_id, "User_Elicitation", "Response", outputs_text)
|
|
|
752 |
print(f"Tab: {tab_name}\nHistory: {history}")
|
753 |
# Log the user message and response
|
754 |
save_feedback(
|
755 |
+
user_id, uuid_this_session, "Interaction", {"type": tab_name, "role": "user", "content": first_message, feedback_file_interaction}
|
756 |
)
|
757 |
save_feedback(
|
758 |
user_id,
|
759 |
uuid_this_session,
|
760 |
"Interaction",
|
761 |
{"type": tab_name, "role": "assistant", "content": outputs_text},
|
762 |
+
feedback_file_interaction
|
763 |
)
|
764 |
# log_action(user_id, tab_name, "User Message", first_message)
|
765 |
# log_action(user_id, tab_name, "Response", outputs_text)
|
|
|
772 |
uuid_this_session,
|
773 |
"Interaction",
|
774 |
{"type": "User_Elicitation", "role": "user", "content": first_message},
|
775 |
+
feedback_file_interaction
|
776 |
)
|
777 |
save_feedback(
|
778 |
user_id,
|
779 |
uuid_this_session,
|
780 |
"Interaction",
|
781 |
{"type": "User_Elicitation", "role": "assistant", "content": outputs_text},
|
782 |
+
feedback_file_interaction
|
783 |
)
|
784 |
# log_action(user_id, "User_Elicitation", "User Message", first_message)
|
785 |
# log_action(user_id, "User_Elicitation", "Response", outputs_text)
|
|
|
833 |
history = huggingface_to_gradio_message(history)
|
834 |
if tab_name is not None:
|
835 |
save_feedback(
|
836 |
+
user_id, uuid_this_session, "Interaction", {"type": tab_name, "role": "user", "content": message, feedback_file_interaction}
|
837 |
)
|
838 |
save_feedback(
|
839 |
user_id,
|
840 |
uuid_this_session,
|
841 |
"Interaction",
|
842 |
{"type": tab_name, "role": "assistant", "content": outputs_text},
|
843 |
+
feedback_file_interaction
|
844 |
)
|
845 |
# log_action(user_id, tab_name, "Show More of the Advisor’s Answer", "User continued the conversation")
|
846 |
# log_action(user_id, tab_name, "Response", outputs_text)
|
|
|
854 |
uuid_this_session,
|
855 |
"Interaction",
|
856 |
{"type": "User_Elicitation", "role": "user", "content": message},
|
857 |
+
feedback_file_interaction
|
858 |
)
|
859 |
save_feedback(
|
860 |
user_id,
|
861 |
uuid_this_session,
|
862 |
"Interaction",
|
863 |
{"type": "User_Elicitation", "role": "assistant", "content": outputs_text},
|
864 |
+
feedback_file_interaction
|
865 |
)
|
866 |
# log_action(user_id, "User_Elicitation", "Response", outputs_text)
|
867 |
tab_data["history"] = history
|
|
|
873 |
|
874 |
# dropdown, readon_button, multi-evaluator
|
875 |
print(f"Tab: {tab_name}\nEvaluation: {evals}")
|
876 |
+
save_feedback(user_id, uuid_this_session, "Round_Evaluation", evals, feedback_file_evaluation_and_summarization)
|
877 |
# log_action(user_id, tab_name, "Round Evaluation", "Following")
|
878 |
# for key, value in evals.items():
|
879 |
# log_action(user_id, tab_name, key, value)
|
|
|
926 |
fourth_comp: ranking_fourth_comp,
|
927 |
fifth_comp: ranking_fifth_comp,
|
928 |
},
|
929 |
+
feedback_file_evaluation_and_summarization
|
930 |
)
|
931 |
# log_action(user_id, "Final_Ranking", first_comp, ranking_first_comp)
|
932 |
# log_action(user_id, "Final_Ranking", second_comp, ranking_second_comp)
|