FEAT: Diagnostic mode

app.py

@@ -56,7 +56,7 @@ model = get_model(model_path)
 inference_tokenizer = get_tokenizer(model_path)
 
 
-def get_evaluation_data(_context: List) -> List[Tuple[List, str, str]]:
+def get_evaluation_data_from_json(_context: List) -> List[Tuple[List, str, str]]:
     output_data = []
     for _dict in _context:
         _dict: Dict
@@ -67,11 +67,24 @@ def get_evaluation_data(_context: List) -> List[Tuple[List, str, str]]:
     return output_data
 
 
+def get_evaluation_data_from_dialogue(_context: List) -> List[Tuple[List, str, Union[str | None]]]:
+    output_data = []
+    for idx, _line in enumerate(_context):
+        if idx == 0:
+            continue
+        actual_context = _context[max(0, idx - 5):idx]
+        actual_sentence = _line
+        for context_idx in range(len(actual_context)):
+            output_data.append((actual_context[-context_idx:], actual_sentence, None))
+    return output_data
+
+
 option = st.selectbox("Choose type of input:",
                       ["01 - String (one turn per line)",
                        "02 - JSON (aggregated)",
                        "03 - JSON (example CA-OOD)",
-                       "04 - JSON (example Elysai)"])
+                       "04 - JSON (example Elysai)",
+                       "05 - Diagnostic mode"])
 
 
 with st.form("input_text"):
@@ -115,13 +128,13 @@ with st.form("input_text"):
         context = st.text_area("Insert JSON here:", value=str(text))
 
         if "{" in context:
-            evaluation_data = get_evaluation_data(_context=json.loads(context))
+            data_for_evaluation = get_evaluation_data_from_json(_context=json.loads(context))
         results = []
         accuracy = []
 
         submitted = st.form_submit_button("Submit")
         if submitted:
-            for datapoint in evaluation_data:
+            for datapoint in data_for_evaluation:
                 c, s, human_label = datapoint
                 input_tensor = inference_tokenizer.get_item(context=c, actual_sentence=s)
                 output_model = model(**input_tensor.data).logits
@@ -138,6 +151,22 @@ with st.form("input_text"):
             df = pandas.DataFrame(results, columns=["Context", "Query", "Human Label", "Probability (follow)",
                                                     "Probability (not-follow)"])
             st.dataframe(df)
+    elif "05" in option:
+        context = st.text_area("Insert dialogue here (one turn per line):")
+        submitted = st.form_submit_button("Submit")
+        if submitted:
+            aggregated_result = []
+            data_for_evaluation = get_evaluation_data_from_dialogue(context.split("\n"))
+            for datapoint in data_for_evaluation:
+                c, s, _ = datapoint
+                input_tensor = inference_tokenizer.get_item(context=c, actual_sentence=s)
+                output_model = model(**input_tensor.data).logits
+                output_model = torch.softmax(output_model, dim=-1).detach().numpy()[0]
+                prop_follow = output_model[0]
+                prop_not_follow = output_model[1]
+
+                aggregated_result.append((c, s, prop_follow))
+            st.table(aggregated_result)
 
 st.markdown("## Description of models:")
 for x in sorted(models.values(), key=lambda x: x["model"]):