Upload 4 files

app.py

@@ -0,0 +1,151 @@
+import gradio as gr
+import os
+from interface_utils import *
+
+maxim = 'quality'
+submaxims = ["The response is factual and supported by adequate evidence whenever possible."]
+checkbox_choices = [
+    ["Yes", "No", "NA"]
+]
+
+conversation_data = load_from_jsonl('./data/conversations_unlabeled.jsonl')
+max_conversation_length = max([len(conversation['transcript']) for conversation in conversation_data])
+conversation = get_conversation(conversation_data)
+
+
+def save_labels(conv_id, skipped, submaxim_0=None):
+    data = {
+        'conv_id': conv_id,
+        'maxim': maxim,
+        'skipped': skipped,
+        'submaxim_0': submaxim_0
+    }
+    os.makedirs("./labels", exist_ok=True)
+
+    with open(f"./labels/{maxim}_human_labels_{conv_id}.json", 'w') as f:
+        json.dump(data, f, indent=4)
+
+
+def update_interface(new_conversation):
+    new_conv_id = new_conversation['conv_id']
+    new_transcript = pad_transcript(new_conversation['transcript'], max_conversation_length)
+
+    markdown_blocks = [None] * max_conversation_length
+    for i in range(max_conversation_length):
+        if new_transcript[i]['speaker'] != '':
+            markdown_blocks[i] = gr.Markdown(f"""  **{new_transcript[i]['speaker']}**:      {new_transcript[i]['response']}""",
+                                             visible=True)
+        else:
+            markdown_blocks[i] = gr.Markdown("", visible=False)
+
+    new_last_response = gr.Text(value=get_last_response(new_transcript),
+                                label="",
+                                lines=1,
+                                container=False,
+                                interactive=False,
+                                autoscroll=True,
+                                visible=True)
+    new_radio_0_base = gr.Radio(label=submaxims[0],
+                                choices=checkbox_choices[0],
+                                value=None,
+                                visible=True)
+    conv_len = gr.Number(value=len(new_transcript), visible=False)
+
+    return [new_conv_id] + list(markdown_blocks) + [new_last_response] + [new_radio_0_base] + [conv_len]
+
+
+def submit(*args):
+    conv_id = args[0]
+    submaxim_0 = args[-2]
+
+    save_labels(conv_id, skipped=False, submaxim_0=submaxim_0)
+
+    new_conversation = get_conversation(conversation_data)
+    return update_interface(new_conversation)
+
+
+def skip(*args):
+    conv_id = args[0]
+    save_labels(conv_id, skipped=True)
+
+    new_conversation = get_conversation(conversation_data)
+    return update_interface(new_conversation)
+
+
+with gr.Blocks(theme=gr.themes.Default()) as interface:
+    conv_id = conversation['conv_id']
+    transcript = conversation['transcript']
+    conv_len = gr.Number(value=len(transcript), visible=False)
+    padded_transcript = pad_transcript(transcript, max_conversation_length)
+
+    markdown_blocks = [None] * max_conversation_length
+    with gr.Column(scale=1, min_width=600):
+        with gr.Group():
+            gr.Markdown("""<span style='font-size: 16px;'>&nbsp;&nbsp;&nbsp;&nbsp;**Conversational context** </span>""",
+                        visible=True)
+        for i in range(max_conversation_length):
+            markdown_blocks[i] = gr.Markdown(f"""&nbsp;&nbsp;**{padded_transcript[i]['speaker']}**: &nbsp;&nbsp;&nbsp;&nbsp;&nbsp;{padded_transcript[i]['response']}""")
+            if i >= conv_len.value:
+                markdown_blocks[i].visible = False
+
+        with gr.Row():
+            with gr.Group(elem_classes="bottom-aligned-group"):
+                speaker_adapted = gr.Markdown(
+                    f"""<span style='font-size: 16px;'>&nbsp;&nbsp;&nbsp;&nbsp;**Response to label** </span>""",
+                    visible=True)
+                last_response = gr.Textbox(value=get_last_response(transcript),
+                                           label="",
+                                           lines=1,
+                                           container=False,
+                                           interactive=False,
+                                           autoscroll=True,
+                                           visible=True)
+                radio_submaxim_0_base = gr.Radio(label=submaxims[0],
+                                                 choices=checkbox_choices[0],
+                                                 value=None,
+                                                 visible=True)
+
+    submit_button = gr.Button("Submit")
+    skip_button = gr.Button("Skip")
+
+    conv_id_element = gr.Text(value=conv_id, visible=False)
+    input_list = [conv_id_element] + \
+                 markdown_blocks + \
+                 [last_response] + \
+                 [radio_submaxim_0_base] + \
+                 [conv_len]
+    submit_button.click(
+        fn=submit,
+        inputs=input_list,
+        outputs=[conv_id_element,
+                 *markdown_blocks,
+                 last_response,
+                 radio_submaxim_0_base,
+                 conv_len]
+    )
+    skip_button.click(
+        fn=skip,
+        inputs=input_list,
+        outputs=[conv_id_element,
+                 *markdown_blocks,
+                 last_response,
+                 radio_submaxim_0_base,
+                 conv_len]
+    )
+
+css = """
+#textbox_id textarea {
+    background-color: white;
+}
+
+.bottom-aligned-group {
+    display: flex;
+    flex-direction: column;
+    justify-content: flex-end;
+    height: 100%;
+}
+"""
+interface.css = css
+interface.launch()
+
+

interface_utils.py

@@ -0,0 +1,50 @@
+import json
+import numpy as np
+import random
+import uuid
+
+
+def load_from_jsonl(filename, n=np.inf):
+    data = []
+    with open(filename, 'r') as file:
+        for i, line in enumerate(file):
+            if i >= n:  # stop after reading n lines
+                break
+            data.append(json.loads(line))
+    return data
+
+
+def append_id(conversations_no_id):
+    conversations = []
+    for conversation in conversations_no_id:
+        conversations.append({
+            'conv_id': uuid.uuid4().hex,
+            'transcript': conversation['transcript']
+        })
+    return conversations
+
+
+def save_to_jsonl(data, filename):
+    with open(filename, 'w') as file:
+        for item in data:
+            json_line = json.dumps(item)
+            file.write(json_line + '\n')
+
+
+def get_conversation(conversation_data):
+    conv = random.choice(conversation_data)
+    return conv
+
+
+def pad_transcript(transcript, max_length):
+    padding_count = max_length - len(transcript)
+    if padding_count > 0:
+        for _ in range(padding_count):
+            transcript.append({'speaker': '', 'response': ''})
+    return transcript
+
+
+def get_last_response(transcript):
+    for turn in reversed(transcript):
+        if turn['speaker'] and turn['response']:
+            return turn['response']

requirements.txt

@@ -0,0 +1,2 @@
+gradio
+numpy