File size: 13,639 Bytes
7d82b61 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 |
import gradio as gr
import plotly.graph_objs as go
import numpy as np
import time
import google.generativeai as genai
from hardCodedData import *
from Helper import *
import google
'''
Model Information
Gemini 1.5 pro
'''
GOOGLE_API_KEY = "api"
genai.configure(api_key="AIzaSyC6msuJuuRiXTplyOzgnlZchpu5_olBXYs")
generation_config = genai.GenerationConfig(temperature=0.5)
# Model configuration
model = genai.GenerativeModel(
model_name='gemini-1.5-pro-latest',
system_instruction= """
You are an assistant chatbot for a Speech Language Pathologist (SLP).
Your task is to help analyze a provided video of a therapy session and answer questions accurately.
Provide timestamps for specific events or behaviors mentioned. Conclude each response with possible follow-up questions.
Follow these steps:
1. Suggest to the user to ask, “To get started, you can try asking me how many people there are in the video.”
2. Detect how many people are in the video.
2. Suggest to the user to tell you the names of the people in the video, starting from left to right.
3. After receiving the names, respond with, “Ok thank you! Now you can ask me any questions about this video.”
4. If the user asks about a behavior, respond with, “My understanding of this behavior is [xxx - AI generated output]. Is this a behavior that you want to track? If it is, please define this behavior and tell me more about it so I can analyze it more accurately according to your practice.”
5. If you receive names, confirm that these are the names of the people from left to right.
"""
)
'''
Video
'''
video_file = None
def handle_video(video=None):
global video_file
if video is None:
# Load example video
video = "./TEST.mp4"
isTest = True
video_file = genai.upload_file(path=video)
while video_file.state.name == "PROCESSING":
print(".", end="")
time.sleep(10)
video_file = genai.get_file(video_file.name)
if video_file.state.name == "FAILED":
raise ValueError(video_file.state.name)
if isTest:
return video
else:
return video_file
'''
Chatbot
'''
chat_history = []
def new_prompt(prompt):
global chat_history, video_file
# Append user prompt to chat history
chat_history.append({'role': 'user', 'parts': [prompt]})
try:
if video_file:
# Video exists and is processed
chat_history[-1]['parts'].extend([" from video: ", video_file])
response = model.generate_content(chat_history, request_options={"timeout": 600})
else:
# No video uploaded yet
response = model.generate_content(chat_history)
# Extract the text content from the response and append it to the chat history
assistant_message = response.candidates[0].content.parts[0].text
chat_history.append({'role': 'model', 'parts': [assistant_message]})
except google.api_core.exceptions.ResourceExhausted:
assistant_message = "API rate limit has been reached. Please wait a moment and try again."
chat_history.append({'role': 'model', 'parts': [assistant_message]})
except Exception as e:
assistant_message = f"An error occurred: {str(e)}"
chat_history.append({'role': 'model', 'parts': [assistant_message]})
return chat_history
def user_input(user_message, history):
return "", history + [[user_message, None]]
def bot_response(history):
user_message = history[-1][0]
print(history)
updated_history = new_prompt(user_message)
print(updated_history)
assistant_message = updated_history[-1]['parts'][0]
for i in range(len(assistant_message)):
time.sleep(0.05)
history[-1][1] = assistant_message[:i+1]
yield history
'''
Behaivor box
'''
initial_behaviors = [
("Initiating Behavioral Request (IBR)",
("The child's skill in using behavior(s) to elicit aid in obtaining an object, or object related event",
["00:10", "00:45", "01:30"])),
("Initiating Joint Attention (IJA)",
("The child's skill in using behavior(s) to initiate shared attention to objects or events.",
["00:15", "00:50", "01:40"])),
("Responding to Joint Attention (RJA)",
("The child's skill in following the examiner’s line of regard and pointing gestures.",
["00:20", "01:00", "02:00"])),
("Initiating Social Interaction (ISI)",
("The child's skill at initiating turn-taking sequences and the tendency to tease the tester",
["00:20", "00:50", "02:00"])),
("Responding to Social Interaction (RSI)",
("The child’s skill in responding to turn-taking interactions initiated by the examiner.",
["00:20", "01:00", "02:00"]))
]
behaviors = initial_behaviors
behavior_bank = []
def add_or_update_behavior(name, definition, timestamps, selected_behavior):
global behaviors, behavior_bank
if selected_behavior: # Update existing behavior
for i, (old_name, _) in enumerate(behaviors):
if old_name == selected_behavior:
behaviors[i] = (name, (definition, timestamps))
break
# Update behavior in the bank if it exists
behavior_bank = [name if b == selected_behavior else b for b in behavior_bank]
else: # Add new behavior
new_behavior = (name, (definition, timestamps))
behaviors.append(new_behavior)
choices = [b[0] for b in behaviors]
return gr.Dropdown(choices=choices, value=None, interactive=True), gr.CheckboxGroup(choices=behavior_bank, value=behavior_bank, interactive=True), "", "", ""
def add_to_behaivor_bank(selected_behavior, checkbox_group_values):
global behavior_bank
if selected_behavior and selected_behavior not in checkbox_group_values:
checkbox_group_values.append(selected_behavior)
behavior_bank = checkbox_group_values
return gr.CheckboxGroup(choices=checkbox_group_values, value=checkbox_group_values, interactive=True), gr.Dropdown(value=None,interactive=True)
def delete_behavior(selected_behavior, checkbox_group_values):
global behaviors, behavior_bank
behaviors = [b for b in behaviors if b[0] != selected_behavior]
behavior_bank = [b for b in behavior_bank if b != selected_behavior]
updated_choices = [b[0] for b in behaviors]
updated_checkbox_group = [cb for cb in checkbox_group_values if cb != selected_behavior]
return gr.Dropdown(choices=updated_choices, value=None, interactive=True), gr.CheckboxGroup(choices=updated_checkbox_group, value=updated_checkbox_group, interactive=True)
def edit_behavior(selected_behavior):
for name, (definition, timestamps) in behaviors:
if name == selected_behavior:
# Return values to populate textboxes
return name, definition, timestamps
return "", "", ""
welcome_message = """
Hello! I'm your AI assistant.
I can help you analyze your video sessions following your instructions.
To get started, please upload a video or add your behaviors to the Behavior Bank using the Behavior Manager.
"""
#If you want to tell me about the people in the video, please name them starting from left to right.
css="""
body {
background-color: #edf1fa; /* offwhite */
}
.gradio-container {
background-color: #edf1fa; /* offwhite */
}
.column-form .wrap {
flex-direction: column;
}
.sidebar {
background: #ffffff;
padding: 10px;
border-right: 1px solid #dee2e6;
}
.content {
padding: 10px;
}
"""
'''
Gradio Demo
'''
with gr.Blocks(theme='base', css=css, title="Soap.AI") as demo:
gr.Markdown("# 🤖 AI-Supported SOAP Generation")
with gr.Row():
with gr.Column():
video = gr.Video(label="Video", visible=True, height=360, container=True)
with gr.Row():
with gr.Column(min_width=1, scale=1):
video_upload_button = gr.Button("Analyze Video", variant="primary")
with gr.Column(min_width=1, scale=1):
example_video_button = gr.Button("Load Example Video")
video_upload_button.click(handle_video, inputs=video, outputs=video)
example_video_button.click(handle_video, None, outputs=video)
with gr.Column():
chat_section = gr.Group(visible=True)
with chat_section:
chatbot = gr.Chatbot(elem_id="chatbot",
container=True,
likeable=True,
value=[[None, welcome_message]],
avatar_images=(None, "./avatar.webp"))
with gr.Row():
txt = gr.Textbox(show_label=False, placeholder="Type here!")
with gr.Row():
send_btn = gr.Button("Send Message", elem_id="send-btn", variant="primary")
clear_btn = gr.Button("Clear Chat", elem_id="clear-btn")
with gr.Row():
behaivor_bank = gr.CheckboxGroup(label="Behavior Bank",
choices=[],
interactive=True,
info="A space to store all the behaviors you want to analyze.")
open_sidebar_btn = gr.Button("Show Behavior Manager", scale=0)
close_sidebar_btn = gr.Button("Hide Behavior Manager", visible=False, scale=0)
txt.submit(user_input, [txt, chatbot], [txt, chatbot], queue=False).then(
bot_response, chatbot, chatbot)
send_btn.click(user_input, [txt, chatbot], [txt, chatbot], queue=False).then(
bot_response, chatbot, chatbot)
clear_btn.click(lambda: None, None, chatbot, queue=False)
# Define a sidebar column that is initially hidden
with gr.Column(visible=False, min_width=200, scale=0.5, elem_classes="sidebar") as sidebar:
behavior_dropdown = gr.Dropdown(label="Behavior Collection",
choices=behaviors,
interactive=True,
container=True,
elem_classes="column-form",
info="Choose a behavior to add to the bank, edit or remove.")
with gr.Row():
add_toBank_button = gr.Button("Add Behavior to Bank", variant="primary")
edit_button = gr.Button("Edit Behavior")
delete_button = gr.Button("Remove Behavior")
with gr.Row():
name_input = gr.Textbox(label="Behavior Name",
placeholder="(e.g., IBR)",
info="The name you give to the specific behavior you're tracking or analyzing.")
timestamps_input = gr.Textbox(label="Timestamps MM:SS",
placeholder="(e.g., (01:15,01:35) )",
info="The exact times during a session when you saw the behavior. The first two digits represent minutes and the last two digits represent seconds.")
definition_input = gr.Textbox(lines=3,
label="Behavior Definition",
placeholder="(e.g., the child's skill in using behavior(s) to elicit aid in obtaining an object, or object related event)",
info="Provide a clear definition of the behavior.")
with gr.Row():
submit_button = gr.Button("Save Behavior", variant="primary")
submit_button.click(fn=add_or_update_behavior,
inputs=[name_input, definition_input, timestamps_input, behavior_dropdown],
outputs=[behavior_dropdown, behaivor_bank, name_input, definition_input, timestamps_input])
add_toBank_button.click(fn=add_to_behaivor_bank,
inputs=[behavior_dropdown, behaivor_bank],
outputs=[behaivor_bank, behavior_dropdown])
delete_button.click(fn=delete_behavior,
inputs=[behavior_dropdown, behaivor_bank],
outputs=[behavior_dropdown, behaivor_bank])
edit_button.click(fn=edit_behavior,
inputs=[behavior_dropdown],
outputs=[name_input, definition_input, timestamps_input])
# Function to open the sidebar
open_sidebar_btn.click(lambda: {
open_sidebar_btn: gr.Button(visible=False),
close_sidebar_btn: gr.Button(visible=True),
sidebar: gr.Column(visible=True)
}, outputs=[open_sidebar_btn, close_sidebar_btn, sidebar])
# Function to close the sidebar
close_sidebar_btn.click(lambda: {
open_sidebar_btn: gr.Button(visible=True),
close_sidebar_btn: gr.Button(visible=False),
sidebar: gr.Column(visible=False)
}, outputs=[open_sidebar_btn, close_sidebar_btn, sidebar])
# Launch the demo
demo.launch(share=True) |