Summary

app/data_init.py

@@ -21,6 +21,7 @@ vad_model, vad_utils = torch.hub.load(
     repo_or_dir=config_data.StaticPaths_VAD_MODEL,
     model="silero_vad",
     force_reload=False,
+    verbose=False,
     onnx=False,
 )
 

app/event_handlers/clear.py

@@ -11,11 +11,19 @@ import gradio as gr
 from app.config import config_data
 
 
-def event_handler_clear() -> (
-    tuple[
-        gr.Video, gr.Button, gr.Button, gr.Textbox, gr.Plot, gr.Plot, gr.Plot, gr.Plot
-    ]
-):
+def event_handler_clear() -> tuple[
+    gr.Video,
+    gr.Button,
+    gr.Button,
+    gr.Textbox,
+    gr.Plot,
+    gr.Plot,
+    gr.Plot,
+    gr.Plot,
+    gr.Row,
+    gr.Textbox,
+    gr.Textbox,
+]:
     return (
         gr.Video(value=None),
         gr.Button(interactive=False),
@@ -30,4 +38,7 @@ def event_handler_clear() -> (
         gr.Plot(value=None, visible=False),
         gr.Plot(value=None, visible=False),
         gr.Plot(value=None, visible=False),
+        gr.Row(visible=False),
+        gr.Textbox(value=None, visible=False),
+        gr.Textbox(value=None, visible=False),
     )

app/event_handlers/event_handlers.py

@@ -22,6 +22,9 @@ def setup_app_event_handlers(
     faces,
     emotion_stats,
     sent_stats,
+    time_row,
+    video_duration,
+    calculate_time,
 ):
     gr.on(
         triggers=[video.change, video.upload, video.stop_recording, video.clear],
@@ -40,6 +43,9 @@ def setup_app_event_handlers(
             faces,
             emotion_stats,
             sent_stats,
+            time_row,
+            video_duration,
+            calculate_time,
         ],
         queue=True,
     )
@@ -56,6 +62,9 @@ def setup_app_event_handlers(
             faces,
             emotion_stats,
             sent_stats,
+            time_row,
+            video_duration,
+            calculate_time,
         ],
         queue=True,
     )

app/event_handlers/submit.py

@@ -14,6 +14,7 @@ import gradio as gr
 # Importing necessary components for the Gradio app
 from app.config import config_data
 from app.utils import (
+    Timer,
     convert_video_to_audio,
     readetect_speech,
     slice_audio,
@@ -44,126 +45,144 @@ from app.load_models import VideoFeatureExtractor
 @spaces.GPU
 def event_handler_submit(
     video: str,
-) -> tuple[gr.Textbox, gr.Plot, gr.Plot, gr.Plot, gr.Plot]:
-    if video:
-        if video.split(".")[-1] == "webm":
-            video = convert_webm_to_mp4(video)
-
-    audio_file_path = convert_video_to_audio(file_path=video, sr=config_data.General_SR)
-    wav, vad_info = readetect_speech(
-        file_path=audio_file_path,
-        read_audio=read_audio,
-        get_speech_timestamps=get_speech_timestamps,
-        vad_model=vad_model,
-        sr=config_data.General_SR,
-    )
+) -> tuple[
+    gr.Textbox,
+    gr.Plot,
+    gr.Plot,
+    gr.Plot,
+    gr.Plot,
+    gr.Row,
+    gr.Textbox,
+    gr.Textbox,
+]:
+    with Timer() as t:
+        if video:
+            if video.split(".")[-1] == "webm":
+                video = convert_webm_to_mp4(video)
+
+        audio_file_path = convert_video_to_audio(
+            file_path=video, sr=config_data.General_SR
+        )
+        wav, vad_info = readetect_speech(
+            file_path=audio_file_path,
+            read_audio=read_audio,
+            get_speech_timestamps=get_speech_timestamps,
+            vad_model=vad_model,
+            sr=config_data.General_SR,
+        )
 
-    audio_windows = slice_audio(
-        start_time=config_data.General_START_TIME,
-        end_time=int(len(wav)),
-        win_max_length=int(config_data.General_WIN_MAX_LENGTH * config_data.General_SR),
-        win_shift=int(config_data.General_WIN_SHIFT * config_data.General_SR),
-        win_min_length=int(config_data.General_WIN_MIN_LENGTH * config_data.General_SR),
-    )
+        audio_windows = slice_audio(
+            start_time=config_data.General_START_TIME,
+            end_time=int(len(wav)),
+            win_max_length=int(
+                config_data.General_WIN_MAX_LENGTH * config_data.General_SR
+            ),
+            win_shift=int(config_data.General_WIN_SHIFT * config_data.General_SR),
+            win_min_length=int(
+                config_data.General_WIN_MIN_LENGTH * config_data.General_SR
+            ),
+        )
 
-    intersections = find_intersections(
-        x=audio_windows,
-        y=vad_info,
-        min_length=config_data.General_WIN_MIN_LENGTH * config_data.General_SR,
-    )
+        intersections = find_intersections(
+            x=audio_windows,
+            y=vad_info,
+            min_length=config_data.General_WIN_MIN_LENGTH * config_data.General_SR,
+        )
 
-    vfe = VideoFeatureExtractor(video_model, file_path=video, with_features=False)
-    vfe.preprocess_video()
-
-    transcriptions, total_text = asr(wav, audio_windows)
-
-    window_frames = []
-    preds_emo = []
-    preds_sen = []
-    for w_idx, window in enumerate(audio_windows):
-        a_w = intersections[w_idx]
-        if not a_w["speech"]:
-            a_pred = None
-        else:
-            wave = wav[a_w["start"] : a_w["end"]].clone()
-            a_pred, _ = audio_model(wave)
-
-        v_pred, _ = vfe(window, config_data.General_WIN_MAX_LENGTH)
-
-        t_pred, _ = text_model(transcriptions[w_idx][0])
-
-        if a_pred:
-            pred_emo = (a_pred["emo"] + v_pred["emo"] + t_pred["emo"]) / 3
-            pred_sen = (a_pred["sen"] + v_pred["sen"] + t_pred["sen"]) / 3
-        else:
-            pred_emo = (v_pred["emo"] + t_pred["emo"]) / 2
-            pred_sen = (v_pred["sen"] + t_pred["sen"]) / 2
-
-        frames = list(
-            range(
-                int(window["start"] * vfe.fps / config_data.General_SR) + 1,
-                int(window["end"] * vfe.fps / config_data.General_SR) + 2,
+        vfe = VideoFeatureExtractor(video_model, file_path=video, with_features=False)
+        vfe.preprocess_video()
+
+        transcriptions, total_text = asr(wav, audio_windows)
+
+        window_frames = []
+        preds_emo = []
+        preds_sen = []
+        for w_idx, window in enumerate(audio_windows):
+            a_w = intersections[w_idx]
+            if not a_w["speech"]:
+                a_pred = None
+            else:
+                wave = wav[a_w["start"] : a_w["end"]].clone()
+                a_pred, _ = audio_model(wave)
+
+            v_pred, _ = vfe(window, config_data.General_WIN_MAX_LENGTH)
+
+            t_pred, _ = text_model(transcriptions[w_idx][0])
+
+            if a_pred:
+                pred_emo = (a_pred["emo"] + v_pred["emo"] + t_pred["emo"]) / 3
+                pred_sen = (a_pred["sen"] + v_pred["sen"] + t_pred["sen"]) / 3
+            else:
+                pred_emo = (v_pred["emo"] + t_pred["emo"]) / 2
+                pred_sen = (v_pred["sen"] + t_pred["sen"]) / 2
+
+            frames = list(
+                range(
+                    int(window["start"] * vfe.fps / config_data.General_SR) + 1,
+                    int(window["end"] * vfe.fps / config_data.General_SR) + 2,
+                )
             )
+            preds_emo.extend([torch.argmax(pred_emo).numpy()] * len(frames))
+            preds_sen.extend([torch.argmax(pred_sen).numpy()] * len(frames))
+            window_frames.extend(frames)
+
+        if max(window_frames) < vfe.frame_number:
+            missed_frames = list(range(max(window_frames) + 1, vfe.frame_number + 1))
+            window_frames.extend(missed_frames)
+            preds_emo.extend([preds_emo[-1]] * len(missed_frames))
+            preds_sen.extend([preds_sen[-1]] * len(missed_frames))
+
+        df_pred = pd.DataFrame(columns=["frames", "pred_emo", "pred_sent"])
+        df_pred["frames"] = window_frames
+        df_pred["pred_emo"] = preds_emo
+        df_pred["pred_sent"] = preds_sen
+
+        df_pred = df_pred.groupby("frames").agg(
+            {
+                "pred_emo": calculate_mode,
+                "pred_sent": calculate_mode,
+            }
         )
-        preds_emo.extend([torch.argmax(pred_emo).numpy()] * len(frames))
-        preds_sen.extend([torch.argmax(pred_sen).numpy()] * len(frames))
-        window_frames.extend(frames)
-
-    if max(window_frames) < vfe.frame_number:
-        missed_frames = list(range(max(window_frames) + 1, vfe.frame_number + 1))
-        window_frames.extend(missed_frames)
-        preds_emo.extend([preds_emo[-1]] * len(missed_frames))
-        preds_sen.extend([preds_sen[-1]] * len(missed_frames))
-
-    df_pred = pd.DataFrame(columns=["frames", "pred_emo", "pred_sent"])
-    df_pred["frames"] = window_frames
-    df_pred["pred_emo"] = preds_emo
-    df_pred["pred_sent"] = preds_sen
-
-    df_pred = df_pred.groupby("frames").agg(
-        {
-            "pred_emo": calculate_mode,
-            "pred_sent": calculate_mode,
-        }
-    )
 
-    frame_indices = get_evenly_spaced_frame_indices(vfe.frame_number, 9)
-    num_frames = len(wav)
-    time_axis = [i / config_data.General_SR for i in range(num_frames)]
-    plt_audio = plot_audio(time_axis, wav.unsqueeze(0), frame_indices, vfe.fps, (12, 2))
-
-    all_idx_faces = list(vfe.faces[1].keys())
-    need_idx_faces = find_nearest_frames(frame_indices, all_idx_faces)
-    faces = []
-    for idx_frame, idx_faces in zip(frame_indices, need_idx_faces):
-        cur_face = cv2.resize(
-            vfe.faces[1][idx_faces], (224, 224), interpolation=cv2.INTER_AREA
+        frame_indices = get_evenly_spaced_frame_indices(vfe.frame_number, 9)
+        num_frames = len(wav)
+        time_axis = [i / config_data.General_SR for i in range(num_frames)]
+        plt_audio = plot_audio(
+            time_axis, wav.unsqueeze(0), frame_indices, vfe.fps, (12, 2)
         )
-        faces.append(
-            display_frame_info(
-                cur_face, "Frame: {}".format(idx_frame + 1), box_scale=0.3
+
+        all_idx_faces = list(vfe.faces[1].keys())
+        need_idx_faces = find_nearest_frames(frame_indices, all_idx_faces)
+        faces = []
+        for idx_frame, idx_faces in zip(frame_indices, need_idx_faces):
+            cur_face = cv2.resize(
+                vfe.faces[1][idx_faces], (224, 224), interpolation=cv2.INTER_AREA
+            )
+            faces.append(
+                display_frame_info(
+                    cur_face, "Frame: {}".format(idx_frame + 1), box_scale=0.3
+                )
             )
+        plt_faces = plot_images(faces)
+
+        plt_emo = plot_predictions(
+            df_pred,
+            "pred_emo",
+            "Emotion",
+            list(config_data.General_DICT_EMO),
+            (12, 2.5),
+            [i + 1 for i in frame_indices],
+            2,
+        )
+        plt_sent = plot_predictions(
+            df_pred,
+            "pred_sent",
+            "Sentiment",
+            list(config_data.General_DICT_SENT),
+            (12, 1.5),
+            [i + 1 for i in frame_indices],
+            2,
         )
-    plt_faces = plot_images(faces)
-
-    plt_emo = plot_predictions(
-        df_pred,
-        "pred_emo",
-        "Emotion",
-        list(config_data.General_DICT_EMO),
-        (12, 2.5),
-        [i + 1 for i in frame_indices],
-        2,
-    )
-    plt_sent = plot_predictions(
-        df_pred,
-        "pred_sent",
-        "Sentiment",
-        list(config_data.General_DICT_SENT),
-        (12, 1.5),
-        [i + 1 for i in frame_indices],
-        2,
-    )
 
     return (
         gr.Textbox(
@@ -176,4 +195,10 @@ def event_handler_submit(
         gr.Plot(value=plt_faces, visible=True),
         gr.Plot(value=plt_emo, visible=True),
         gr.Plot(value=plt_sent, visible=True),
+        gr.Row(visible=True),
+        gr.Textbox(
+            value=config_data.InformationMessages_VIDEO_DURATION.format(vfe.dur),
+            visible=True,
+        ),
+        gr.Textbox(value=t, visible=True),
     )

app/load_models.py

@@ -20,7 +20,15 @@ from transformers.models.wav2vec2.modeling_wav2vec2 import (
     Wav2Vec2PreTrainedModel,
 )
 
-from transformers import AutoConfig, Wav2Vec2Processor, AutoTokenizer, AutoModel
+from transformers import (
+    AutoConfig,
+    Wav2Vec2Processor,
+    AutoTokenizer,
+    AutoModel,
+    logging,
+)
+
+logging.set_verbosity_error()
 
 from app.utils import pth_processing, get_idx_frames_in_windows
 
@@ -838,9 +846,6 @@ class VideoFeatureExtractor:
                 need_features += curr_features.cpu().detach().numpy()[0]
                 count_face += 1
 
-                # face_region = cv2.resize(face_region, (224,224), interpolation = cv2.INTER_AREA)
-                # face_region = display_frame_info(face_region, 'Frame: {}'.format(count_face), box_scale=.3)
-
                 if idx_box in self.faces:
                     self.faces[idx_box].update({counter: face_region})
                 else:

app/tabs.py

@@ -33,7 +33,7 @@ def app_tab():
                 show_label=True,
                 interactive=True,
                 visible=True,
-                mirror_webcam=True,
+                mirror_webcam=False,
                 include_audio=True,
                 elem_classes="video",
                 autoplay=False,
@@ -123,6 +123,50 @@ def app_tab():
                 elem_classes="sent-stats",
             )
 
+            with gr.Row(
+                visible=False,
+                render=True,
+                variant="default",
+                elem_classes="time-container",
+            ) as time_row:
+                video_duration = gr.Textbox(
+                    value=None,
+                    max_lines=1,
+                    placeholder=None,
+                    label=None,
+                    info=None,
+                    show_label=False,
+                    container=False,
+                    interactive=False,
+                    visible=False,
+                    autofocus=False,
+                    autoscroll=True,
+                    render=True,
+                    type="text",
+                    show_copy_button=False,
+                    max_length=50,
+                    elem_classes="video_duration",
+                )
+
+                calculate_time = gr.Textbox(
+                    value=None,
+                    max_lines=1,
+                    placeholder=None,
+                    label=None,
+                    info=None,
+                    show_label=False,
+                    container=False,
+                    interactive=False,
+                    visible=False,
+                    autofocus=False,
+                    autoscroll=True,
+                    render=True,
+                    type="text",
+                    show_copy_button=False,
+                    max_length=50,
+                    elem_classes="calculate_time",
+                )
+
     return (
         video,
         clear,
@@ -132,6 +176,9 @@ def app_tab():
         faces,
         emotion_stats,
         sent_stats,
+        time_row,
+        video_duration,
+        calculate_time,
     )
 
 

app/utils.py

@@ -5,6 +5,7 @@ Description: Utility functions.
 License: MIT License
 """
 
+import time
 import torch
 import os
 import subprocess
@@ -17,10 +18,26 @@ from transformers import WhisperProcessor, WhisperForConditionalGeneration
 from pathlib import Path
 from contextlib import suppress
 from urllib.parse import urlparse
+from contextlib import ContextDecorator
 
 from typing import Callable
 
 
+class Timer(ContextDecorator):
+    """Context manager for measuring code execution time"""
+
+    def __enter__(self):
+        self.start = time.time()
+        return self
+
+    def __exit__(self, *args):
+        self.end = time.time()
+        self.execution_time = f"Inference time: {self.end - self.start:.2f} seconds"
+
+    def __str__(self):
+        return self.execution_time
+
+
 def load_model(
     model_url: str, folder_path: str, force_reload: bool = False
 ) -> str | None:
@@ -259,7 +276,11 @@ class ASRModel:
             transcription = self.processor.batch_decode(
                 predicted_ids, skip_special_tokens=False
             )
-            texts.append(re.findall(r"> ([^<>]+)", transcription[0]))
+            curr_text = re.findall(r"> ([^<>]+)", transcription[0])
+            if curr_text:
+                texts.append(curr_text)
+            else:
+                texts.appemd("")
 
         # for drawing
         input_features = self.processor(

config.toml

@@ -30,6 +30,7 @@ NOTI_RESULTS = [
     "Video uploaded, you can perform calculations",
 ]
 REC_TEXT = "Recognized text"
+VIDEO_DURATION = "Video duration: {:.2f} seconds"
 
 [OtherMessages]
 CLEAR = "Clear"