update

examples/evaluation/step_1_run_evaluation.py

@@ -26,7 +26,7 @@ def get_args():
     )
     parser.add_argument(
         "--output_file",
-        default=r"fsmn-vad.jsonl",
+        default=r"evaluation.jsonl",
         type=str
     )
     parser.add_argument("--expected_sample_rate", default=8000, type=int)
@@ -105,12 +105,13 @@ def main():
                 },
                 audio_microphone_t=None,
                 start_ring_rate=0.5,
-                end_ring_rate=0.5,
-                ring_max_length=1,
+                end_ring_rate=0.3,
+                ring_max_length=10,
                 min_silence_length=6,
                 max_speech_length=100000,
                 min_speech_length=15,
-                engine="fsmn-vad-by-webrtcvad-nx2-dns3",
+                # engine="fsmn-vad-by-webrtcvad-nx2-dns3",
+                engine="silero-vad-by-webrtcvad-nx2-dns3",
                 api_name="/when_click_vad_button"
             )
             js = json.loads(message)
@@ -138,16 +139,16 @@ def main():
             f.write(f"{row_}\n")
 
             total += 1
-            total_accuracy += accuracy
-            total_precision += precision
-            total_recall += recall
-            total_f1 += f1
             total_duration += duration
-
-            average_accuracy = total_accuracy / total
-            average_precision = total_precision / total
-            average_recall = total_recall / total
-            average_f1 = total_f1 / total
+            total_accuracy += accuracy * duration
+            total_precision += precision * duration
+            total_recall += recall * duration
+            total_f1 += f1 * duration
+
+            average_accuracy = total_accuracy / total_duration
+            average_precision = total_precision / total_duration
+            average_recall = total_recall / total_duration
+            average_f1 = total_f1 / total_duration
 
             progress_bar.update(1)
             progress_bar.set_postfix({

examples/evaluation/step_2_show_metrics.py

@@ -0,0 +1,70 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+import argparse
+import json
+import os
+import sys
+
+pwd = os.path.abspath(os.path.dirname(__file__))
+sys.path.append(os.path.join(pwd, "../../"))
+
+from tqdm import tqdm
+
+
+def get_args():
+    parser = argparse.ArgumentParser()
+
+    parser.add_argument(
+        "--eval_file",
+        default=r"evaluation.jsonl",
+        type=str
+    )
+    args = parser.parse_args()
+    return args
+
+
+def main():
+    args = get_args()
+
+    total = 0
+    total_duration = 0
+    total_accuracy = 0
+    total_precision = 0
+    total_recall = 0
+    total_f1 = 0
+    progress_bar = tqdm(desc="evaluation")
+    with open(args.eval_file, "r", encoding="utf-8") as f:
+        for row in f:
+            row = json.loads(row)
+            duration = row["duration"]
+            accuracy = row["accuracy"]
+            precision = row["precision"]
+            recall = row["recall"]
+            f1 = row["f1"]
+
+            total += 1
+            total_duration += duration
+            total_accuracy += accuracy * duration
+            total_precision += precision * duration
+            total_recall += recall * duration
+            total_f1 += f1 * duration
+
+            average_accuracy = total_accuracy / total_duration
+            average_precision = total_precision / total_duration
+            average_recall = total_recall / total_duration
+            average_f1 = total_f1 / total_duration
+
+            progress_bar.update(1)
+            progress_bar.set_postfix({
+                "total": total,
+                "accuracy": average_accuracy,
+                "precision": average_precision,
+                "recall": average_recall,
+                "f1": average_f1,
+                "total_duration": f"{round(total_duration / 60, 4)}min",
+            })
+    return
+
+
+if __name__ == "__main__":
+    main()

examples/evaluation/step_3_show_vad.py

@@ -0,0 +1,105 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+import argparse
+import json
+import os
+import sys
+import tempfile
+
+pwd = os.path.abspath(os.path.dirname(__file__))
+sys.path.append(os.path.join(pwd, "../../"))
+
+import matplotlib.pyplot as plt
+import numpy as np
+from scipy.io import wavfile
+from tqdm import tqdm
+
+
+def get_args():
+    parser = argparse.ArgumentParser()
+
+    parser.add_argument(
+        "--eval_file",
+        default=r"evaluation.jsonl",
+        type=str
+    )
+    args = parser.parse_args()
+    return args
+
+
+def show_image(signal: np.ndarray,
+               ground_truth_probs: np.ndarray,
+               prediction_probs: np.ndarray,
+               sample_rate: int = 8000,
+               ):
+    duration = np.arange(0, len(signal)) / sample_rate
+    plt.figure(figsize=(12, 5))
+
+    plt.subplot(2, 1, 1)  # 2行1列，第1个位置
+    plt.plot(duration, signal, color="b")
+    plt.plot(duration, ground_truth_probs, color="gray")
+    plt.title("ground_truth")
+
+    plt.subplot(2, 1, 2)  # 2行1列，第2个位置
+    plt.plot(duration, signal, color="b")
+    plt.plot(duration, prediction_probs, color="gray")
+    plt.title("prediction")
+
+    # plt.tight_layout()
+    plt.subplots_adjust(hspace=0.5)  # 调整上下间距
+
+    plt.show()
+
+
+def main():
+    args = get_args()
+
+    with open(args.eval_file, "r", encoding="utf-8") as f:
+        for row in f:
+            row = json.loads(row)
+            filename = row["filename"]
+            duration = row["duration"]
+            ground_truth = row["ground_truth"]
+            prediction = row["prediction"]
+
+            accuracy = row["accuracy"]
+            precision = row["precision"]
+            recall = row["recall"]
+            f1 = row["f1"]
+
+            sample_rate, signal = wavfile.read(
+                filename=filename,
+            )
+            signal = np.array(signal / (1 << 15), dtype=np.float32)
+            signal_length = len(signal)
+            ground_truth_probs = np.zeros(shape=(signal_length,), dtype=np.float32)
+            for begin, end in ground_truth:
+                begin = int(begin * sample_rate)
+                end = int(end * sample_rate)
+                ground_truth_probs[begin:end] = 1
+            prediction_probs = np.zeros(shape=(signal_length,), dtype=np.float32)
+            for begin, end in prediction:
+                begin = int(begin * sample_rate)
+                end = int(end * sample_rate)
+                prediction_probs[begin:end] = 1
+
+            # p = encoder_num_layers * (encoder_kernel_size - 1) // 2 * hop_size * sample_rate
+            p = 3 * (3 - 1) // 2 * 80
+            p = int(p)
+            print(f"p: {p}")
+            prediction_probs = np.concat(
+                [
+                    prediction_probs[p:], prediction_probs[-p:]
+                ],
+                axis=-1
+            )
+
+            show_image(signal,
+                       ground_truth_probs, prediction_probs,
+                       sample_rate=sample_rate,
+                       )
+    return
+
+
+if __name__ == "__main__":
+    main()

examples/fsmn_vad_by_webrtcvad/run.sh

@@ -2,7 +2,7 @@
 
 : <<'END'
 
-bash run.sh --stage 3 --stop_stage 3 --system_version centos \
+bash run.sh --stage 3 --stop_stage 5 --system_version centos \
 --file_folder_name fsmn-vad-by-webrtcvad-nx2-dns3 \
 --final_model_name fsmn-vad-by-webrtcvad-nx2-dns3 \
 --noise_patterns "/data/tianxing/HuggingDatasets/nx_noise/data/noise/**/*.wav" \

examples/fsmn_vad_by_webrtcvad/step_4_train_model.py

@@ -127,6 +127,7 @@ def main():
         max_wave_value=32768.0,
         min_snr_db=config.min_snr_db,
         max_snr_db=config.max_snr_db,
+        do_volume_enhancement=True,
         # skip=225000,
     )
     valid_dataset = VadPaddingJsonlDataset(

examples/silero_vad_by_webrtcvad/step_4_train_model.py

@@ -127,6 +127,7 @@ def main():
         max_wave_value=32768.0,
         min_snr_db=config.min_snr_db,
         max_snr_db=config.max_snr_db,
+        do_volume_enhancement=True,
         # skip=225000,
     )
     valid_dataset = VadPaddingJsonlDataset(

requirements.txt

@@ -4,6 +4,7 @@ datasets==3.2.0
 python-dotenv==1.0.1
 scipy==1.15.1
 librosa==0.10.2.post1
+pydub==0.25.1
 pandas==2.2.3
 openpyxl==3.1.5
 torch==2.5.1

toolbox/pydub/__init__.py

@@ -0,0 +1,6 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+
+
+if __name__ == "__main__":
+    pass

toolbox/pydub/volume.py

@@ -0,0 +1,106 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+import argparse
+from typing import List
+
+import librosa
+import numpy as np
+from pydub import AudioSegment
+from scipy.io import wavfile
+
+from project_settings import project_path
+
+
+def score_transform(x: float, stages: List[float], scores: List[float], ndigits: int = 4):
+    last_stage = stages[0]
+    last_score = scores[0]
+    stages = stages[1:]
+    scores = scores[1:]
+    for stage, score in zip(stages, scores):
+        if x >= stage:
+            result = score + (x - stage) / (last_stage - stage + 1e-7) * (last_score - score)
+            return round(result, ndigits)
+        last_stage = stage
+        last_score = score
+    raise ValueError(f"values of x, stages and scores should between 0 and 1, "
+                     f"stages and scores should be same length and decreased. "
+                     f"x: {x}, stages: {stages}, scores: {scores}")
+
+
+def set_volume(waveform: np.ndarray, sample_rate: int = 8000, volume: int = 0):
+    if np.min(waveform) < -1 or np.max(waveform) > 1:
+        raise AssertionError(f"waveform type: {type(waveform)}, dtype: {waveform.dtype}")
+    waveform = np.array(waveform * (1 << 15), dtype=np.int16)
+    raw_data = waveform.tobytes()
+
+    audio_segment = AudioSegment(
+        data=raw_data,
+        sample_width=2,
+        frame_rate=sample_rate,
+        channels=1
+    )
+
+    map_list = [
+        [0, -150],
+        [10, -60],
+        [50, -35],
+        [100, -20],
+    ]
+    stages = [a for a, b in map_list]
+    scores = [b for a, b in map_list]
+
+    # 计算目标 dBFS
+    target_db = score_transform(
+        x=volume,
+        stages=list(reversed(stages)),
+        scores=list(reversed(scores)),
+    )
+
+    audio_segment = audio_segment.apply_gain(target_db - audio_segment.dBFS)
+
+    samples = np.array(audio_segment.get_array_of_samples())
+
+    if audio_segment.sample_width == 2:
+        samples = samples.astype(np.float32) / (1 << (2*8-1))
+    elif audio_segment.sample_width == 3:
+        samples = samples.astype(np.float32) / (1 << (3*8-1))
+    elif audio_segment.sample_width == 4:
+        samples = samples.astype(np.float32) / (1 << (4*8-1))
+    else:
+        raise AssertionError
+    return samples
+
+
+def get_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--filename",
+        default=(project_path / "data/examples/ai_agent/chinese-1.wav").as_posix(),
+        type=str
+    )
+    args = parser.parse_args()
+    return args
+
+
+def main():
+    args = get_args()
+
+    waveform, sample_rate = librosa.load(args.filename, sr=8000)
+
+    waveform = set_volume(
+        waveform=waveform,
+        sample_rate=sample_rate,
+        volume=10
+    )
+    waveform = np.array(waveform * (1 << 15), dtype=np.int16)
+
+    wavfile.write(
+        "temp.wav",
+        rate=8000,
+        data=waveform,
+    )
+    return
+
+
+if __name__ == "__main__":
+    main()

toolbox/torch/utils/data/dataset/vad_padding_jsonl_dataset.py

@@ -9,6 +9,8 @@ import numpy as np
 import torch
 from torch.utils.data import Dataset, IterableDataset
 
+from toolbox.pydub.volume import set_volume
+
 
 class VadPaddingJsonlDataset(IterableDataset):
     def __init__(self,
@@ -19,6 +21,7 @@ class VadPaddingJsonlDataset(IterableDataset):
                  buffer_size: int = 1000,
                  min_snr_db: float = None,
                  max_snr_db: float = None,
+                 do_volume_enhancement: bool = False,
                  speech_target_duration: float = 8.0,
                  eps: float = 1e-8,
                  skip: int = 0,
@@ -29,6 +32,7 @@ class VadPaddingJsonlDataset(IterableDataset):
         self.max_wave_value = max_wave_value
         self.min_snr_db = min_snr_db
         self.max_snr_db = max_snr_db
+        self.do_volume_enhancement = do_volume_enhancement
         self.speech_target_duration = speech_target_duration
         self.eps = eps
         self.skip = skip
@@ -134,6 +138,10 @@ class VadPaddingJsonlDataset(IterableDataset):
         speech_wave_np, left_pad_duration, _ = self.pad_waveform(speech_wave_np, self.expected_sample_rate, self.speech_target_duration)
         speech_wave_np = self.make_sure_duration(speech_wave_np, self.expected_sample_rate, self.speech_target_duration)
 
+        # volume enhancement
+        volume = random.randint(0, 100)
+        speech_wave_np = set_volume(speech_wave_np, sample_rate=self.expected_sample_rate, volume=volume)
+
         noise_wave_list = list()
         for noise in noise_list:
             filename = noise["filename"]