update

examples/cnn_vad_by_webrtcvad/step_1_prepare_data.py

@@ -56,7 +56,7 @@ def target_second_noise_signal_generator(filename_patterns: List[str],
 
     for epoch_idx in range(max_epoch):
         for filename_pattern in filename_patterns:
-            for filename in glob(filename_pattern):
+            for filename in glob(filename_pattern, recursive=True):
                 signal, _ = librosa.load(filename, sr=sample_rate)
 
                 if signal.ndim != 1:
@@ -109,7 +109,7 @@ def target_second_speech_signal_generator(filename_patterns: List[str],
                                           sample_rate: int = 8000, max_epoch: int = 1):
     for epoch_idx in range(max_epoch):
         for filename_pattern in filename_patterns:
-            for filename in glob(filename_pattern):
+            for filename in glob(filename_pattern, recursive=True):
                 signal, _ = librosa.load(filename, sr=sample_rate)
                 raw_duration = librosa.get_duration(y=signal, sr=sample_rate)
 

examples/fsmn_vad_by_webrtcvad/run.sh

@@ -2,20 +2,6 @@
 
 : <<'END'
 
-bash run.sh --stage 1 --stop_stage 1 --system_version windows \
---file_folder_name fsmn-vad-by-webrtcvad-nx2-dns3 \
---final_model_name fsmn-vad-by-webrtcvad-nx2-dns3 \
---noise_patterns "D:/Users/tianx/HuggingDatasets/nx_noise/data/noise/**/*.wav" \
---speech_patterns "D:/Users/tianx/HuggingDatasets/nx_noise/data/speech/**/*.wav"
-
-
-bash run.sh --stage 1 --stop_stage 1 --system_version centos \
---file_folder_name fsmn-vad-by-webrtcvad-nx2-dns3 \
---final_model_name fsmn-vad-by-webrtcvad-nx2-dns3 \
---noise_patterns "/data/tianxing/HuggingDatasets/nx_noise/data/noise/**/*.wav" \
---speech_patterns "/data/tianxing/HuggingDatasets/nx_noise/data/speech/dns3-speech/**/*.wav \
-/data/tianxing/HuggingDatasets/nx_noise/data/speech/nx-speech2/**/*.wav"
-
 bash run.sh --stage 3 --stop_stage 3 --system_version centos \
 --file_folder_name fsmn-vad-by-webrtcvad-nx2-dns3 \
 --final_model_name fsmn-vad-by-webrtcvad-nx2-dns3 \

examples/silero_vad_by_webrtcvad/run.sh

@@ -2,13 +2,6 @@
 
 : <<'END'
 
-bash run.sh --stage 2 --stop_stage 2 --system_version centos \
---file_folder_name silero-vad-by-webrtcvad-nx2-dns3 \
---final_model_name silero-vad-by-webrtcvad-nx2-dns3 \
---noise_patterns "/data/tianxing/HuggingDatasets/nx_noise/data/noise/**/*.wav" \
---speech_patterns "/data/tianxing/HuggingDatasets/nx_noise/data/speech/dns3-speech/**/*.wav \
-/data/tianxing/HuggingDatasets/nx_noise/data/speech/nx-speech2/**/*.wav"
-
 bash run.sh --stage 3 --stop_stage 3 --system_version centos \
 --file_folder_name silero-vad-by-webrtcvad-nx2-dns3 \
 --final_model_name silero-vad-by-webrtcvad-nx2-dns3 \
@@ -16,7 +9,6 @@ bash run.sh --stage 3 --stop_stage 3 --system_version centos \
 --speech_patterns "/data/tianxing/HuggingDatasets/nx_noise/data/speech/dns3-speech/**/*.wav \
 /data/tianxing/HuggingDatasets/nx_noise/data/speech/nx-speech2/**/*.wav"
 
-
 END
 
 

examples/silero_vad_by_webrtcvad/step_1_prepare_data.py

@@ -35,7 +35,7 @@ def get_args():
     parser.add_argument("--valid_dataset", default="valid.jsonl", type=str)
 
     parser.add_argument("--duration", default=8.0, type=float)
-    parser.add_argument("--min_speech_duration", default=6.0, type=float)
+    parser.add_argument("--min_speech_duration", default=4.0, type=float)
     parser.add_argument("--max_speech_duration", default=8.0, type=float)
     parser.add_argument("--min_snr_db", default=-10, type=float)
     parser.add_argument("--max_snr_db", default=20, type=float)
@@ -56,7 +56,7 @@ def target_second_noise_signal_generator(filename_patterns: List[str],
 
     for epoch_idx in range(max_epoch):
         for filename_pattern in filename_patterns:
-            for filename in glob(filename_pattern):
+            for filename in glob(filename_pattern, recursive=True):
                 signal, _ = librosa.load(filename, sr=sample_rate)
 
                 if signal.ndim != 1:
@@ -109,7 +109,7 @@ def target_second_speech_signal_generator(filename_patterns: List[str],
                                           sample_rate: int = 8000, max_epoch: int = 1):
     for epoch_idx in range(max_epoch):
         for filename_pattern in filename_patterns:
-            for filename in glob(filename_pattern):
+            for filename in glob(filename_pattern, recursive=True):
                 signal, _ = librosa.load(filename, sr=sample_rate)
                 raw_duration = librosa.get_duration(y=signal, sr=sample_rate)
 

examples/silero_vad_by_webrtcvad/step_2_make_vad_segments.py

@@ -4,6 +4,7 @@ import argparse
 import json
 import os
 import sys
+from typing import List, Tuple
 
 pwd = os.path.abspath(os.path.dirname(__file__))
 sys.path.append(os.path.join(pwd, "../../"))
@@ -42,6 +43,54 @@ def get_args():
     return args
 
 
+def get_non_silence_segments(waveform: np.ndarray, sample_rate: int = 8000):
+    non_silent_intervals = librosa.effects.split(
+        waveform,
+        top_db=40,  # 静音阈值（单位：dB）
+        frame_length=512,  # 分析帧长
+        hop_length=128  # 帧移
+    )
+
+    # 输出非静音段的时间区间（单位：秒）
+    result = [(start / sample_rate, end / sample_rate) for (start, end) in non_silent_intervals]
+    return result
+
+
+def get_intersection(non_silence: list[tuple[float, float]],
+                     speech: list[tuple[float, float]]) -> list[tuple[float, float]]:
+    """
+    计算语音段与非静音段的交集
+    :param non_silence: 非静音段列表，格式 [(start1, end1), ...]
+    :param speech: 语音检测段列表，格式 [(start2, end2), ...]
+    :return: 交集段列表，格式 [(start, end), ...]
+    """
+    # 按起始时间排序（假设输入已排序可不排）
+    non_silence = sorted(non_silence, key=lambda x: x[0])
+    speech = sorted(speech, key=lambda x: x[0])
+
+    result = []
+    i = j = 0
+
+    while i < len(non_silence) and j < len(speech):
+        ns_start, ns_end = non_silence[i]
+        sp_start, sp_end = speech[j]
+
+        # 计算重叠区间
+        overlap_start = max(ns_start, sp_start)
+        overlap_end = min(ns_end, sp_end)
+
+        if overlap_start < overlap_end:
+            result.append((overlap_start, overlap_end))
+
+        # 移动指针策略：优先处理先结束的区间
+        if ns_end < sp_end:
+            i += 1  # 非静音段先结束
+        else:
+            j += 1  # 语音段先结束
+
+    return result
+
+
 def main():
     args = get_args()
 
@@ -68,8 +117,8 @@ def main():
         end_ring_rate=0.1,
         frame_size_ms=30,
         frame_step_ms=30,
-        padding_length_ms=90,
-        max_silence_length_ms=100,
+        padding_length_ms=30,
+        max_silence_length_ms=0,
         max_speech_length_s=100,
         min_speech_length_s=0.1,
         sample_rate=args.expected_sample_rate,
@@ -114,6 +163,9 @@ def main():
             )
             waveform = np.array(waveform * (1 << 15), dtype=np.int16)
 
+            # non_silence_segments
+            non_silence_segments = get_non_silence_segments(waveform, sample_rate=args.expected_sample_rate)
+
             # vad
             vad_segments = list()
             segments = w_vad.vad(waveform)
@@ -122,6 +174,7 @@ def main():
             vad_segments += segments
             w_vad.reset()
 
+            vad_segments = get_intersection(non_silence_segments, vad_segments)
             row["vad_segments"] = vad_segments
 
             row = json.dumps(row, ensure_ascii=False)
@@ -168,6 +221,9 @@ def main():
             )
             waveform = np.array(waveform * (1 << 15), dtype=np.int16)
 
+            # non_silence_segments
+            non_silence_segments = get_non_silence_segments(waveform, sample_rate=args.expected_sample_rate)
+
             # vad
             vad_segments = list()
             segments = w_vad.vad(waveform)
@@ -176,6 +232,7 @@ def main():
             vad_segments += segments
             w_vad.reset()
 
+            vad_segments = get_intersection(non_silence_segments, vad_segments)
             row["vad_segments"] = vad_segments
 
             row = json.dumps(row, ensure_ascii=False)

examples/silero_vad_by_webrtcvad/step_4_train_model.py

@@ -255,19 +255,22 @@ def main():
             desc="Training; epoch-{}".format(epoch_idx),
         )
         for train_batch in train_data_loader:
-            noisy_audios, batch_vad_segments = train_batch
+            noisy_audios, clean_audios, batch_vad_segments = train_batch
             noisy_audios: torch.Tensor = noisy_audios.to(device)
+            clean_audios: torch.Tensor = clean_audios.to(device)
             # noisy_audios shape: [b, num_samples]
             num_samples = noisy_audios.shape[-1]
 
-            logits, probs = model.forward(noisy_audios)
+            logits, probs, lsnr = model.forward(noisy_audios)
+            lsnr = torch.squeeze(lsnr, dim=-1)
 
             targets = BaseVadLoss.get_targets(probs, batch_vad_segments, duration=num_samples / config.sample_rate)
 
             bce_loss = bce_loss_fn.forward(probs, targets)
             dice_loss = dice_loss_fn.forward(probs, targets)
+            lsnr_loss = model.lsnr_loss_fn(lsnr, clean_audios, noisy_audios)
 
-            loss = 1.0 * bce_loss + 1.0 * dice_loss
+            loss = 1.0 * bce_loss + 1.0 * dice_loss + 0.03 * lsnr_loss
             if torch.any(torch.isnan(loss)) or torch.any(torch.isinf(loss)):
                 logger.info(f"find nan or inf in loss. continue.")
                 continue
@@ -284,11 +287,13 @@ def main():
             total_loss += loss.item()
             total_bce_loss += bce_loss.item()
             total_dice_loss += dice_loss.item()
+            total_lsnr_loss += lsnr_loss.item()
             total_batches += 1
 
             average_loss = round(total_loss / total_batches, 4)
             average_bce_loss = round(total_bce_loss / total_batches, 4)
             average_dice_loss = round(total_dice_loss / total_batches, 4)
+            average_lsnr_loss = round(total_lsnr_loss / total_batches, 4)
 
             metrics = vad_accuracy_metrics_fn.get_metric()
             accuracy = metrics["accuracy"]
@@ -303,6 +308,7 @@ def main():
                 "loss": average_loss,
                 "bce_loss": average_bce_loss,
                 "dice_loss": average_dice_loss,
+                "lsnr_loss": average_lsnr_loss,
                 "accuracy": accuracy,
                 "f1": f1,
                 "precision": precision,
@@ -322,6 +328,7 @@ def main():
                     total_loss = 0.
                     total_bce_loss = 0.
                     total_dice_loss = 0.
+                    total_lsnr_loss = 0.
                     total_batches = 0.
 
                     progress_bar_train.close()
@@ -329,19 +336,22 @@ def main():
                         desc="Evaluation; steps-{}k".format(int(step_idx/1000)),
                     )
                     for eval_batch in valid_data_loader:
-                        noisy_audios, batch_vad_segments = eval_batch
+                        noisy_audios, clean_audios, batch_vad_segments = eval_batch
                         noisy_audios: torch.Tensor = noisy_audios.to(device)
+                        clean_audios: torch.Tensor = clean_audios.to(device)
                         # noisy_audios shape: [b, num_samples]
                         num_samples = noisy_audios.shape[-1]
 
-                        logits, probs = model.forward(noisy_audios)
+                        logits, probs, lsnr = model.forward(noisy_audios)
+                        lsnr = torch.squeeze(lsnr, dim=-1)
 
                         targets = BaseVadLoss.get_targets(probs, batch_vad_segments, duration=num_samples / config.sample_rate)
 
                         bce_loss = bce_loss_fn.forward(probs, targets)
                         dice_loss = dice_loss_fn.forward(probs, targets)
+                        lsnr_loss = model.lsnr_loss_fn(lsnr, clean_audios, noisy_audios)
 
-                        loss = 1.0 * bce_loss + 1.0 * dice_loss
+                        loss = 1.0 * bce_loss + 1.0 * dice_loss + 0.03 * lsnr_loss
                         if torch.any(torch.isnan(loss)) or torch.any(torch.isinf(loss)):
                             logger.info(f"find nan or inf in loss. continue.")
                             continue
@@ -352,11 +362,13 @@ def main():
                         total_loss += loss.item()
                         total_bce_loss += bce_loss.item()
                         total_dice_loss += dice_loss.item()
+                        total_lsnr_loss += lsnr_loss.item()
                         total_batches += 1
 
                         average_loss = round(total_loss / total_batches, 4)
                         average_bce_loss = round(total_bce_loss / total_batches, 4)
                         average_dice_loss = round(total_dice_loss / total_batches, 4)
+                        average_lsnr_loss = round(total_lsnr_loss / total_batches, 4)
 
                         metrics = vad_accuracy_metrics_fn.get_metric()
                         accuracy = metrics["accuracy"]
@@ -371,6 +383,7 @@ def main():
                             "loss": average_loss,
                             "bce_loss": average_bce_loss,
                             "dice_loss": average_dice_loss,
+                            "lsnr_loss": average_lsnr_loss,
                             "accuracy": accuracy,
                             "f1": f1,
                             "precision": precision,
@@ -384,6 +397,7 @@ def main():
                 total_loss = 0.
                 total_bce_loss = 0.
                 total_dice_loss = 0.
+                total_lsnr_loss = 0.
                 total_batches = 0.
 
                 progress_bar_eval.close()
@@ -425,8 +439,12 @@ def main():
                     "loss": average_loss,
                     "bce_loss": average_bce_loss,
                     "dice_loss": average_dice_loss,
+                    "lsnr_loss": average_lsnr_loss,
 
                     "accuracy": accuracy,
+                    "f1": f1,
+                    "precision": precision,
+                    "recall": recall,
                 }
                 metrics_filename = save_dir / "metrics_epoch.json"
                 with open(metrics_filename, "w", encoding="utf-8") as f:

toolbox/torchaudio/models/vad/fsmn_vad/fsmn_encoder.py

@@ -197,7 +197,6 @@ class FSMN(nn.Module):
         basic_block_rstride: int,
         output_affine_size: int,
         output_size: int,
-        use_softmax: bool = True,
     ):
         super(FSMN, self).__init__()
         self.input_size = input_size

toolbox/torchaudio/models/vad/fsmn_vad/inference_fsmn_vad.py

@@ -68,7 +68,7 @@ class InferenceFSMNVad(object):
         # inputs shape: [1, num_samples,]
 
         with torch.no_grad():
-            logits, probs = self.model.forward(inputs)
+            logits, probs, lsnr = self.model.forward(inputs)
 
         # probs shape: [b, t, 1]
         probs = torch.squeeze(probs, dim=-1)
@@ -92,15 +92,24 @@ def get_args():
         # default=(project_path / "data/examples/hado/b556437e-c68b-4f6d-9eed-2977c29db887.wav").as_posix(),
         # default=(project_path / "data/examples/hado/eae93a33-8ee0-4d86-8f85-cac5116ae6ef.wav").as_posix(),
         # default=(project_path / "data/examples/speech/active_media_r_0ba69730-66a4-4ecd-8929-ef58f18f4612_2.wav").as_posix(),
-        # default=(project_path / "data/examples/speech/active_media_r_2a2f472b-a0b8-4fd5-b1c4-1aedc5d2ce57_0.wav").as_posix(),
-        # default=r"D:\Users\tianx\HuggingDatasets\nx_noise\data\speech\nx-speech\en-SG\2025-05-29\active_media_r_1d4edd08-c6db-41a1-a349-7a22ac36f684_6.wav",
-        # default=r"D:\Users\tianx\HuggingDatasets\nx_noise\data\speech\nx-speech\en-SG\2025-05-29\active_media_r_04f6d842-488e-4e34-967b-2980fdd877c7_5.wav",
-        # default=r"D:\Users\tianx\HuggingDatasets\nx_noise\data\speech\nx-speech\en-SG\2025-05-29\active_media_r_7f6670aa-5600-44c0-9bce-77c1d2b739c7_8.wav",
-        # default=r"D:\Users\tianx\HuggingDatasets\nx_noise\data\speech\nx-speech\en-SG\2025-05-29\active_media_r_1187ff81-3a38-4b0b-846f-b81ad6540ce9_5.wav",
-        # default=r"D:\Users\tianx\HuggingDatasets\nx_noise\data\speech\nx-speech\en-SG\2025-05-29\active_media_r_e44bbfaa-f332-4c02-90a3-cc98505d9a1b_3.wav",
-        # default=r"D:\Users\tianx\HuggingDatasets\nx_noise\data\speech\nx-speech\en-SG\2025-05-29\active_media_r_f89cf1af-f556-42fd-9a42-6c9431002a12_11.wav",
-        # default=r"D:\Users\tianx\HuggingDatasets\nx_noise\data\speech\nx-speech\en-SG\2025-05-29\active_media_r_f89cf1af-f556-42fd-9a42-6c9431002a12_15.wav",
-        default=r"D:\Users\tianx\HuggingDatasets\nx_noise\data\speech\nx-speech\en-SG\2025-05-29\active_media_w_8b6e28e2-a238-4c8c-b2e3-426b1fca149b_6.wav",
+        default=(project_path / "data/examples/speech/active_media_r_2a2f472b-a0b8-4fd5-b1c4-1aedc5d2ce57_0.wav").as_posix(),
+        # default=r"D:\Users\tianx\HuggingDatasets\nx_noise\data\speech\en-SG\2025-05-19\active_media_w_8b6e28e2-a238-4c8c-b2e3-426b1fca149b_6.wav",
+        # default=r"D:\Users\tianx\HuggingDatasets\nx_noise\data\speech\en-SG\2025-05-19\active_media_r_0a56f035-40f6-4530-b852-613f057d718d_6.wav",
+        # default=r"D:\Users\tianx\HuggingDatasets\nx_noise\data\speech\en-SG\2025-05-19\active_media_r_0ae70b76-3651-4a71-bc0c-9e1429e4c854_5.wav",
+        # default=r"D:\Users\tianx\HuggingDatasets\nx_noise\data\speech\en-SG\2025-05-19\active_media_r_0d483249-57f8-4d45-b4c6-bda82d6816ae_2.wav",
+        # default=r"D:\Users\tianx\HuggingDatasets\nx_noise\data\speech\en-SG\2025-05-19\active_media_r_0d952885-5bc2-4633-81b6-e0e809e113f1_2.wav",
+        # default=r"D:\Users\tianx\HuggingDatasets\nx_noise\data\speech\en-SG\2025-05-19\active_media_r_0ddac777-d986-4a5c-9c7c-ff64be0a463d_11.wav",
+
+        # default=r"D:\Users\tianx\HuggingDatasets\nx_noise\data\noise\en-SG\2025-05-19\active_media_r_0b8a8e80-52af-423b-8877-03a78b1e6e43_0.wav",
+        # default=r"D:\Users\tianx\HuggingDatasets\nx_noise\data\noise\en-SG\2025-05-19\active_media_r_0ebffb68-6490-4a8b-8eb6-eb82443d7d75_0.wav",
+        # default=r"D:\Users\tianx\HuggingDatasets\nx_noise\data\noise\en-SG\2025-05-19\active_media_r_0f6ec933-90df-447b-aca4-6ddc149452ab_0.wav",
+        # default=r"D:\Users\tianx\HuggingDatasets\nx_noise\data\noise\en-SG\2025-05-19\active_media_r_1aac396f-1661-4f26-ab49-1a4879684567_0.wav",
+        # default=r"D:\Users\tianx\HuggingDatasets\nx_noise\data\noise\en-SG\2025-05-19\active_media_r_1aac396f-1661-4f26-ab49-1a4879684567_1.wav",
+        # default=r"D:\Users\tianx\HuggingDatasets\nx_noise\data\noise\en-SG\2025-05-19\active_media_r_1aff518b-4749-42fc-adfe-64046f9baeb6_0.wav",
+        # default=r"D:\Users\tianx\HuggingDatasets\nx_noise\data\noise\en-SG\2025-05-19\active_media_r_1b16f2a3-a8c9-4739-9a76-59faf1c64d79_0.wav",
+        # default=r"D:\Users\tianx\HuggingDatasets\nx_noise\data\noise\en-SG\2025-05-19\active_media_r_1b16f2a3-a8c9-4739-9a76-59faf1c64d79_1.wav",
+        # default=r"D:\Users\tianx\HuggingDatasets\nx_noise\data\noise\en-SG\2025-05-19\active_media_r_1bb1f22e-9c3a-4aea-b53f-71cc6547a6ee_0.wav",
+        # default=r"D:\Users\tianx\HuggingDatasets\nx_noise\data\noise\en-SG\2025-05-19\active_media_r_1dab161b-2a76-4491-abd1-60dba6172f8d_2.wav",
         type=str,
     )
     args = parser.parse_args()
@@ -119,7 +128,8 @@ def main():
     signal = signal / (1 << 15)
 
     infer = InferenceFSMNVad(
-        pretrained_model_path_or_zip_file=(project_path / "trained_models/fsmn-vad-by-webrtcvad-nx-dns3.zip").as_posix()
+        # pretrained_model_path_or_zip_file=(project_path / "trained_models/fsmn-vad-by-webrtcvad-nx-dns3.zip").as_posix(),
+        pretrained_model_path_or_zip_file = (project_path / "trained_models/fsmn-vad-by-webrtcvad-nx2-dns3.zip").as_posix(),
     )
     frame_step = infer.config.hop_size