quality of life UI improvements

.gitignore

@@ -201,4 +201,6 @@ lib/
 _outputs/
 debug.txt
 
-scratch/*
+scratch/*
+
+.gradio

README.md

@@ -4,8 +4,8 @@ emoji: 🥗
 colorFrom: yellow
 colorTo: green
 sdk: gradio
-sdk_version: 4.43.0
-python_version: 3.9.17
+sdk_version: 5.23.2
+python_version: 3.11
 app_file: app.py
 pinned: false
 license: cc-by-nc-4.0

app.py

@@ -107,11 +107,30 @@ def new_vampnet_mask(self,
     mask = mask.to(self.device)
     return mask[:, :, :]
 
+def mask_preview(periodic_p, n_mask_codebooks, onset_mask_width, dropout):
+    # make a mask preview
+    codes = torch.zeros((1, 14, 80)).to(device)
+    mask = interface.build_mask(
+        codes,
+        periodic_prompt=periodic_p,
+        # onset_mask_width=onset_mask_width,
+        _dropout=dropout,
+        upper_codebook_mask=n_mask_codebooks,
+    )
+    # mask = mask.cpu().numpy()
+    import matplotlib.pyplot as plt
+    plt.clf()
+    interface.visualize_codes(mask)
+    plt.title("mask preview")
+    plt.savefig("scratch/mask-prev.png")
+    return "scratch/mask-prev.png"
+
+
 @spaces.GPU
-def _vamp(
+def _vamp_internal(
         seed, input_audio, model_choice, 
         pitch_shift_amt, periodic_p, 
-        n_mask_codebooks, periodic_w, onset_mask_width, 
+        n_mask_codebooks, onset_mask_width, 
         dropout, sampletemp, typical_filtering, 
         typical_mass, typical_min_tokens, top_p, 
         sample_cutoff, stretch_factor, sampling_steps, beat_mask_ms, num_feedback_steps, api=False
@@ -124,7 +143,6 @@ def _vamp(
     print(f"pitch_shift_amt: {pitch_shift_amt}")
     print(f"periodic_p: {periodic_p}")
     print(f"n_mask_codebooks: {n_mask_codebooks}")
-    print(f"periodic_w: {periodic_w}")
     print(f"onset_mask_width: {onset_mask_width}")
     print(f"dropout: {dropout}")
     print(f"sampletemp: {sampletemp}")
@@ -149,6 +167,8 @@ def _vamp(
         _seed = int(torch.randint(0, 2**32, (1,)).item())
     at.util.seed(_seed)
 
+    if input_audio is None:
+        raise gr.Error("no input audio received!")
     sr, input_audio = input_audio
     input_audio = input_audio / np.iinfo(input_audio.dtype).max
     
@@ -180,7 +200,6 @@ def _vamp(
         codes,
         sig=sig, 
         periodic_prompt=periodic_p,
-        periodic_prompt_width=periodic_w,
         onset_mask_width=onset_mask_width,
         _dropout=dropout,
         upper_codebook_mask=n_mask_codebooks,
@@ -188,7 +207,7 @@ def _vamp(
     if beat_mask_ms > 0:
         # bm = pmask.mask_or(
         #     pmask.periodic_mask(
-        #         codes, periodic_p, periodic_w, random_roll=False
+        #         codes, periodic_p, random_roll=False
         #     ),
         # )
         mask = pmask.mask_and(
@@ -208,7 +227,7 @@ def _vamp(
         else:
             top_p = None
 
-    codes, mask = interface.vamp(
+    codes, mask_z = interface.vamp(
         codes, mask,
         batch_size=2,
         feedback_steps=num_feedback_steps,
@@ -228,36 +247,64 @@ def _vamp(
     sig = interface.decode(codes)
     sig = sig.normalize(loudness)
 
-    return to_output(sig[0]), to_output(sig[1])
-
-def vamp(data):
-    return _vamp(
-        seed=data[seed], 
-        input_audio=data[input_audio],
-        model_choice=data[model_choice],
-        pitch_shift_amt=data[pitch_shift_amt],
-        periodic_p=data[periodic_p],
-        n_mask_codebooks=data[n_mask_codebooks],
-        periodic_w=data[periodic_w],
-        onset_mask_width=data[onset_mask_width],
-        dropout=data[dropout],
-        sampletemp=data[sampletemp],
-        typical_filtering=data[typical_filtering],
-        typical_mass=data[typical_mass],
-        typical_min_tokens=data[typical_min_tokens],
-        top_p=data[top_p],
-        sample_cutoff=data[sample_cutoff],
-        stretch_factor=data[stretch_factor],
-        sampling_steps=data[sampling_steps],
-        beat_mask_ms=data[beat_mask_ms],
-        num_feedback_steps=data[num_feedback_steps],
-        api=False, 
+    import matplotlib.pyplot as plt
+    plt.clf()
+    # plt.imshow(mask_z[0].cpu().numpy(), aspect='auto
+    interface.visualize_codes(mask)
+    plt.title("actual mask")
+    plt.savefig("scratch/mask.png")
+    plt.clf()
+
+    if not api:
+        return to_output(sig[0]), to_output(sig[1]), "scratch/mask.png"
+    else:
+        return to_output(sig[0]), to_output(sig[1])
+
+
+def vamp(input_audio, 
+        sampletemp,
+        top_p,
+        periodic_p, 
+        dropout,
+        stretch_factor, 
+        onset_mask_width, 
+        typical_filtering,
+        typical_mass,
+        typical_min_tokens,
+        seed, 
+        model_choice,
+        n_mask_codebooks,
+        pitch_shift_amt, 
+        sample_cutoff, 
+        sampling_steps, 
+        beat_mask_ms,
+        num_feedback_steps):
+    return _vamp_internal(
+        seed=seed,
+        input_audio=input_audio,
+        model_choice=model_choice,
+        pitch_shift_amt=pitch_shift_amt,
+        periodic_p=periodic_p,
+        n_mask_codebooks=n_mask_codebooks,
+        onset_mask_width=onset_mask_width,
+        dropout=dropout,
+        sampletemp=sampletemp,
+        typical_filtering=typical_filtering,
+        typical_mass=typical_mass,
+        typical_min_tokens=typical_min_tokens,
+        top_p=top_p,
+        sample_cutoff=sample_cutoff,
+        stretch_factor=stretch_factor,
+        sampling_steps=sampling_steps,
+        beat_mask_ms=beat_mask_ms,
+        num_feedback_steps=num_feedback_steps,
+        api=False,
     )
 
 
 def api_vamp(input_audio, 
                 sampletemp, top_p, 
-                periodic_p, periodic_w, 
+                periodic_p,
                 dropout, 
                 stretch_factor,
                 onset_mask_width,
@@ -271,14 +318,13 @@ def api_vamp(input_audio,
                 sample_cutoff, 
                 sampling_steps, 
                 beat_mask_ms, num_feedback_steps):
-    return _vamp(
+    return _vamp_internal(
         seed=seed, 
         input_audio=input_audio,
         model_choice=model_choice,
         pitch_shift_amt=pitch_shift_amt,
         periodic_p=periodic_p,
         n_mask_codebooks=n_mask_codebooks,
-        periodic_w=periodic_w,
         onset_mask_width=onset_mask_width,
         dropout=dropout,
         sampletemp=sampletemp,
@@ -294,6 +340,29 @@ def api_vamp(input_audio,
         api=True, 
     )
 
+def harp_vamp(input_audio, sampletemp, periodic_p, dropout, n_mask_codebooks, model_choice, beat_mask_ms):
+    return _vamp_internal(
+        seed=0, 
+        input_audio=input_audio,
+        model_choice=model_choice,
+        pitch_shift_amt=0,
+        periodic_p=periodic_p,
+        n_mask_codebooks=n_mask_codebooks,
+        onset_mask_width=0,
+        dropout=dropout,
+        sampletemp=sampletemp,
+        typical_filtering=False,
+        typical_mass=0.15,
+        typical_min_tokens=1,
+        top_p=None,
+        sample_cutoff=1.0,
+        stretch_factor=1.0,
+        sampling_steps=36,
+        beat_mask_ms=beat_mask_ms,
+        num_feedback_steps=1
+    )
+
+
 with gr.Blocks() as demo:
     with gr.Row():
         with gr.Column():
@@ -309,11 +378,11 @@ with gr.Blocks() as demo:
                 type="numpy",
             )
 
-            audio_mask = gr.Audio(
-                label="audio mask (listen to this to hear the mask hints)",
-                interactive=False, 
-                type="numpy",
-            )
+            # audio_mask = gr.Audio(
+            #     label="audio mask (listen to this to hear the mask hints)",
+            #     interactive=False, 
+            #     type="numpy",
+            # )
 
             # connect widgets
             load_example_audio_button.click(
@@ -341,13 +410,22 @@ with gr.Blocks() as demo:
                 )
 
                 onset_mask_width = gr.Slider(
-                    label="onset mask width (multiplies with the periodic mask, 1 step ~= 10milliseconds) ",
+                    label="onset mask width (multiplies with the periodic mask, 1 step ~= 10milliseconds) does not affect mask preview",
                     minimum=0,
                     maximum=100,
                     step=1,
                     value=0, visible=True
                 )
 
+                beat_mask_ms = gr.Slider(
+                    label="beat mask width (milliseconds) does not affect mask preview",
+                    minimum=1,
+                    maximum=200, 
+                    step=1,
+                    value=0, 
+                    visible=True
+                )
+
                 n_mask_codebooks = gr.Slider(
                     label="compression prompt ",
                     value=3,
@@ -355,7 +433,15 @@ with gr.Blocks() as demo:
                     maximum=14,
                     step=1,
                 )
-            
+
+                dropout = gr.Slider(
+                    label="mask dropout",
+                    minimum=0.0,
+                    maximum=1.0,
+                    step=0.01,
+                    value=0.0
+                )
+
             maskimg = gr.Image(
                 label="mask image",
                 interactive=False,
@@ -379,18 +465,7 @@ with gr.Blocks() as demo:
                     value=1, 
                 )
 
-                periodic_w = gr.Slider(
-                    label="periodic prompt width (steps, 1 step ~= 10milliseconds)",
-                    minimum=1,
-                    maximum=20,
-                    step=1,
-                    value=1,
-                )
 
-                beat_mask_ms = gr.Number(
-                    label="beat mask width (milliseconds)",
-                    value=0,
-                )
 
 
             with gr.Accordion("sampling settings", open=False):
@@ -448,17 +523,6 @@ with gr.Blocks() as demo:
                 )
 
 
-
-
-            dropout = gr.Slider(
-                label="mask dropout",
-                minimum=0.0,
-                maximum=1.0,
-                step=0.01,
-                value=0.0
-            )
-
-
             seed = gr.Number(
                 label="seed (0 for random)",
                 value=0,
@@ -499,11 +563,24 @@ with gr.Blocks() as demo:
             # download = gr.File(type="filepath", label="download outputs")
 
 
-    _inputs = {
+    # mask preview change
+    for widget in (
+        periodic_p, n_mask_codebooks, 
+        onset_mask_width, dropout
+    ):
+        widget.change(
+            fn=mask_preview,
+            inputs=[periodic_p, n_mask_codebooks, 
+                    onset_mask_width, dropout],
+            outputs=[maskimg]
+        )
+
+
+    _inputs = [
             input_audio, 
             sampletemp,
             top_p,
-            periodic_p, periodic_w,
+            periodic_p,
             dropout,
             stretch_factor, 
             onset_mask_width, 
@@ -518,13 +595,13 @@ with gr.Blocks() as demo:
             sampling_steps, 
             beat_mask_ms,
             num_feedback_steps
-        }
+    ]
   
     # connect widgets
     vamp_button.click(
         fn=vamp,
         inputs=_inputs,
-        outputs=[audio_outs[0], audio_outs[1]], 
+        outputs=[audio_outs[0], audio_outs[1], maskimg], 
     )
 
     api_vamp_button = gr.Button("api vamp", visible=True)
@@ -532,7 +609,7 @@ with gr.Blocks() as demo:
         fn=api_vamp,
         inputs=[input_audio, 
                 sampletemp, top_p, 
-                periodic_p, periodic_w, 
+                periodic_p, 
                 dropout, 
                 stretch_factor,
                 onset_mask_width,

scratch/rms_mask.txt

@@ -1,6 +1,6 @@
-0 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 0 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 0 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 0 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 0 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 0 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 0 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 0 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 0 0 1 0 1 0 1 0 1 0 1 0 1 0 0 0 1 0 1 0 1 0 1 0 1 0 1 0 0 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 0 0 1 0 1 0 1 0 1 0 1 0 1 0 0 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 0 0 0 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 0 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 0 0 1 0 1 0 1 0 1 0 1 0 1 0 0 0 1 0 1 0 1 0 1
-1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1
-1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1
+1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 1 1 1 1 1 1 1 1 1 1 1 1 0 0 1 1 1 1 1 1 1 1 1 1 1 1 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 1 1 1 1 1 1 1 1 1 1 1 0 0 1 1 1 1 1 1 1 1 1 1 1 1 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 1 1 1 1 1 1 1 1 1 1 1 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 1 1 1 1 1 1 1 1 1 1 1 1 0 0 1 1 1 1 1 1 1 1 1 1 1 1 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 1 1 1 1 1 1 1 1 1 1 1 0 0 1 1 1 1 1 1 1 1 1 1 1 1 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 1 1 1 1 1 1 1 1 1 1 1 0 0 1 1 1 1 1 1 1 1 1 1 1 1 0 0 1 1 1 1 1 1 1
+1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 1 1 1 1 1 1 1 1 1 1 1 1 0 0 1 1 1 1 1 1 1 1 1 1 1 1 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 1 1 1 1 1 1 1 1 1 1 1 0 0 1 1 1 1 1 1 1 1 1 1 1 1 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 1 1 1 1 1 1 1 1 1 1 1 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 1 1 1 1 1 1 1 1 1 1 1 1 0 0 1 1 1 1 1 1 1 1 1 1 1 1 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 1 1 1 1 1 1 1 1 1 1 1 0 0 1 1 1 1 1 1 1 1 1 1 1 1 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 1 1 1 1 1 1 1 1 1 1 1 0 0 1 1 1 1 1 1 1 1 1 1 1 1 0 0 1 1 1 1 1 1 1
+1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 1 1 1 1 1 1 1 1 1 1 1 1 0 0 1 1 1 1 1 1 1 1 1 1 1 1 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 1 1 1 1 1 1 1 1 1 1 1 0 0 1 1 1 1 1 1 1 1 1 1 1 1 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 1 1 1 1 1 1 1 1 1 1 1 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 1 1 1 1 1 1 1 1 1 1 1 1 0 0 1 1 1 1 1 1 1 1 1 1 1 1 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 1 1 1 1 1 1 1 1 1 1 1 0 0 1 1 1 1 1 1 1 1 1 1 1 1 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 1 1 1 1 1 1 1 1 1 1 1 0 0 1 1 1 1 1 1 1 1 1 1 1 1 0 0 1 1 1 1 1 1 1
 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1
 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1
 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1

setup.py

@@ -21,7 +21,7 @@ setup(
     long_description=long_description,
     long_description_content_type="text/markdown",
     author="Hugo Flores García, Prem Seetharaman",
-    author_email="hfgacrcia@descript.com",
+    author_email="hugggofloresgarcia@gmail.com",
     url="https://github.com/hugofloresgarcia/vampnet",
     license="MIT",
     packages=find_packages(),

unloop/client.py

@@ -89,8 +89,6 @@ class GradioOSCClient:
         ip: str,
         s_port: int, r_port: int,
         vampnet_url: str = None, # url for vampnet
-        s2s_urls: str = None, # urls for sketch2sound (multiple for parallel processing)
-        s2s_vim_url: str = None, # for sketch2sound old version (good at vocal imitations, (vims))
     ):
         self.osc_manager = OSCManager(
             ip=ip, s_port=s_port, r_port=r_port, 
@@ -100,13 +98,6 @@ class GradioOSCClient:
         self.clients = {}
         if vampnet_url is not None:
             self.clients["vampnet"] = Client(src=vampnet_url, download_files=DOWNLOADS_DIR)
-        if s2s_urls is not None:
-            # self.clients["s2s"] = Client(src=s2s_url, download_files=DOWNLOADS_DIR)
-            for i, s2s_url in enumerate(s2s_urls):
-                if s2s_url is not None:
-                    self.clients[f"s2s_{i}"] = Client(src=s2s_url, download_files=DOWNLOADS_DIR)
-        if s2s_vim_url is not None:
-            self.clients["s2s_vim"] = Client(src=s2s_vim_url, download_files=DOWNLOADS_DIR)
         
         assert len(self.clients) > 0, "At least one client must be specified!"
 
@@ -174,7 +165,6 @@ class GradioOSCClient:
             sampletemp=temperature,
             top_p=top_p,
             periodic_p=periodic_p,
-            periodic_w=1,
             dropout=dropout,
             stretch_factor=1,
             onset_mask_width=onset_mask_width,
@@ -237,154 +227,12 @@ class GradioOSCClient:
             return
         else:
             raise ValueError(f"Unknown client type {client_type}")
-        
-    def process_s2s(self, address: str, *args):
-        # client = self.clients["s2s"]
-
-        if address != "/process":
-            raise ValueError(f"Unknown address {address}")
-
-        print(f"Processing {address} with args {args}")
-        # unpack the args
-        query_id = args[0]
-        client_type = args[1]
-        audio_path = Path(args[2])
-        text_prompt = args[3]
-        use_control = args[4] == 1
-        looplength = args[5]
-        guidance_scale = args[6]
-        seed = args[7]
-        median_filter_length = args[8]
-
-        sig = at.AudioSignal(audio_path)
-        looplength_ms = looplength
-        # grab the looplength only
-        # TODO: although I added this, 
-        # the max patch is still configured to crop anything past the looplength off
-        # so we'll have to change that in order to make an effect. 
-        end_sample = int(looplength_ms * sig.sample_rate / 1000)
-        sig.samples = sig.samples[..., :end_sample]
-
-        # grab  the remainder of the waveform
-        num_cut_samples = sig.samples.shape[-1] - end_sample
-        cut_wav = sig.samples[..., -num_cut_samples:]
-
-        # write the file back
-        sig.write(audio_path)
-
-        # make sure it exists, otherwise send an error message
-        if not audio_path.exists():
-            print(f"File {audio_path} does not exist")
-            self.osc_manager.error(f"File {audio_path} does not exist")
-            return
-
-        
-        timer.tick(f"predict-{query_id}")
-        # NEW API
-        if use_control and "s2s_vim" in self.clients:
-            client = self.clients["s2s_vim"]
-            job = client.submit(
-                data=handle_file(audio_path),
-                param_1=-20, # comp threshold
-                param_2=1, # comp ratio
-                param_3=seed, # random seed
-                param_4=text_prompt, # text prompt
-                param_5=5, # text guid
-                param_6=1, # ctrl guid
-                param_7=-1,  # t_low
-                param_8=1, # t_high
-                param_9=15, # median filt
-                param_10=True, # use centroid
-                param_11=True, # use pitch
-                param_12=True, # use ldns
-                param_13=0, # transpose (semis)
-                param_14=0, # pitch shift semis
-                param_15=0, # pitchiness ofset
-                param_16=0, # gain db
-                param_17=100, # num steps
-                api_name="/generate"
-            )
-        else: 
-            client = self.clients[f"s2s_{self.inf_idx % len(self.clients)}"]
-            self.inf_idx += 1
-            params = {  
-                    'control_guidance_scale': 1.0,
-                    'guidance_scale': guidance_scale,
-                    'logsnr_max': 5.0,
-                    'logsnr_min': -8,
-                    'num_seconds': looplength / 1000.,
-                    'num_steps': 24,
-                    'rho': 7.0,
-                    'sampler': 'dpmpp-2m-sde',
-                    'schedule': 'karras'
-            }
-            job = client.submit(
-                    text_prompt=text_prompt,
-                    control_audio=handle_file(audio_path) if use_control else None,
-                    seed=seed,
-                    median_filter_length=median_filter_length,
-                    normalize_db=-16,
-                    duration=looplength / 1000.,
-                    params_str=json.dumps(params),
-                    api_name="/generate_with_params"
-            )
-
-        while not job.done():
-            time.sleep(0.1)
-            status = str(job.status().code)
-
-            if "FINISHED" in status:
-                status = "STATUS.WAITING"
-        
-            self.osc_manager.client.send_message("/progress", [query_id, status])
-
-        result = job.result()
-        self.osc_manager.client.send_message("/progress", [query_id, "STATUS.FINISHED"])
-        timer.tock(f"predict-{query_id}")
-        timer.tick(f"postprocess-{query_id}")
-        audio_files = list(result[:self.batch_size])
-        # if each file is missing a .wav at the end, add it 
-        first_audio = audio_files[0]
-        if not first_audio.endswith(".wav"):
-            for audio_file in set(audio_files):
-                if not audio_file.endswith(".wav"):
-                    shutil.move(audio_file, f"{audio_file}.wav")
-            audio_files = [f"{audio}.wav" for audio in audio_files]
-        
-        for audio_file in audio_files:
-            # load the file, add the cut samples back
-            sig = at.AudioSignal(audio_file)
-            sig = sig.to_mono()
-            sig.samples = torch.cat([sig.samples, cut_wav], dim=-1)
-            sig.write(audio_file)
-        seed = result[-1]
-        timer.tock(f"postprocess-{query_id}")
-
-        # send a message that the process is done
-        self.osc_manager.log(f"query {query_id} has been processed")
-        self.osc_manager.client.send_message("/process-result", [query_id] + audio_files)
-
-        # schedule to delete the file after 30 seconds
-        time.sleep(30)
-        print("deleting files from query", query_id)
-        for audio_file in audio_files:
-            clear_file(audio_file)
-        # import shutil
-        print(str(Path(audio_files[0]).parent))
-        shutil.rmtree(str(Path(audio_files[0]).parent))
-
-
+     
 def gradio_main(
-    s2s_url: str = None, 
-    s2s_url_2: str = None,
-    s2s_vim_url: str = None,
     vampnet_url: str = None
 ):
-    s2s_urls = [s2s_url, s2s_url_2]
     system = GradioOSCClient(
         vampnet_url=vampnet_url,
-        s2s_urls=s2s_urls,
-        s2s_vim_url=s2s_vim_url,
         ip="127.0.0.1", s_port=8003, r_port=8001,
     )
 

vampnet/interface.py

@@ -568,7 +568,7 @@ class Interface(torch.nn.Module):
         # in subplots, plot z[0] and the mask
         # set title to "codes" and "mask"
         fig.add_subplot(2, 1, 1)
-        plt.imshow(z[0].cpu().numpy(), aspect='auto', origin='lower', cmap="tab20")
+        plt.imshow(z[0].cpu().numpy(), aspect='auto', origin='lower', cmap="tab20", interpolation='none')
         plt.title("codes")
         plt.ylabel("codebook index")
         # set the xticks to seconds