Update app.py

app.py

@@ -6,6 +6,8 @@ from tempfile import _TemporaryFileWrapper
 import torch
 import torchaudio
 
+SAMPLE_RATE=16000
+
 if torch.cuda.is_available():
     device = "cuda"
     torch_dtype = torch.float16
@@ -21,10 +23,12 @@ pipe = pipe.to(device)
 generator = torch.Generator(device)
 
 
-def predict(midi_file=None, prompt="", negative_prompt="", audio_length_in_s=5, random_seed=0, controlnet_conditioning_scale=1, num_inference_steps=20, guidance_scale=2.5, guess_mode=False):
+def predict(midi_file=None, prompt="", negative_prompt="", audio_length_in_s=10, random_seed=0, controlnet_conditioning_scale=1, num_inference_steps=20, guidance_scale=2.5, guess_mode=False):
     if isinstance(midi_file, _TemporaryFileWrapper):
         midi_file = midi_file.name
     midi = PrettyMIDI(midi_file)
+    midi_synth = midi.synthesize(fs=SAMPLE_RATE)[:int(SAMPLE_RATE*audio_length_in_s)]
+    midi_synth = midi_synth.reshape(midi_synth.shape[0], 1)
     audio = pipe(
         prompt,
         negative_prompt=negative_prompt,
@@ -36,7 +40,8 @@ def predict(midi_file=None, prompt="", negative_prompt="", audio_length_in_s=5,
         generator=generator.manual_seed(int(random_seed)),
         guidance_scale=float(guidance_scale),
     )
-    return (16000, audio.audios.T)
+    return (SAMPLE_RATE, midi_synth), (SAMPLE_RATE, audio.audios.T)
+    
 
 with gr.Blocks(title="🎹 MIDI-AudioLDM", theme=gr.themes.Base(text_size=gr.themes.sizes.text_md, font=[gr.themes.GoogleFont("Nunito Sans")])) as demo:
     gr.HTML(
@@ -52,7 +57,8 @@ with gr.Blocks(title="🎹 MIDI-AudioLDM", theme=gr.themes.Base(text_size=gr.the
             midi = gr.File(label="midi file", file_types=[".mid"])
             prompt = gr.Textbox(label="prompt", info="Enter a descriptive text prompt to guide the audio generation.")
         with gr.Column(variant='panel'):
-            audio = gr.Audio(label="audio")
+            synth = gr.Audio(label="synthesized audio")
+            audio = gr.Audio(label="generated audio")
     with gr.Accordion("Advanced Settings", open=False):
         duration = gr.Slider(0, 30, value=10, step=2.5, label="duration", info="Modify the duration in seconds of the output audio file.")
         inf = gr.Slider(0, 50, value=20, step=1, label="inference steps", info="Edit the number of denoising steps. A larger number usually leads to higher quality but slower results.")
@@ -62,7 +68,7 @@ with gr.Blocks(title="🎹 MIDI-AudioLDM", theme=gr.themes.Base(text_size=gr.the
         cond = gr.Slider(0.0, 1.0, value=1.0, step=0.1, label="conditioning scale", info="Choose a value between 0 and 1. The larger the more it will take the conditioning into account. Lower values are recommended for more creative prompts.")
         guess = gr.Checkbox(label="guess mode", info="Optionally select guess mode. If so, the model will try to recognize the content of the MIDI without the need of a text prompt.")
     btn = gr.Button("Generate")
-    btn.click(predict, inputs=[midi, prompt, neg_prompt, duration, seed, cond, inf, guidance_scale, guess], outputs=[audio])
+    btn.click(predict, inputs=[midi, prompt, neg_prompt, duration, seed, cond, inf, guidance_scale, guess], outputs=[synth, audio])
     gr.Examples(examples=[["S00.mid", "piano", "", 10, 25, 1.0, 20, 2.5, False], ["S00.mid", "violin", "", 10, 25, 1.0, 20, 2.5, False], ["S00.mid", "woman singing, studio recording", "noise", 10, 25, 1.0, 20, 2.5, False], ["S00.mid", "jazz band, clean", "noise", 10, 25, 0.8, 20, 2.5, False], ["S00.mid", "choir", "noise, percussion", 10, 25, 0.7, 20, 2.5, False]], inputs=[midi, prompt, neg_prompt, duration, seed, cond, inf, guidance_scale, guess], fn=predict, outputs=audio, cache_examples=True)
     
 demo.launch()