app.py

#==================================================================================
# https://huggingface.co/spaces/asigalov61/Guided-Accompaniment-Transformer
#==================================================================================

print('=' * 70)
print('Guided Accompaniment Transformer Gradio App')

print('=' * 70)
print('Loading core Guided Accompaniment Transformer modules...')

import os

import time as reqtime
import datetime
from pytz import timezone

print('=' * 70)
print('Loading main Guided Accompaniment Transformer modules...')

os.environ['USE_FLASH_ATTENTION'] = '1'

import torch

torch.set_float32_matmul_precision('high')
torch.backends.cuda.matmul.allow_tf32 = True # allow tf32 on matmul
torch.backends.cudnn.allow_tf32 = True # allow tf32 on cudnn
torch.backends.cuda.enable_mem_efficient_sdp(True)
torch.backends.cuda.enable_math_sdp(True)
torch.backends.cuda.enable_flash_sdp(True)
torch.backends.cuda.enable_cudnn_sdp(True)

from huggingface_hub import hf_hub_download

import TMIDIX

from midi_to_colab_audio import midi_to_colab_audio

from x_transformer_1_23_2 import *

import random

print('=' * 70)
print('Loading aux Guided Accompaniment Transformer modules...')

import matplotlib.pyplot as plt

import gradio as gr
import spaces

print('=' * 70)
print('PyTorch version:', torch.__version__)
print('=' * 70)
print('Done!')
print('Enjoy! :)')
print('=' * 70)

#==================================================================================

MODEL_CHECKPOINTS = 'Guided_Accompaniment_Transformer_Trained_Model_59896_steps_0.9055_loss_0.735_acc.pth'

SOUDFONT_PATH = 'SGM-v2.01-YamahaGrand-Guit-Bass-v2.7.sf2'

#==================================================================================

def load_model():

    print('=' * 70)
    print('Instantiating model...')
    
    device_type = 'cuda'
    dtype = 'bfloat16'
    
    ptdtype = {'bfloat16': torch.bfloat16, 'float16': torch.float16}[dtype]
    ctx = torch.amp.autocast(device_type=device_type, dtype=ptdtype)
    
    SEQ_LEN = 2048

    if model_selector == 'with velocity - 3 epochs':
        PAD_IDX = 512

    else:
        PAD_IDX = 384
    
    model = TransformerWrapper(
            num_tokens = PAD_IDX+1,
            max_seq_len = SEQ_LEN,
            attn_layers = Decoder(dim = 2048,
                                  depth = 4,
                                  heads = 32,
                                  rotary_pos_emb = True,
                                  attn_flash = True
                                  )
    )
    
    model = AutoregressiveWrapper(model, ignore_index=PAD_IDX, pad_value=PAD_IDX)
    
    print('=' * 70)
    print('Loading model checkpoint...')      
    
    model_checkpoint = hf_hub_download(repo_id='asigalov61/Guided-Accompaniment-Transformer', filename=MODEL_CHECKPOINTS[model_selector])
    
    model.load_state_dict(torch.load(model_checkpoint, map_location='cpu', weights_only=True))
    
    model = torch.compile(model, mode='max-autotune')
    
    print('=' * 70)
    print('Done!')
    print('=' * 70)
    print('Model will use', dtype, 'precision...')
    print('=' * 70)

    return [model, ctx]

#==================================================================================

def load_midi(input_midi, model_selector=''):

    raw_score = TMIDIX.midi2single_track_ms_score(input_midi.name)

    escore_notes = TMIDIX.advanced_score_processor(raw_score, return_enhanced_score_notes=True)[0]
    escore_notes = TMIDIX.augment_enhanced_score_notes(escore_notes, timings_divider=32)
    
    sp_escore_notes = TMIDIX.solo_piano_escore_notes(escore_notes, keep_drums=False)
    zscore = TMIDIX.recalculate_score_timings(sp_escore_notes)
    
    cscore = TMIDIX.chordify_score([1000, zscore])
    
    score = []
    
    pc = cscore[0]
    
    for c in cscore:
        score.append(max(0, min(127, c[0][1]-pc[0][1])))
    
        for n in c:
            if model_selector == 'with velocity - 3 epochs':
                score.extend([max(1, min(127, n[2]))+128, max(1, min(127, n[4]))+256, max(1, min(127, n[5]))+384])

            else:
                score.extend([max(1, min(127, n[2]))+128, max(1, min(127, n[4]))+256])
    
        pc = c

    return score

#==================================================================================

def save_midi(tokens, batch_number=None, model_selector=''):

    song = tokens
    song_f = []
    
    time = 0
    dur = 0
    vel = 90
    pitch = 0
    channel = 0
    patch = 0

    patches = [0] * 16

    for m in song:

        if 0 <= m < 128:
            time += m * 32

        elif 128 < m < 256:
            dur = (m-128) * 32

        elif 256 < m < 384:
            pitch = (m-256)

            if model_selector == 'without velocity - 3 epochs' or model_selector == 'without velocity - 7 epochs':
                song_f.append(['note', time, dur, 0, pitch, max(40, pitch), 0])

        elif 384 < m < 512:
            vel = (m-384)

            if model_selector == 'with velocity - 3 epochs':
                song_f.append(['note', time, dur, 0, pitch, vel, 0])

    if batch_number == None:
        fname = 'Guided-Accompaniment-Transformer-Music-Composition'
        
    else:
        fname = 'Guided-Accompaniment-Transformer-Music-Composition_'+str(batch_number)
    
    data = TMIDIX.Tegridy_ms_SONG_to_MIDI_Converter(song_f,
                                                  output_signature = 'Guided Accompaniment Transformer',
                                                  output_file_name = fname,
                                                  track_name='Project Los Angeles',
                                                  list_of_MIDI_patches=patches,
                                                  verbose=False
                                                  )

    return song_f

#==================================================================================

@spaces.GPU
def Generate_Accompaniment(input_midi, 
                           num_gen_tokens,
                           model_temperature
                          ):

    #===============================================================================
    
    print('=' * 70)
    print('Req start time: {:%Y-%m-%d %H:%M:%S}'.format(datetime.datetime.now(PDT)))
    start_time = reqtime.time()
    print('=' * 70)

    fn = os.path.basename(input_midi)
    fn1 = fn.split('.')[0]

    print('=' * 70)
    print('Requested settings:')
    print('=' * 70)
    print('Input MIDI file name:', fn)
    print('Input MIDI type:', input_midi_type)
    print('Conversion type:', input_conv_type)
    print('Number of prime notes:', input_number_prime_notes)
    print('Number of notes to convert:', input_number_conv_notes)
    print('Model durations sampling top value:', input_model_dur_top_k)
    print('Model durations temperature:', input_model_dur_temperature)
    print('Model velocities temperature:', input_model_vel_temperature)
    
    print('=' * 70)

    #==================================================================

    src_melody_chords_f = load_midi(input_midi.name)

    #==================================================================
    
    print('Sample output events', src_melody_chords_f[0][1][:3])
    print('=' * 70)
    print('Generating...')

    model.to(DEVICE)
    model.eval()

    #==================================================================
    
    print('=' * 70)
    print('Done!')
    print('=' * 70)
    
    #===============================================================================
    
    print('Rendering results...')
    
    print('=' * 70)
    print('Sample INTs', final_song[:15])
    print('=' * 70)

    song_f = []
    
    if len(final_song) != 0:
    
        time = 0
        dur = 0
        vel = 90
        pitch = 60
        channel = 0
        patch = 0
        
        patches = [0] * 16
        
        for ss in final_song:
        
            if 0 <= ss < 256:
        
                time += ss * 16
        
            if 256 <= ss < 384:
        
                pitch = ss-256
        
            if 384 <= ss < 640:
        
                dur = (ss-384) * 16
        
            if 640 <= ss < 768:
                
                vel = (ss-640)
            
                song_f.append(['note', time, dur, channel, pitch, vel, patch])

    fn1 = "Score-2-Performance-Transformer-Composition"
    
    detailed_stats = TMIDIX.Tegridy_ms_SONG_to_MIDI_Converter(song_f,
                                                              output_signature = 'Score 2 Performance Transformer',
                                                              output_file_name = fn1,
                                                              track_name='Project Los Angeles',
                                                              list_of_MIDI_patches=patches
                                                              )
    
    new_fn = fn1+'.mid'
            
    
    audio = midi_to_colab_audio(new_fn, 
                        soundfont_path=soundfont,
                        sample_rate=16000,
                        volume_scale=10,
                        output_for_gradio=True
                        )
    
    print('Done!')
    print('=' * 70)

    #========================================================

    output_midi_title = str(fn1)
    output_midi_summary = str(song_f[:3])
    output_midi = str(new_fn)
    output_audio = (16000, audio)
    
    output_plot = TMIDIX.plot_ms_SONG(song_f, plot_title=output_midi, return_plt=True)

    print('Output MIDI file name:', output_midi)
    print('Output MIDI title:', output_midi_title)
    print('Output MIDI summary:', output_midi_summary)
    print('=' * 70) 
    

    #========================================================
    
    print('-' * 70)
    print('Req end time: {:%Y-%m-%d %H:%M:%S}'.format(datetime.datetime.now(PDT)))
    print('-' * 70)
    print('Req execution time:', (reqtime.time() - start_time), 'sec')

    return output_midi, output_audio, output_plot
    
#==================================================================================

PDT = timezone('US/Pacific')

print('=' * 70)
print('App start time: {:%Y-%m-%d %H:%M:%S}'.format(datetime.datetime.now(PDT)))
print('=' * 70)

#==================================================================================

with gr.Blocks() as demo:

    #==================================================================================

    gr.Markdown("<h1 style='text-align: center; margin-bottom: 1rem'>Guided Accompaniment Transformer</h1>")
    gr.Markdown("<h1 style='text-align: center; margin-bottom: 1rem'>Guided melody accompaniment generation with transformers</h1>")
    gr.HTML("""
            Check out <a href="https://github.com/asigalov61/monsterpianotransformer">Guided Accompaniment Transformer</a> on GitHub or on
            
            <p>
                <a href="https://pypi.org/project/monsterpianotransformer/">
                    <img src="https://upload.wikimedia.org/wikipedia/commons/6/64/PyPI_logo.svg" alt="PyPI Project" style="width: 100px; height: auto;">
                </a> or 
                <a href="https://huggingface.co/spaces/asigalov61/Guided-Accompaniment-Transformer?duplicate=true">
                    <img src="https://huggingface.co/datasets/huggingface/badges/resolve/main/duplicate-this-space-md.svg" alt="Duplicate in Hugging Face">
                </a>
            </p>
            
            for faster execution and endless generation!
            """)
    
    #==================================================================================
    
    gr.Markdown("## Upload seed MIDI or click 'Generate' button for random output")
    
    input_midi = gr.File(label="Input MIDI", file_types=[".midi", ".mid", ".kar"])
    
    gr.Markdown("## Generate")
    
    num_gen_tokens = gr.Slider(15, 1024, value=1024, step=1, label="Number of tokens to generate")
    model_temperature = gr.Slider(0.1, 1, value=0.9, step=0.01, label="Model temperature")
    
    generate_btn = gr.Button("Generate", variant="primary")

    gr.Markdown("## Results")
    
    output_audio = gr.Audio(label="Output MIDI audio", format="wav", elem_id="midi_audio")
    output_plot = gr.Plot(label="Output MIDI score plot")
    output_midi = gr.File(label="Output MIDI file", file_types=[".mid"])
            
    outputs.extend([model_state])

    generate_btn.click(Generate_Accompaniment, 
                       [input_midi, 
                        num_gen_tokens,
                        model_temperature
                       ], 
                       [
                        output_audio,
                        output_plot,
                        output_midi,                           
                       ]
                      )

    '''gr.Examples(
        [["asap_midi_score_21.mid", "Score", "Durations and Velocities", 8, 600, 1, 1.1, 1.5],
         ["asap_midi_score_45.mid", "Score", "Durations and Velocities", 8, 600, 1, 1.1, 1.5],
         ["asap_midi_score_69.mid", "Score", "Durations and Velocities", 8, 600, 1, 1.1, 1.5],
         ["asap_midi_score_118.mid", "Score", "Durations and Velocities", 8, 600, 1, 1.1, 1.5],
         ["asap_midi_score_167.mid", "Score", "Durations and Velocities", 8, 600, 1, 1.1, 1.5],
        ],
        [input_midi,
         input_midi_type,
         input_conv_type,
         input_number_prime_notes,
         input_number_conv_notes,
         input_model_dur_top_k,
         input_model_dur_temperature,
         input_model_vel_temperature
        ],
        [output_midi_title, output_midi_summary, output_midi, output_audio, output_plot],
        Convert_Score_to_Performance
    )'''
        
#==================================================================================

demo.launch()

#==================================================================================