|
|
|
|
|
|
|
|
|
print('=' * 70) |
|
print('Guided Accompaniment Transformer Gradio App') |
|
|
|
print('=' * 70) |
|
print('Loading core Guided Accompaniment Transformer modules...') |
|
|
|
import os |
|
import copy |
|
|
|
import time as reqtime |
|
import datetime |
|
from pytz import timezone |
|
|
|
print('=' * 70) |
|
print('Loading main Guided Accompaniment Transformer modules...') |
|
|
|
os.environ['USE_FLASH_ATTENTION'] = '1' |
|
|
|
import torch |
|
|
|
torch.set_float32_matmul_precision('medium') |
|
torch.backends.cuda.matmul.allow_tf32 = True |
|
torch.backends.cudnn.allow_tf32 = True |
|
torch.backends.cuda.enable_mem_efficient_sdp(True) |
|
torch.backends.cuda.enable_math_sdp(True) |
|
torch.backends.cuda.enable_flash_sdp(True) |
|
torch.backends.cuda.enable_cudnn_sdp(True) |
|
|
|
from huggingface_hub import hf_hub_download |
|
|
|
import TMIDIX |
|
|
|
from midi_to_colab_audio import midi_to_colab_audio |
|
|
|
from x_transformer_1_23_2 import * |
|
|
|
import random |
|
|
|
import tqdm |
|
|
|
print('=' * 70) |
|
print('Loading aux Guided Accompaniment Transformer modules...') |
|
|
|
import matplotlib.pyplot as plt |
|
|
|
import gradio as gr |
|
import spaces |
|
|
|
print('=' * 70) |
|
print('PyTorch version:', torch.__version__) |
|
print('=' * 70) |
|
print('Done!') |
|
print('Enjoy! :)') |
|
print('=' * 70) |
|
|
|
|
|
|
|
MODEL_CHECKPOINT = 'Guided_Accompaniment_Transformer_Trained_Model_36457_steps_0.5384_loss_0.8417_acc.pth' |
|
|
|
SOUDFONT_PATH = 'SGM-v2.01-YamahaGrand-Guit-Bass-v2.7.sf2' |
|
|
|
MAX_MELODY_NOTES = 64 |
|
|
|
MAX_GEN_TOKS = 3072 |
|
|
|
|
|
|
|
print('=' * 70) |
|
print('Loading popular hook melodies dataset...') |
|
|
|
popular_hook_melodies_pickle = hf_hub_download(repo_id='asigalov61/Guided-Accompaniment-Transformer', |
|
filename='popular_hook_melodies_24_64_CC_BY_NC_SA.pickle' |
|
) |
|
|
|
popular_hook_melodies = TMIDIX.Tegridy_Any_Pickle_File_Reader(popular_hook_melodies_pickle) |
|
|
|
print('=' * 70) |
|
print('Done!') |
|
print('=' * 70) |
|
|
|
|
|
|
|
print('=' * 70) |
|
print('Instantiating model...') |
|
|
|
device_type = 'cuda' |
|
dtype = 'bfloat16' |
|
|
|
ptdtype = {'bfloat16': torch.bfloat16, 'float16': torch.float16}[dtype] |
|
ctx = torch.amp.autocast(device_type=device_type, dtype=ptdtype) |
|
|
|
SEQ_LEN = 4096 |
|
PAD_IDX = 1794 |
|
|
|
model = TransformerWrapper( |
|
num_tokens = PAD_IDX+1, |
|
max_seq_len = SEQ_LEN, |
|
attn_layers = Decoder(dim = 2048, |
|
depth = 4, |
|
heads = 32, |
|
rotary_pos_emb = True, |
|
attn_flash = True |
|
) |
|
) |
|
|
|
model = AutoregressiveWrapper(model, ignore_index=PAD_IDX, pad_value=PAD_IDX) |
|
|
|
print('=' * 70) |
|
print('Loading model checkpoint...') |
|
|
|
model_checkpoint = hf_hub_download(repo_id='asigalov61/Guided-Accompaniment-Transformer', filename=MODEL_CHECKPOINT) |
|
|
|
model.load_state_dict(torch.load(model_checkpoint, map_location='cpu', weights_only=True)) |
|
|
|
model = torch.compile(model, mode='max-autotune') |
|
|
|
print('=' * 70) |
|
print('Done!') |
|
print('=' * 70) |
|
print('Model will use', dtype, 'precision...') |
|
print('=' * 70) |
|
|
|
|
|
|
|
def load_midi(input_midi, melody_patch=-1, use_nth_note=1): |
|
|
|
raw_score = TMIDIX.midi2single_track_ms_score(input_midi) |
|
|
|
escore_notes = TMIDIX.advanced_score_processor(raw_score, return_enhanced_score_notes=True)[0] |
|
escore_notes = TMIDIX.augment_enhanced_score_notes(escore_notes, timings_divider=32) |
|
|
|
sp_escore_notes = TMIDIX.solo_piano_escore_notes(escore_notes, keep_drums=False) |
|
|
|
if melody_patch == -1: |
|
zscore = TMIDIX.recalculate_score_timings(sp_escore_notes) |
|
|
|
else: |
|
mel_score = [e for e in sp_escore_notes if e[6] == melody_patch] |
|
|
|
if mel_score: |
|
zscore = TMIDIX.recalculate_score_timings(mel_score) |
|
|
|
else: |
|
zscore = TMIDIX.recalculate_score_timings(sp_escore_notes) |
|
|
|
cscore = TMIDIX.chordify_score([1000, zscore])[:MAX_MELODY_NOTES:use_nth_note] |
|
|
|
score = [] |
|
|
|
score_list = [] |
|
|
|
pc = cscore[0] |
|
|
|
for c in cscore: |
|
score.append(max(0, min(127, c[0][1]-pc[0][1]))) |
|
|
|
scl = [[max(0, min(127, c[0][1]-pc[0][1]))]] |
|
|
|
n = c[0] |
|
|
|
score.extend([max(1, min(127, n[2]))+128, max(1, min(127, n[4]))+256]) |
|
scl.append([max(1, min(127, n[2]))+128, max(1, min(127, n[4]))+256]) |
|
|
|
score_list.append(scl) |
|
|
|
pc = c |
|
|
|
score_list.append(scl) |
|
|
|
return score, score_list |
|
|
|
|
|
|
|
@spaces.GPU |
|
def Generate_Accompaniment(input_midi, |
|
input_melody, |
|
melody_patch, |
|
use_nth_note, |
|
model_temperature, |
|
model_sampling_top_k |
|
): |
|
|
|
|
|
|
|
def generate_full_seq(input_seq, |
|
max_toks=3072, |
|
temperature=0.9, |
|
top_k_value=15, |
|
verbose=True |
|
): |
|
|
|
seq_abs_run_time = sum([t for t in input_seq if t < 128]) |
|
|
|
cur_time = 0 |
|
|
|
full_seq = copy.deepcopy(input_seq) |
|
|
|
toks_counter = 0 |
|
|
|
while cur_time <= seq_abs_run_time+32: |
|
|
|
if verbose: |
|
if toks_counter % 128 == 0: |
|
print('Generated', toks_counter, 'tokens') |
|
|
|
x = torch.LongTensor(full_seq).cuda() |
|
|
|
with ctx: |
|
out = model.generate(x, |
|
1, |
|
filter_logits_fn=top_k, |
|
filter_kwargs={'k': top_k_value}, |
|
temperature=temperature, |
|
return_prime=False, |
|
verbose=False) |
|
|
|
y = out.tolist()[0][0] |
|
|
|
if y < 128: |
|
cur_time += y |
|
|
|
full_seq.append(y) |
|
|
|
toks_counter += 1 |
|
|
|
if toks_counter == max_toks: |
|
return full_seq |
|
|
|
return full_seq |
|
|
|
|
|
|
|
print('=' * 70) |
|
print('Req start time: {:%Y-%m-%d %H:%M:%S}'.format(datetime.datetime.now(PDT))) |
|
start_time = reqtime.time() |
|
print('=' * 70) |
|
|
|
print('=' * 70) |
|
print('Requested settings:') |
|
print('=' * 70) |
|
if input_midi: |
|
fn = os.path.basename(input_midi) |
|
fn1 = fn.split('.')[0] |
|
print('Input MIDI file name:', fn) |
|
|
|
else: |
|
print('Input sample melody:', input_melody) |
|
print('Source melody patch:', melody_patch) |
|
print('Use nth melody note:', use_nth_note) |
|
print('Model temperature:', model_temperature) |
|
print('Model top k:', model_sampling_top_k) |
|
|
|
print('=' * 70) |
|
|
|
|
|
|
|
print('Prepping melody...') |
|
|
|
if input_midi: |
|
inp_mel = 'Custom MIDI' |
|
score, score_list = load_midi(input_midi.name, melody_patch, use_nth_note) |
|
|
|
else: |
|
mel_list = [m[0].lower() for m in popular_hook_melodies] |
|
|
|
inp_mel = random.choice(mel_list).title() |
|
|
|
for m in mel_list: |
|
if input_melody.lower().strip() in m: |
|
inp_mel = m.title() |
|
break |
|
|
|
score = popular_hook_melodies[[m[0] for m in popular_hook_melodies].index(inp_mel)][1] |
|
score_list = [[[score[i]], score[i+1:i+3]] for i in range(0, len(score)-3, 3)] |
|
|
|
print('Selected melody:', inp_mel) |
|
|
|
print('Sample score events', score[:12]) |
|
|
|
|
|
|
|
print('=' * 70) |
|
print('Generating...') |
|
|
|
model.to(device_type) |
|
model.eval() |
|
|
|
|
|
|
|
start_score_seq = [1792] + score + [1793] |
|
|
|
|
|
|
|
input_seq = generate_full_seq(start_score_seq, |
|
max_toks=MAX_GEN_TOKS, |
|
temperature=model_temperature, |
|
top_k_value=model_sampling_top_k, |
|
) |
|
|
|
final_song = input_seq[len(start_score_seq):] |
|
|
|
print('=' * 70) |
|
print('Done!') |
|
print('=' * 70) |
|
|
|
|
|
|
|
print('Rendering results...') |
|
|
|
print('=' * 70) |
|
print('Sample INTs', final_song[:15]) |
|
print('=' * 70) |
|
|
|
song_f = [] |
|
|
|
if len(final_song) != 0: |
|
|
|
time = 0 |
|
dur = 0 |
|
vel = 90 |
|
pitch = 0 |
|
channel = 0 |
|
patch = 0 |
|
|
|
channels_map = [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 9, 12, 13, 14, 15] |
|
patches_map = [40, 0, 10, 19, 24, 35, 40, 52, 56, 9, 65, 73, 0, 0, 0, 0] |
|
velocities_map = [125, 80, 100, 80, 90, 100, 100, 80, 110, 110, 110, 110, 80, 80, 80, 80] |
|
|
|
for m in final_song: |
|
|
|
if 0 <= m < 128: |
|
time += m * 32 |
|
|
|
elif 128 < m < 256: |
|
dur = (m-128) * 32 |
|
|
|
elif 256 < m < 1792: |
|
cha = (m-256) // 128 |
|
pitch = (m-256) % 128 |
|
|
|
channel = channels_map[cha] |
|
patch = patches_map[channel] |
|
vel = velocities_map[channel] |
|
|
|
song_f.append(['note', time, dur, channel, pitch, vel, patch]) |
|
|
|
fn1 = "Guided-Accompaniment-Transformer-Composition" |
|
|
|
detailed_stats = TMIDIX.Tegridy_ms_SONG_to_MIDI_Converter(song_f, |
|
output_signature = 'Guided Accompaniment Transformer', |
|
output_file_name = fn1, |
|
track_name='Project Los Angeles', |
|
list_of_MIDI_patches=patches_map |
|
) |
|
|
|
new_fn = fn1+'.mid' |
|
|
|
|
|
audio = midi_to_colab_audio(new_fn, |
|
soundfont_path=SOUDFONT_PATH, |
|
sample_rate=16000, |
|
volume_scale=10, |
|
output_for_gradio=True |
|
) |
|
|
|
print('Done!') |
|
print('=' * 70) |
|
|
|
|
|
|
|
output_title = str(inp_mel) |
|
output_midi = str(new_fn) |
|
output_audio = (16000, audio) |
|
|
|
output_plot = TMIDIX.plot_ms_SONG(song_f, plot_title=output_midi, return_plt=True) |
|
|
|
print('Output MIDI file name:', output_midi) |
|
print('Output MIDI melody title:', output_title) |
|
print('=' * 70) |
|
|
|
|
|
|
|
print('-' * 70) |
|
print('Req end time: {:%Y-%m-%d %H:%M:%S}'.format(datetime.datetime.now(PDT))) |
|
print('-' * 70) |
|
print('Req execution time:', (reqtime.time() - start_time), 'sec') |
|
|
|
return output_title, output_audio, output_plot, output_midi |
|
|
|
|
|
|
|
PDT = timezone('US/Pacific') |
|
|
|
print('=' * 70) |
|
print('App start time: {:%Y-%m-%d %H:%M:%S}'.format(datetime.datetime.now(PDT))) |
|
print('=' * 70) |
|
|
|
|
|
|
|
with gr.Blocks() as demo: |
|
|
|
|
|
|
|
gr.Markdown("<h1 style='text-align: center; margin-bottom: 1rem'>Guided Accompaniment Transformer</h1>") |
|
gr.Markdown("<h1 style='text-align: center; margin-bottom: 1rem'>Guided melody accompaniment generation with transformers</h1>") |
|
gr.HTML(""" |
|
<p> |
|
<a href="https://huggingface.co/spaces/asigalov61/Guided-Accompaniment-Transformer?duplicate=true"> |
|
<img src="https://huggingface.co/datasets/huggingface/badges/resolve/main/duplicate-this-space-md.svg" alt="Duplicate in Hugging Face"> |
|
</a> |
|
</p> |
|
|
|
for faster execution and endless generation! |
|
""") |
|
|
|
|
|
|
|
gr.Markdown("## Upload source melody MIDI or enter a search query for a sample melody below") |
|
|
|
input_midi = gr.File(label="Input MIDI", |
|
file_types=[".midi", ".mid", ".kar"] |
|
) |
|
|
|
input_melody = gr.Textbox(value="Hotel California", |
|
label="Popular melodies database search query", |
|
info='If the query is not found, random melody will be selected. Custom MIDI overrides search query' |
|
) |
|
|
|
gr.Markdown("## Generation options") |
|
|
|
melody_patch = gr.Slider(-1, 127, value=-1, step=1, label="Source melody MIDI patch") |
|
use_nth_note = gr.Slider(1, 8, value=1, step=1, label="Use each nth melody note") |
|
model_temperature = gr.Slider(0.1, 1, value=0.9, step=0.01, label="Model temperature") |
|
model_sampling_top_k = gr.Slider(1, 100, value=15, step=1, label="Model sampling top k value") |
|
|
|
generate_btn = gr.Button("Generate", variant="primary") |
|
|
|
gr.Markdown("## Generation results") |
|
|
|
output_title = gr.Textbox(label="MIDI melody title") |
|
output_audio = gr.Audio(label="MIDI audio", format="wav", elem_id="midi_audio") |
|
output_plot = gr.Plot(label="MIDI score plot") |
|
output_midi = gr.File(label="MIDI file", file_types=[".mid"]) |
|
|
|
generate_btn.click(Generate_Accompaniment, |
|
[input_midi, |
|
input_melody, |
|
melody_patch, |
|
use_nth_note, |
|
model_temperature, |
|
model_sampling_top_k |
|
], |
|
[output_title, |
|
output_audio, |
|
output_plot, |
|
output_midi |
|
] |
|
) |
|
|
|
gr.Examples( |
|
[["USSR-National-Anthem-Seed-Melody.mid", "Custom MIDI", -1, 1, 0.9, 15], |
|
["Sparks-Fly-Seed-Melody.mid", "Custom MIDI", -1, 1, 0.9, 15] |
|
], |
|
[input_midi, |
|
input_melody, |
|
melody_patch, |
|
use_nth_note, |
|
model_temperature, |
|
model_sampling_top_k |
|
], |
|
[output_title, |
|
output_audio, |
|
output_plot, |
|
output_midi |
|
], |
|
Generate_Accompaniment |
|
) |
|
|
|
|
|
|
|
demo.launch() |
|
|
|
|
