app.py

import gradio as gr
import torch
import soundfile as sf
import spaces
import os
from transformers import SpeechT5Processor, SpeechT5ForTextToSpeech, SpeechT5HifiGan
from speechbrain.pretrained import EncoderClassifier

device = "cuda" if torch.cuda.is_available() else "cpu"

def load_models_and_data():
    model_name = "microsoft/speecht5_tts"
    processor = SpeechT5Processor.from_pretrained(model_name)
    model = SpeechT5ForTextToSpeech.from_pretrained("emirhanbilgic/speecht5_finetuned_emirhan_tr").to(device)
    vocoder = SpeechT5HifiGan.from_pretrained("microsoft/speecht5_hifigan").to(device)
    
    spk_model_name = "speechbrain/spkrec-xvect-voxceleb"
    speaker_model = EncoderClassifier.from_hparams(
        source=spk_model_name,
        run_opts={"device": device},
        savedir=os.path.join("/tmp", spk_model_name),
    )
    
    return model, processor, vocoder, speaker_model

model, processor, vocoder, speaker_model = load_models_and_data()

def create_speaker_embedding(waveform):
    with torch.no_grad():
        speaker_embeddings = speaker_model.encode_batch(torch.tensor(waveform).unsqueeze(0).to(device))
        speaker_embeddings = torch.nn.functional.normalize(speaker_embeddings, dim=2)
        speaker_embeddings = speaker_embeddings.squeeze().to(device)
    return speaker_embeddings

@spaces.GPU(duration = 60)
def text_to_speech(text, audio_file):
    inputs = processor(text=text, return_tensors="pt").to(device)
    
    # Load the audio file and create speaker embedding
    waveform, sample_rate = sf.read(audio_file)
    if len(waveform.shape) > 1:
        waveform = waveform[:, 0]  # Take the first channel if stereo
    speaker_embeddings = create_speaker_embedding(waveform)
    
    speech = model.generate_speech(inputs["input_ids"], speaker_embeddings, vocoder=vocoder)
    sf.write("output.wav", speech.cpu().numpy(), samplerate=16000)
    return "output.wav"

iface = gr.Interface(
    fn=text_to_speech,
    inputs=[
        gr.Textbox(label="Enter Turkish text to convert to speech"),
        gr.Audio(label="Upload a short audio sample of the target speaker", type="filepath")
    ],
    outputs=gr.Audio(label="Generated Speech"),
    title="Turkish SpeechT5 Text-to-Speech Demo with Custom Voice",
    description="Enter Turkish text, upload a short audio sample of the target speaker, and listen to the generated speech using the fine-tuned SpeechT5 model."
)

iface.launch()