Spaces:
Runtime error
Runtime error
File size: 2,912 Bytes
d347764 7f18a5d f2eb61e 9e55989 d347764 fd6ca3f d347764 fd6ca3f d347764 9e55989 d347764 fd6ca3f d347764 9e55989 f2eb61e d347764 fd6ca3f d347764 f805e49 b5787bb f2eb61e f805e49 c737803 d347764 96018e7 d347764 f805e49 d347764 c737803 96018e7 c737803 3946ba6 c737803 d347764 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 |
import gradio as gr
import numpy as np
import torch
from datasets import load_dataset
from transformers import pipeline
from transformers import BarkModel, BarkProcessor
from transformers import Speech2TextProcessor, Speech2TextForConditionalGeneration
SAMPLE_RATE = 16000
device = "cuda:0" if torch.cuda.is_available() else "cpu"
# asr_model = Speech2TextForConditionalGeneration.from_pretrained("facebook/s2t-medium-mustc-multilingual-st")
# asr_processor = Speech2TextProcessor.from_pretrained("facebook/s2t-medium-mustc-multilingual-st")
asr_pipe = pipeline("automatic-speech-recognition", model="openai/whisper-base", device=device)
bark_model = BarkModel.from_pretrained("suno/bark")
bark_processor = BarkProcessor.from_pretrained("suno/bark")
def translate(audio):
# inputs = asr_processor(audio, sampling_rate=16000, return_tensors="pt")
# generated_ids = asr_model.generate(inputs["input_features"],attention_mask=inputs["attention_mask"],
# forced_bos_token_id=asr_processor.tokenizer.lang_code_to_id["it"],)
# translation = asr_processor.batch_decode(generated_ids, skip_special_tokens=True)
translation = asr_pipe(audio, max_new_tokens=256, generate_kwargs={"task": "transcribe", "language": "it"})
return translation["text"]
def synthesise(text):
inputs = bark_processor(text=text, voice_preset="v2/it_speaker_4",return_tensors="pt")
speech = bark_model.generate(**inputs, do_sample=True)
return speech
def speech_to_speech_translation(audio):
translated_text = translate(audio)
synthesised_speech = synthesise(translated_text)
synthesised_speech = (synthesised_speech.numpy() * 32767).astype(np.int16)
return SAMPLE_RATE, synthesised_speech
title = "Cascaded STST"
description = """
Demo for cascaded speech-to-speech translation (STST), mapping from source speech in any language to target speech in Italian. Demo uses Meta's [Speech2Text](https://huggingface.co/facebook/s2t-medium-mustc-multilingual-st) model for speech translation, and Suno's
[Bark](https://huggingface.co/suno/bark) model for text-to-speech:
"""
demo = gr.Blocks()
mic_translate = gr.Interface(
fn=speech_to_speech_translation,
inputs=gr.Audio(source="microphone", type="filepath"),
outputs=gr.Audio(label="Generated Speech", type="numpy"),
title=title,
description=description,
)
file_translate = gr.Interface(
fn=speech_to_speech_translation,
inputs=gr.Audio(source="upload", type="filepath"),
outputs=gr.Audio(label="Generated Speech", type="numpy"),
examples=[["./example.wav"]],
title=title,
description=description,
)
with demo:
gr.TabbedInterface([mic_translate, file_translate], ["Microphone", "Audio File"])
demo.launch()
|