Update app.py

app.py

@@ -2,59 +2,31 @@ import gradio as gr
 import numpy as np
 import torch
 from datasets import load_dataset
+
 from transformers import SpeechT5ForTextToSpeech, SpeechT5HifiGan, SpeechT5Processor, pipeline
 
 
 device = "cuda:0" if torch.cuda.is_available() else "cpu"
 
 # load speech translation checkpoint
-asr_pipe = pipeline("automatic-speech-recognition", model="openai/whisper-large-v2", device=device)
+asr_pipe = pipeline("automatic-speech-recognition", model="openai/whisper-base", device=device)
 
 # load text-to-speech checkpoint and speaker embeddings
-model_id = "jjyaoao/speecht5_voxpopuli_spanish"  # update with your model id
-# pipe = pipeline("automatic-speech-recognition", model=model_id)
-model = SpeechT5ForTextToSpeech.from_pretrained(model_id)
-vocoder = SpeechT5HifiGan.from_pretrained("microsoft/speecht5_hifigan")
+processor = SpeechT5Processor.from_pretrained("jjyaoao/speecht5_voxpopuli_spanish")
+
+model = SpeechT5ForTextToSpeech.from_pretrained("jjyaoao/speecht5_voxpopuli_spanish").to(device)
+vocoder = SpeechT5HifiGan.from_pretrained("microsoft/speecht5_hifigan").to(device)
+
 embeddings_dataset = load_dataset("Matthijs/cmu-arctic-xvectors", split="validation")
-speaker_embeddings = torch.tensor(embeddings_dataset[7440]["xvector"]).unsqueeze(0)
-
-processor = SpeechT5Processor.from_pretrained(model_id)
-
-replacements = [
-     ("á", "a"),
-     ("ç", "c"),
-     ("è", "e"),
-     ("ì", "i"),
-     ("í", "i"),
-     ("ò", "o"),
-     ("ó", "o"),
-     ("ù", "u"),
-     ("ú", "u"),
-     ("š", "s"),
-     ("ï", "i"),
-     ("ñ", "n"),
-     ("ü", "u"),
-]
-
-def cleanup_text(text):
-    for src, dst in replacements:
-        text = text.replace(src, dst)
-    return text
-
-def synthesize_speech(text):
-    text = cleanup_text(text)
-    inputs = processor(text=text, return_tensors="pt")
-    speech = model.generate_speech(inputs["input_ids"].to(device), speaker_embeddings.to(device), vocoder=vocoder)
+speaker_embeddings = torch.tensor(embeddings_dataset[7306]["xvector"]).unsqueeze(0)
 
-    return gr.Audio.update(value=(16000, speech.cpu().numpy()))
 
 def translate(audio):
-    outputs = asr_pipe(audio, max_new_tokens=256, generate_kwargs={"task": "transcribe", "language": "spanish"})
+    outputs = asr_pipe(audio, max_new_tokens=256, generate_kwargs={"task": "translate"})
     return outputs["text"]
 
 
 def synthesise(text):
-    text = cleanup_text(text)
     inputs = processor(text=text, return_tensors="pt")
     speech = model.generate_speech(inputs["input_ids"].to(device), speaker_embeddings.to(device), vocoder=vocoder)
     return speech.cpu()
@@ -69,8 +41,8 @@ def speech_to_speech_translation(audio):
 
 title = "Cascaded STST"
 description = """
-Demo for cascaded speech-to-speech translation (STST), mapping from source speech in any language to target speech in Spanish. Demo uses OpenAI's [Whisper Large v2](https://huggingface.co/openai/whisper-large-v2) model for speech translation, and [jjyaoao/speecht5_voxpopuli_spanish](https://huggingface.co/jjyaoao/speecht5_voxpopuli_spanish) checkpoint for text-to-speech, which is based on Microsoft's
-[SpeechT5 TTS](https://huggingface.co/microsoft/speecht5_tts) model for text-to-speech, fine-tuned in Spanish Audio dataset:
+Demo for cascaded speech-to-speech translation (STST), mapping from source speech in any language to target speech in English. Demo uses OpenAI's [Whisper Base](https://huggingface.co/openai/whisper-base) model for speech translation, and 
+[jjyaoao/speecht5_voxpopuli_spanish](https://huggingface.co/jjyaoao/speecht5_voxpopuli_spanish) model for text-to-speech:
 ![Cascaded STST](https://huggingface.co/datasets/huggingface-course/audio-course-images/resolve/main/s2st_cascaded.png "Diagram of cascaded speech to speech translation")
 """