demo

app.py

@@ -0,0 +1,115 @@
+import gradio as gr
+import librosa
+import numpy as np
+import torch
+
+from transformers import SpeechT5Processor, SpeechT5ForTextToSpeech, SpeechT5HifiGan
+
+
+checkpoint = "microsoft/speecht5_tts"
+processor = SpeechT5Processor.from_pretrained(checkpoint)
+model = SpeechT5ForTextToSpeech.from_pretrained(checkpoint)
+vocoder = SpeechT5HifiGan.from_pretrained("microsoft/speecht5_hifigan")
+
+
+speaker_embeddings = {
+    "GGP": "spkemb/speaker0.npy",
+    "BGP": "spkemb/speaker1.npy",
+    "BDP": "spkemb/speaker2.npy",
+}
+
+
+def predict(text, speaker):
+    if len(text.strip()) == 0:
+        return (16000, np.zeros(0).astype(np.int16))
+
+    inputs = processor(text=text, return_tensors="pt")
+
+    # limit input length
+    input_ids = inputs["input_ids"]
+    input_ids = input_ids[..., :model.config.max_text_positions]
+
+    if speaker == "Surprise Me!":
+        # load one of the provided speaker embeddings at random
+        idx = np.random.randint(len(speaker_embeddings))
+        key = list(speaker_embeddings.keys())[idx]
+        speaker_embedding = np.load(speaker_embeddings[key])
+
+        # randomly shuffle the elements
+        np.random.shuffle(speaker_embedding)
+
+        # randomly flip half the values
+        x = (np.random.rand(512) >= 0.5) * 1.0
+        x[x == 0] = -1.0
+        speaker_embedding *= x
+
+        #speaker_embedding = np.random.rand(512).astype(np.float32) * 0.3 - 0.15
+    else:
+        speaker_embedding = np.load(speaker_embeddings[speaker[:3]])
+
+    speaker_embedding = torch.tensor(speaker_embedding).unsqueeze(0)
+
+    speech = model.generate_speech(input_ids, speaker_embedding, vocoder=vocoder)
+
+    speech = (speech.numpy() * 32767).astype(np.int16)
+    return (16000, speech)
+
+
+title = "SpeechT5: Speech Synthesis"
+
+description = """
+The <b>SpeechT5</b> model is pre-trained on text as well as speech inputs, with targets that are also a mix of text and speech.
+By pre-training on text and speech at the same time, it learns unified representations for both, resulting in improved modeling capabilities.
+SpeechT5 can be fine-tuned for different speech tasks. This space demonstrates the <b>text-to-speech</b> (TTS) checkpoint for the English language.
+See also the <a href="https://huggingface.co/spaces/Matthijs/speecht5-asr-demo">speech recognition (ASR) demo</a>
+and the <a href="https://huggingface.co/spaces/Matthijs/speecht5-vc-demo">voice conversion demo</a>.
+Refer to <a href="https://colab.research.google.com/drive/1i7I5pzBcU3WDFarDnzweIj4-sVVoIUFJ">this Colab notebook</a> to learn how to fine-tune the SpeechT5 TTS model on your own dataset or language.
+<b>How to use:</b> Enter some English text and choose a speaker. The output is a mel spectrogram, which is converted to a mono 16 kHz waveform by the
+HiFi-GAN vocoder. Because the model always applies random dropout, each attempt will give slightly different results.
+The <em>Surprise Me!</em> option creates a completely randomized speaker.
+"""
+
+article = """
+<div style='margin:20px auto;'>
+<p>References: <a href="https://arxiv.org/abs/2110.07205">SpeechT5 paper</a> |
+<a href="https://github.com/microsoft/SpeechT5/">original GitHub</a> |
+<a href="https://huggingface.co/mechanicalsea/speecht5-tts">original weights</a></p>
+<pre>
+@article{Ao2021SpeechT5,
+  title   = {SpeechT5: Unified-Modal Encoder-Decoder Pre-training for Spoken Language Processing},
+  author  = {Junyi Ao and Rui Wang and Long Zhou and Chengyi Wang and Shuo Ren and Yu Wu and Shujie Liu and Tom Ko and Qing Li and Yu Zhang and Zhihua Wei and Yao Qian and Jinyu Li and Furu Wei},
+  eprint={2110.07205},
+  archivePrefix={arXiv},
+  primaryClass={eess.AS},
+  year={2021}
+}
+</pre>
+</div>
+"""
+
+examples = [
+    ["Rhyfeddod neu ffenomenon optegol a meteorolegol yw enfys, pan fydd sbectrwm o olau yn ymddangos yn yr awyr pan fo'r haul yn disgleirio ar ddiferion o leithder yn atmosffer y ddaear.", "GGP (gwryw-gogledd-pro)"],
+    ["Rhyfeddod neu ffenomenon optegol a meteorolegol yw enfys, pan fydd sbectrwm o olau yn ymddangos yn yr awyr pan fo'r haul yn disgleirio ar ddiferion o leithder yn atmosffer y ddaear.", "BGP (benyw-gogledd-pro)"],
+    ["Rhyfeddod neu ffenomenon optegol a meteorolegol yw enfys, pan fydd sbectrwm o olau yn ymddangos yn yr awyr pan fo'r haul yn disgleirio ar ddiferion o leithder yn atmosffer y ddaear.", "BDP (benyw-de-pro)"],
+]
+
+gr.Interface(
+    fn=predict,
+    inputs=[
+        gr.Text(label="Input Text"),
+        gr.Radio(label="Speaker", choices=[
+            "GGP (gwryw-gogledd-pro)",
+            "BGP (benyw-gogledd-pro)",
+            "BDP (benyw-de-pro)",
+            "Surprise Me!"
+        ],
+        value="GGP (gwryw-gogledd-pro)"),
+    ],
+    outputs=[
+        gr.Audio(label="Generated Speech", type="numpy"),
+    ],
+    title=title,
+    description=description,
+    article=article,
+    examples=examples,
+).launch()

requirements.txt

@@ -0,0 +1,3 @@
+transformers
+sentencepiece
+datasets[audio]

spkemb/speaker0.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:29a89ec165f28301a1dda1ef5e1a5c8b5ddd60c0bfa4094f7fb6b88035812ca3
+size 229448

spkemb/speaker1.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cc9f3a6e58a1e2f52c9258a7e9372b30a0966dd5b3fdc82de23cd2e38fd61c67
+size 599888

spkemb/speaker2.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e61aa815909de199d2261c806674d5cbdc264120c930838dffbe4c21d957845c
+size 158888