import torch
import scipy
import os
import streamlit as st
import pandas as pd
from transformers import pipeline #set_seed,
from transformers import VitsTokenizer, VitsModel
from datasets import load_dataset, Audio
from huggingface_hub.inference_api import InferenceApi
from src import *
########################
col1, col2 = st.columns([20,3])
with col2:
st.image('logo.png', use_container_width=True)
with col1:
st.title("Mockingbird")
st.header("A demo of open Text to Speech tools")
tts, about = st.tabs(["Text to speech", "**About**"])
########################
with tts:
# Configurations -- language choice and text
tts_lang = st.selectbox('Language of text', (language_list), format_func = decode_iso)
tts_text = st.text_area(label = "Please enter your sentence here:",
value="", placeholder=placeholders[tts_lang] )
target_speaker_file = st.file_uploader("If you would like to test voice conversion, you may upload your audio below. You should upload one file in .wav format. If you don't, a default file will be used.",
type=['wav'])
# Inference
if st.button("Generate"):
# Warning about alphabet support
if tts_lang in ['rus', 'fas']:
st.warning("WARNING! On Windows, ESpeak-NG has trouble synthesizing output when input is provided from non-Latin alphabets.")
st.divider()
# Synthesis
with st.spinner(":rainbow[Synthesizing, please wait... (this will be slowest the first time you generate audio in a new language)]"):
if tts_text == "":
tts_text=placeholders[tts_lang]
# First, make the audio
base_mms = synth_mms(tts_text, models[tts_lang]['mms'])
base_coqui= synth_coqui(tts_text, models[tts_lang]['coqui'])
base_espeakng= synth_espeakng(tts_text, models[tts_lang]['espeakng'])
try:
base_toucan= synth_toucan(tts_text, models[tts_lang]['toucan'])
except:
base_toucan=None
base_piper= synth_piper(tts_text, models[tts_lang]['piper'])
if tts_lang=="swh":
finetuned_mms1 = synth_mms(tts_text, "khof312/mms-tts-swh-female-1")
finetuned_mms2 = synth_mms(tts_text, "khof312/mms-tts-swh-female-2")
if tts_lang=="spa":
finetuned_mms1 = synth_mms(tts_text, "ylacombe/mms-spa-finetuned-argentinian-monospeaker")
finetuned_mms2 = synth_mms(tts_text, "ylacombe/mms-spa-finetuned-chilean-monospeaker")
finetuned_mms3 = synth_mms(tts_text, "ylacombe/mms-spa-finetuned-colombian-monospeaker")
finetuned_mms4 = synth_mms(tts_text, "khof312/mms-tts-spa-female")
if tts_lang=="lin":
finetuned_mms1 = synth_mms(tts_text, "khof312/mms-tts-lin-female")
try:
finetuned_africanvoices = synth_africanvoices(tts_text, models[tts_lang]['africanvoices'])
except:
pass
#vc_mms
#vc_coqui
#vc_espeakng
"## Synthesis"
"### Default models"
row1 = st.columns([1,1,2])
row2 = st.columns([1,1,2])
row3 = st.columns([1,1,2])
row4 = st.columns([1,1,2])
row5 = st.columns([1,1,2])
row6 = st.columns([1,1,2])
row1[0].write("**Model**")
row1[1].write("**Configuration**")
row1[2].write("**Audio**")
if base_mms is not None:
row2[0].write(f"[Meta MMS](https://huggingface.co/docs/transformers/main/en/model_doc/mms)")
row2[1].write("default")
row2[2].audio(base_mms[0], sample_rate = base_mms[1])
if base_coqui is not None:
row3[0].write(f"[Coqui](https://docs.coqui.ai/en/latest/index.html)")
row3[1].write("default")
row3[2].audio(base_coqui[0], sample_rate = base_coqui[1])
if base_espeakng is not None:
row4[0].write(f"[Espeak-ng](https://github.com/espeak-ng/espeak-ng)")
row4[1].write("default")
row4[2].audio(base_espeakng[0], sample_rate = base_espeakng[1])
if base_toucan is not None:
row5[0].write(f"[IMS-Toucan](https://github.com/DigitalPhonetics/IMS-Toucan)")
row5[1].write("default")
row5[2].audio(base_toucan[0], sample_rate = base_toucan[1])
if base_piper is not None:
row6[0].write(f"[Piper](https://github.com/rhasspy/piper)")
row6[1].write("default")
row6[2].audio(base_piper[0], sample_rate = base_piper[1])
#################################################################
if tts_lang == "swh":
"### Fine Tuned"
row1 = st.columns([1,1,2])
row2 = st.columns([1,1,2])
row3 = st.columns([1,1,2])
row1[0].write("**Model**")
row1[1].write("**Configuration**")
row1[2].write("**Audio**")
row2[0].write(f"Meta MMS")
row2[1].write("[female 1](https://huggingface.co/khof312/mms-tts-swh-female-1)")
row2[2].audio(finetuned_mms1[0], sample_rate = finetuned_mms1[1])
row3[0].write(f"Meta MMS")
row3[1].write("[female 2](https://huggingface.co/khof312/mms-tts-swh-female-2)")
row3[2].audio(finetuned_mms2[0], sample_rate = finetuned_mms2[1])
if tts_lang == "spa":
"### Fine Tuned"
row1 = st.columns([1,1,2])
row2 = st.columns([1,1,2])
row3 = st.columns([1,1,2])
row4 = st.columns([1,1,2])
row5 = st.columns([1,1,2])
row1[0].write("**Model**")
row1[1].write("**Configuration**")
row1[2].write("**Audio**")
row2[0].write(f"Meta MMS")
row2[1].write("[ylacombe - Argentinian](https://huggingface.co/ylacombe/mms-spa-finetuned-argentinian-monospeaker)")
row2[2].audio(finetuned_mms1[0], sample_rate = finetuned_mms1[1])
row3[0].write(f"Meta MMS")
row3[1].write("[ylacombe - Chilean](https://huggingface.co/ylacombe/mms-spa-finetuned-chilean-monospeaker)")
row3[2].audio(finetuned_mms2[0], sample_rate = finetuned_mms2[1])
row4[0].write(f"Meta MMS")
row4[1].write("[ylacombe - Colombian](https://huggingface.co/ylacombe/mms-spa-finetuned-colombian-monospeaker)")
row4[2].audio(finetuned_mms3[0], sample_rate = finetuned_mms3[1])
row5[0].write(f"Meta MMS")
row5[1].write("[khof312 - female](https://huggingface.co/khof312/mms-tts-spa-female)")
row5[2].audio(finetuned_mms4[0], sample_rate = finetuned_mms4[1])
if tts_lang == "lin":
"### Fine Tuned"
row1 = st.columns([1,1,2])
row2 = st.columns([1,1,2])
row3 = st.columns([1,1,2])
row1[0].write("**Model**")
row1[1].write("**Configuration**")
row1[2].write("**Audio**")
row2[0].write(f"Meta MMS")
row2[1].write("[khof312 - female](https://huggingface.co/khof312/mms-tts-lin-female)")
row2[2].audio(finetuned_mms1[0], sample_rate = finetuned_mms1[1])
try:
row3[0].write(f"African voices")
row3[1].write("[African Voices]()")
row3[2].audio(finetuned_africanvoices[0], sample_rate = finetuned_africanvoices[1])
except:
pass
st.divider()
"## Voice conversion" #################################################################
st.warning('''Note: The naturalness of the audio will only be as good as that of the audio in "default models" above.''')
if target_speaker_file is not None:
rate, wav = scipy.io.wavfile.read(target_speaker_file)
scipy.io.wavfile.write("target_speaker_custom.wav", data=wav, rate=rate)
target_speaker = "target_speaker_custom.wav"
else:
target_speaker = "target_speaker.wav"
if base_mms is not None:
scipy.io.wavfile.write("source_speaker_mms.wav", rate=base_mms[1], data=base_mms[0].T)
converted_mms = convert_coqui('source_speaker_mms.wav', target_speaker)
if base_coqui is not None:
scipy.io.wavfile.write("source_speaker_coqui.wav", rate=base_coqui[1], data=base_coqui[0].T)
converted_coqui = convert_coqui('source_speaker_coqui.wav', target_speaker)
if base_espeakng is not None:
scipy.io.wavfile.write("source_speaker_espeakng.wav", rate=base_espeakng[1], data=base_espeakng[0].T)
converted_espeakng = convert_coqui('source_speaker_espeakng.wav', target_speaker)
scipy.io.wavfile.write("source_speaker_toucan.wav", rate=base_toucan[1], data=base_toucan[0].T)
converted_toucan = convert_coqui('source_speaker_toucan.wav', target_speaker)
row1 = st.columns([1,1,2])
row2 = st.columns([1,1,2])
row3 = st.columns([1,1,2])
row4 = st.columns([1,1,2])
row1[0].write("**Model**")
row1[1].write("**Configuration**")
row1[2].write("**Audio**")
if base_mms is not None:
row1[0].write(f"Meta MMS")
row1[1].write(f"converted")
row1[2].audio(converted_mms[0], sample_rate = converted_mms[1])
if base_coqui is not None:
row2[0].write(f"Coqui")
row2[1].write(f"converted")
row2[2].audio(converted_coqui[0], sample_rate = converted_coqui[1])
if base_espeakng is not None:
row3[0].write(f"Espeak-ng")
row3[1].write(f"converted")
row3[2].audio(converted_espeakng[0], sample_rate = converted_espeakng[1])
row4[0].write(f"IMS Toucan")
row4[1].write(f"converted")
row4[2].audio(converted_toucan[0], sample_rate = converted_toucan[1])
#row3[0].write("MMS-TTS-SWH")
#row3[1].audio(synth, sample_rate=16_000)
#row3[2].audio(synth, sample_rate=16_000)
#st.audio(synth, sample_rate=16_000)
#data.write(np.random.randn(10, 1)
#col1.subheader("A wide column with a chart")
#col1.line_chart(data)
#col2.subheader("A narrow column with the data")
#col2.write(data)
with about:
#st.header("How it works")
st.markdown('''# Mockingbird TTS Demo
This page is a demo of the openly available Text to Speech models for various languages of interest. Currently, 3 synthesizers with multilingual offerings are supported out of the box:
- [**Meta's Massively Multilingual Speech (MMS)**](https://ai.meta.com/blog/multilingual-model-speech-recognition/) model, which supports over 1000 languages.[^1]
- [**IMS Toucan**](https://github.com/DigitalPhonetics/IMS-Toucan), which supports 7000 languages.[^4]
- [**ESpeak-NG's**](https://github.com/espeak-ng/espeak-ng/tree/master)'s synthetic voices**[^3]
On a case-by-case basis, for different languages of interest, I have added:
- [**Coqui's TTS**](https://docs.coqui.ai/en/latest/#) package;[^2] while no longer supported, Coqui acted as a hub for TTS model hosting and these models are still available. Languages must be added on a model-by-model basis.
- Specific fine-tuned variants of Meta's MMS (either fine-tuned by [Yoach Lacombe](https://huggingface.co/ylacombe), or fine-tuned by me using his scripts).
I am in the process of adding support for:
- [**Piper**](https://github.com/rhasspy/piper), a TTS system that supports multiple voices per language and approximately 30 languages. To test different voices, please see the [Huggingface demo](https://huggingface.co/spaces/k2-fsa/text-to-speech).[^5]
- [**African Voices**](https://github.com/neulab/AfricanVoices), a CMU research project that fine-tuned synthesizers for different African languages. The site hosting the synthesizers is deprecated but they can be downloaded from Google's Wayback Machine. [^6]
Voice conversion is currently achieved through Coqui.
Notes:
1. ESpeak-NG seems to have the worst performance out of the box, but it has a lot of options for controlling voice output.
2. Where a synthesizer supports multiple models/voices, I manually pick the appropriate model.
3. Not all synthesizers support a given language.
[^1]: Endpoints used are of the form https://huggingface.co/facebook/mms-tts-[LANG].
Learn more:
[Docs](https://huggingface.co/docs/transformers/model_doc/mms) |
[Paper](https://arxiv.org/abs/2305.13516) |
[Supported languages](https://dl.fbaipublicfiles.com/mms/misc/language_coverage_mms.html)
[^2]: [Available models](https://github.com/coqui-ai/TTS/blob/dev/TTS/.models.json)
[^3]: [Language list](https://github.com/espeak-ng/espeak-ng/blob/master/docs/languages.md)
[^4]: Language list is available in the Gradio API documentation [here](https://huggingface.co/spaces/Flux9665/MassivelyMultilingualTTS).
[^5]: The list of available voices is [here](https://github.com/rhasspy/piper/blob/master/VOICES.md), model checkpoints are [here](https://huggingface.co/datasets/rhasspy/piper-checkpoints/tree/main), and they can be tested [here](https://rhasspy.github.io/piper-samples/).
[^6]:
''')