Create app.py

app.py

@@ -0,0 +1,82 @@
+import gradio as gr
+from transformers import pipeline, AutoTokenizer, AutoModelForCausalLM
+from gtts import gTTS
+import tempfile
+
+# Function to initialize models with exception handling
+def initialize_model():
+    try:
+        # Load ASR (Automatic Speech Recognition) model for voice-to-text
+        asr_model = pipeline("automatic-speech-recognition", model="openai/whisper-medium")
+
+        # Load Translation model (supports multiple language pairs)
+        translation_model = pipeline("translation", model="Helsinki-NLP/opus-mt-en-mul")  # Choose a supported model
+
+        # Load conversational model (fine-tuned on dialogues)
+        model_name = "microsoft/DialoGPT-medium"  # Example conversational model
+        tokenizer = AutoTokenizer.from_pretrained(model_name)
+        model = AutoModelForCausalLM.from_pretrained(model_name)
+
+        return asr_model, translation_model, tokenizer, model
+    except Exception as e:
+        print(f"Error initializing models: {e}")
+        return None, None, None, None
+
+# Initialize the models
+asr_model, translation_model, tokenizer, conversation_model = initialize_model()
+
+def chatbot_speech_to_speech(audio_input, target_language):
+    try:
+        # Step 1: Convert Audio to Text
+        text_input = asr_model(audio_input)["text"]
+
+        # Step 2: Translate Text to English if the input language is not English
+        if target_language != "en":
+            translated_text = translation_model(text_input, src_lang=target_language, tgt_lang="en")[0]['translation_text']
+        else:
+            translated_text = text_input
+
+        # Step 3: Generate conversational response using the dialogue model
+        inputs = tokenizer.encode(translated_text + tokenizer.eos_token, return_tensors='pt')
+        response_ids = conversation_model.generate(inputs, max_length=100, pad_token_id=tokenizer.eos_token_id)
+        response_text = tokenizer.decode(response_ids[:, inputs.shape[-1]:][0], skip_special_tokens=True)
+
+        # Step 4: Translate the response text back to the target language
+        if target_language != "en":
+            final_response = translation_model(response_text, src_lang="en", tgt_lang=target_language)[0]['translation_text']
+        else:
+            final_response = response_text
+
+        # Step 5: Convert text to speech using gTTS
+        tts = gTTS(final_response, lang=target_language)
+        temp_file = tempfile.NamedTemporaryFile(delete=False, suffix=".mp3")
+        tts.save(temp_file.name)
+
+        return temp_file.name
+    except Exception as e:
+        return f"Error in processing: {e}"
+
+# Gradio Interface Function
+def interface(audio, language):
+    result = chatbot_speech_to_speech(audio, language)
+    return result
+
+# Define the Gradio app with Blocks using the latest syntax
+with gr.Blocks() as gradio_ui:
+    gr.Markdown("# Multilingual Voice-to-Voice Chatbot for Kids")
+    gr.Markdown("### Speak to the chatbot in your selected language and receive a spoken response.")
+    
+    audio_input = gr.Audio(type="filepath", label="Record your message")
+    language_dropdown = gr.Dropdown(choices=["en", "fr", "es", "de", "zh", "ur"], label="Select Language")
+    
+    result_audio = gr.Audio(type="filepath", label="Chatbot Response")
+    
+    
+    submit_btn = gr.Button("Submit")
+    submit_btn.click(fn=interface, inputs=[audio_input, language_dropdown], outputs=result_audio)
+
+# Launch the app
+if asr_model and translation_model and tokenizer and conversation_model:
+    gradio_ui.launch()
+else:
+    print("Error initializing one or more models. Please check your model configuration.")