Update app.py

app.py

@@ -1,32 +1,38 @@
 import gradio as gr
-from transformers import AutoTokenizer, AutoModelForSeq2SeqLM
-import torchaudio
-from torchaudio.transforms import Resample
-import torch
+from transformers import BlipProcessor, BlipForConditionalGeneration, AutoTokenizer, AutoModelForSeq2SeqLM, pipeline
 
-# สร้างโมเดล TTS
-model_name = "facebook/tts-crdnn-baker-softmax"
-tokenizer = AutoTokenizer.from_pretrained(model_name)
-model = AutoModelForSeq2SeqLM.from_pretrained(model_name)
+# Initialize Blip model for image captioning
+model_id = "dblasko/blip-dalle3-img2prompt"
+blip_model = BlipForConditionalGeneration.from_pretrained(model_id)
+blip_processor = BlipProcessor.from_pretrained(model_id)
 
-# ฟังก์ชันสำหรับแปลงข้อความเป็นเสียง
-def text_to_speech(text, output_path="generated_audio.wav"):
-    input_ids = tokenizer.encode(text, return_tensors="pt", max_length=150, truncation=True)
-    with torch.no_grad():
-        audio = model.generate(input_ids)
-    waveform = torchaudio.transforms.Resample(48_000, 24_000)(audio.squeeze().numpy())
-    torchaudio.save(output_path, waveform, 24_000)
+# Initialize TTS model from Hugging Face
+tts_model_name = "tts-mozilla/tts-ljspeech-multilingual"
+tts_tokenizer = AutoTokenizer.from_pretrained(tts_model_name)
+tts_model = AutoModelForSeq2SeqLM.from_pretrained(tts_model_name)
+tts = pipeline(task="text2speech", model=tts_model, tokenizer=tts_tokenizer)
 
-def play_audio(audio_path):
-    gr.audio(audio_path, type="player")
+def generate_caption(image):
+    # Generate caption from image using Blip model
+    inputs = blip_processor(images=image, return_tensors="pt")
+    pixel_values = inputs.pixel_values
+    generated_ids = blip_model.generate(pixel_values=pixel_values, max_length=50)
+    generated_caption = blip_processor.batch_decode(generated_ids, skip_special_tokens=True, temperature=0.8, top_k=40, top_p=0.9)[0]
 
-# สร้าง Gradio interface ที่ใช้ image input, textbox output, button และ audio player
+    # Use TTS model to convert generated caption to audio
+    audio_output = tts(generated_caption)
+    audio_output.save_to_path("generated_audio.mp3")
+
+    return generated_caption, "generated_audio.mp3"
+
+# Create a Gradio interface with an image input, a textbox output, a button, and an audio player
 demo = gr.Interface(
-    fn=text_to_speech, 
-    inputs=gr.Textbox(label="Enter Text"), 
+    fn=generate_caption, 
+    inputs=gr.Image(), 
     outputs=[
-        gr.Audio("audio", type="player"), 
-        gr.Button("Convert to Audio", play_audio),
+        gr.Textbox(label="Generated caption"), 
+        gr.Button("Convert to Audio"), 
+        gr.Audio(type="player", label="Generated Audio")
     ],
     live=True  # ทำให้ Gradio ทำงานแบบไม่บล็อก
 )