Update app.py

app.py

@@ -1,123 +1,73 @@
 from fastapi import FastAPI
 from fastapi.responses import RedirectResponse
-import fitz  # PyMuPDF
-import docx
-import pptx
-import openpyxl
-import io
-from PIL import Image
 import gradio as gr
+from PIL import Image
+import io
+import numpy as np
+import easyocr
 from transformers import pipeline
-
-# Load models
-summarizer = pipeline("summarization", model="sshleifer/distilbart-cnn-12-6")
-image_captioner = pipeline("image-to-text", model="nlpconnect/vit-gpt2-image-captioning")
+from gtts import gTTS
+import tempfile
+import os
 
 app = FastAPI()
 
-# -------------------------
-# Extraction Functions
-# -------------------------
+# OCR Reader
+ocr_reader = easyocr.Reader(['en'], gpu=False)
 
-def extract_text_from_pdf(data: bytes):
-    try:
-        with fitz.open(stream=data, filetype="pdf") as doc:
-            return "\n".join([page.get_text() for page in doc])
-    except Exception as e:
-        return f"❌ PDF error: {e}"
+# Captioning and VQA Pipelines
+caption_model = pipeline("image-to-text", model="nlpconnect/vit-gpt2-image-captioning")
+vqa_model = pipeline("visual-question-answering", model="dandelin/vilt-b32-finetuned-vqa")
 
-def extract_text_from_docx(data: bytes):
-    try:
-        doc = docx.Document(io.BytesIO(data))
-        return "\n".join(p.text for p in doc.paragraphs if p.text.strip())
-    except Exception as e:
-        return f"❌ DOCX error: {e}"
+def process_image_question(image: Image.Image, question: str):
+    if image is None:
+        return "No image uploaded.", None
 
-def extract_text_from_pptx(data: bytes):
     try:
-        prs = pptx.Presentation(io.BytesIO(data))
-        text = []
-        for slide in prs.slides:
-            for shape in slide.shapes:
-                if hasattr(shape, "text"):
-                    text.append(shape.text)
-        return "\n".join(text)
-    except Exception as e:
-        return f"❌ PPTX error: {e}"
+        # Convert PIL image to numpy array
+        np_image = np.array(image)
 
-def extract_text_from_xlsx(data: bytes):
-    try:
-        wb = openpyxl.load_workbook(io.BytesIO(data))
-        text = []
-        for sheet in wb.sheetnames:
-            ws = wb[sheet]
-            for row in ws.iter_rows(values_only=True):
-                line = " ".join(str(cell) for cell in row if cell)
-                text.append(line)
-        return "\n".join(text)
-    except Exception as e:
-        return f"❌ XLSX error: {e}"
+        # OCR extraction
+        ocr_texts = ocr_reader.readtext(np_image, detail=0)
+        extracted_text = "\n".join(ocr_texts)
 
-# -------------------------
-# Main Logic
-# -------------------------
+        # Generate caption
+        caption = caption_model(image)[0]['generated_text']
 
-def summarize_document(file):
-    try:
-        filename = file.name.lower()
-        data = file.read()
+        # Ask question on image using VQA
+        vqa_result = vqa_model(image=image, question=question)
+        answer = vqa_result[0]['answer']
 
-        if filename.endswith(".pdf"):
-            text = extract_text_from_pdf(data)
-        elif filename.endswith(".docx"):
-            text = extract_text_from_docx(data)
-        elif filename.endswith(".pptx"):
-            text = extract_text_from_pptx(data)
-        elif filename.endswith(".xlsx"):
-            text = extract_text_from_xlsx(data)
-        else:
-            return "❌ Unsupported file format."
+        # Combine results
+        final_output = f"🖼️ Caption: {caption}\n\n📖 OCR Text:\n{extracted_text}\n\n❓ Answer: {answer}"
 
-        if not isinstance(text, str) or not text.strip():
-            return "❗ No extractable text."
+        # Convert answer to speech
+        tts = gTTS(text=answer)
+        with tempfile.NamedTemporaryFile(delete=False, suffix=".mp3") as tmp:
+            tts.save(tmp.name)
+            audio_path = tmp.name
 
-        summary = summarizer(text[:3000], max_length=150, min_length=30, do_sample=False)
-        return f"📄 Summary:\n{summary[0]['summary_text']}"
+        return final_output, audio_path
 
     except Exception as e:
-        return f"⚠️ Unexpected error: {e}"
-
-def interpret_image(image):
-    try:
-        return f"🖼️ Caption:\n{image_captioner(image)[0]['generated_text']}"
-    except Exception as e:
-        return f"⚠️ Image captioning error: {e}"
-
-# -------------------------
-# Gradio Interfaces
-# -------------------------
-
-doc_summary = gr.Interface(
-    fn=summarize_document,
-    inputs=gr.File(label="Upload a Document"),
-    outputs="text",
-    title="📄 Document Summarizer"
+        return f"❌ Error processing image: {e}", None
+
+gui = gr.Interface(
+    fn=process_image_question,
+    inputs=[
+        gr.Image(type="pil", label="Upload Image"),
+        gr.Textbox(lines=2, placeholder="Ask a question about the image...", label="Question")
+    ],
+    outputs=[
+        gr.Textbox(label="Result", lines=10),
+        gr.Audio(label="Answer (Audio)", type="filepath")
+    ],
+    title="🧠 Image QA with Voice",
+    description="Upload an image and ask any question. Supports OCR, captioning, visual QA, and audio response."
 )
 
-img_caption = gr.Interface(
-    fn=interpret_image,
-    inputs=gr.Image(type="pil", label="Upload an Image"),
-    outputs="text",
-    title="🖼️ Image Interpreter"
-)
-
-# -------------------------
-# FastAPI + Gradio Mount
-# -------------------------
-
-demo = gr.TabbedInterface([doc_summary, img_caption], ["Document Summary", "Image Captioning"])
-app = gr.mount_gradio_app(app, demo, path="/")
+app = gr.mount_gradio_app(app, gui, path="/")
 
 @app.get("/")
-def root():
+def home():
     return RedirectResponse(url="/")