Updated with OCR model and Gradio integration

app.py

@@ -1,24 +1,28 @@
 import gradio as gr
-from transformers import AutoTokenizer, AutoModelForVision2Seq
+from lmdeploy import pipeline, TurbomindEngineConfig
+from lmdeploy.vl import load_image
 
-# Load the tokenizer and model, trusting remote code
-model_name = "OpenGVLab/InternVL2-1B"
-tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
-model = AutoModelForVision2Seq.from_pretrained(model_name, trust_remote_code=True)
+# Model configuration
+model = 'OpenGVLab/InternVL2-1B'
 
-def analyze_image_text(image, text):
-    # Tokenize the input
-    inputs = tokenizer(text, return_tensors="pt")
-    # Use the model to get outputs
-    outputs = model.generate(**inputs)
-    return tokenizer.decode(outputs[0])
+# Configure the pipeline for the model
+pipe = pipeline(model, backend_config=TurbomindEngineConfig(session_len=8192))
 
-# Gradio interface
+# Function to process and describe the image
+def analyze_image(image):
+    # Convert PIL image to the format the model expects
+    img = load_image(image)  # `load_image` can handle both URLs and PIL images
+    # Run inference on the uploaded image
+    response = pipe(('describe this image', img))
+    return response.text
+
+# Gradio interface for image input
 demo = gr.Interface(
-    fn=analyze_image_text,
-    inputs=[gr.Image(type="pil"), gr.Textbox()],
-    outputs="text",
-    title="InternVL2-1B Image-Text Model"
+    fn=analyze_image,
+    inputs=gr.Image(type="pil"),  # Upload an image
+    outputs="text",  # Output the extracted text
+    title="Image Description using OpenGVLab/InternVL2-1B",
+    description="Upload an image and get a description generated by the InternVL2-1B model."
 )
 
 if __name__ == "__main__":

requirements.txt

@@ -4,4 +4,5 @@ torch
 gradio
 datasets
 pytesseract
-Pillow
+Pillow
+lmdeploy