app.py

import gradio as gr 
import gradio.components as grc
import torch
from PIL import Image

from lavis.models import load_model_and_preprocess
from lavis.processors import load_processor

raw_image = Image.open("merlion.png").convert("RGB")

# setup device to use
device = torch.device("cuda") if torch.cuda.is_available() else "cpu"
caption = "merlion in Singapore"

model, vis_processors, text_processors = load_model_and_preprocess("blip2_image_text_matching", "pretrain", device=device, is_eval=True)

def predict(raw_image, caption):
    raw_image = raw_image.convert("RGB")
    img = vis_processors["eval"](raw_image).unsqueeze(0).to(device)
    txt = text_processors["eval"](caption)
    itm_output = model({"image": img, "text_input": txt}, match_head="itm")
    itm_scores = torch.nn.functional.softmax(itm_output, dim=1)
    itm_score = itm_scores[:, 1].item()
    itc_score = model({"image": img, "text_input": txt}, match_head='itc')
    return '%.3f' % itm_score, '%.4f' % itc_score

app = gr.Interface(fn=predict, inputs=[grc.Image(type="pil"), grc.Textbox()], outputs=[grc.Text(label="itm score"), grc.Text(label="itc score")])
app.launch()