app.py

import gradio as gr
from transformers import CLIPProcessor, CLIPModel

clip = CLIPModel.from_pretrained("openai/clip-vit-base-patch32")
processor = CLIPProcessor.from_pretrained("openai/clip-vit-base-patch32")

def inference(input_img, captions):
    captions_list = captions.split(",")
    inputs = processor(text=captions_list, images=input_img, return_tensors="pt", padding=True)
    outputs = clip(**inputs)
    # this is the image-text similarity score
    logits_per_image = outputs.logits_per_image
    probs = logits_per_image.softmax(dim=1)
    probabilities_percentages = ', '.join(['{:.2f}%'.format(prob.item() * 100) for prob in probs[0]])
    return probabilities_percentages

title = "CLIP Inference: Application using a pretrained CLIP model"
description = "An application using Gradio interface that accepts an image and some captions, and displays a probability score with which each caption describes the image "

examples = [
    ["examples/woman_standing.jpg","woman standing inside a house, a photo of dog, running water, cupboard, home interiors"],
    ["examples/city.jpg","long shot of a city, sunsetting on a urban place, river with animals"],
    ["examples/dinning_tables.jpg","a bunch of dinning tables, cricket ground with players, movie theater, plants with music"],
    ["examples/giraffe.jpg","tall giraffe standing and turning back, luxurious car on a road, a bunch of people standing"],
    ["examples/dogs.jpg","a couple of dogs standing, woman standing inside a house, a photo of MJ"]
]

demo = gr.Interface(
    inference,
    inputs = [
        gr.Image(shape=(416, 416), label="Input Image"),
        gr.Textbox(placeholder="List of captions")],
    outputs = [gr.Textbox(label="Probability score on captions likely describing the image")],
    title = title,
    description = description,
    examples = examples,
)
demo.launch()