gradio_app.py

import gradio as gr

from torchvision import transforms
from torchvision.transforms.functional import InterpolationMode

import os
import onnxruntime
import numpy as np

def predict_fault(image, model):
    image = image.detach().cpu().numpy()
    input = {model.get_inputs()[0].name: image}
    output = model.run(None, input)
    preds = np.argmax(output[0], 1)
    return preds.item()

def detect(image, writing_type, post_it, corner, empty):
    writing_type_model, post_it_model, corner_model, empty_model = models
    
    res_dict = {}

    if writing_type:
        input_image = writing_type_transforms(image).unsqueeze(0)
        label = predict_fault(input_image, writing_type_model)
        res_dict['writing_type'] = label

    if post_it:
        input_image = data_transforms(image).unsqueeze(0)
        label = predict_fault(input_image, post_it_model)
        res_dict['post_it'] = label

    if corner:
        input_image = data_transforms(image).unsqueeze(0)
        label = predict_fault(input_image, corner_model)
        res_dict['corner'] = label

    if empty:
        input_image = empty_transforms(image).unsqueeze(0)
        label = predict_fault(input_image, empty_model)
        res_dict['empty'] = 1 - label

    return res_dict

def load_models():
    try:
        MODEL_PATH = os.environ.get("MODEL_PATH", './models/')
        POST_IT_MODEL = os.environ.get("POST_IT_MODEL", 'post_it_model.onnx')
        CORNER_MODEL = os.environ.get("CORNER_MODEL", 'corner_model.onnx')
        EMPTY_MODEL = os.environ.get("EMPTY_MODEL", 'empty_v5_24_08_23.onnx')
        WRITING_TYPE_MODEL = os.environ.get("WRITING_TYPE_MODEL", 'writing_type_v1.onnx')

        print(f"ORT device: {onnxruntime.get_device()}")

        # Load the models and the trained weights
        writing_type_model = onnxruntime.InferenceSession(os.path.join(MODEL_PATH, WRITING_TYPE_MODEL))
        post_it_model = onnxruntime.InferenceSession(os.path.join(MODEL_PATH, POST_IT_MODEL))
        corner_model = onnxruntime.InferenceSession(os.path.join(MODEL_PATH, CORNER_MODEL))
        empty_model = onnxruntime.InferenceSession(os.path.join(MODEL_PATH, EMPTY_MODEL))

        return writing_type_model, post_it_model, corner_model, empty_model
    except Exception as e:
        print("Failed to load pretrained models: {}".format(e))

# Load the models
models = load_models()

# Transform methods for corner & post-it model inputs
data_transforms = transforms.Compose([
        transforms.Resize((224, 224)),
        transforms.ToTensor()
    ])

# Transform methods for empty model inputs
empty_transforms = transforms.Compose([
        transforms.Resize((224, 224), interpolation=InterpolationMode.BICUBIC),
        transforms.ToTensor(),
        transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
    ])

# Transform methods for writing-type model inputs
writing_type_transforms = transforms.Compose([
        transforms.Resize((224, 224), interpolation=InterpolationMode.BICUBIC),
        transforms.ToTensor(),
        transforms.Normalize([0.882, 0.883, 0.899], [0.088, 0.089, 0.094])
    ])


with gr.Blocks(title="Image Faulty Demo") as demo:
    gr.Markdown("""
    # Image Faulty
    Find the project [here](https://github.com/xiaoyao9184/image-faulty).
    """)

    with gr.Row():
        with gr.Column():
            detecting_img = gr.Image(label="Input Image", type="pil", height=512)
        with gr.Column():
            writing_ckb = gr.Checkbox(label="Writing type", value=True)
            postit_ckb = gr.Checkbox(label="Post it", value=True)
            corner_ckb = gr.Checkbox(label="Folded corner", value=True)
            empty_ckb = gr.Checkbox(label="Parper empty", value=True)
            detecting_btn = gr.Button("Detect")
            predicted_messages = gr.JSON(label="Detected Messages")

    detecting_btn.click(
        fn=detect,
        inputs=[detecting_img, writing_ckb, postit_ckb, corner_ckb, empty_ckb],
        outputs=[predicted_messages]
    )

if __name__ == '__main__':
    demo.launch()