app.py

from transformers import Pix2StructForConditionalGeneration, Pix2StructProcessor
import gradio as gr
from PIL import Image

# Use a valid model identifier.
# Replace "google/matcha-base" with your checkpoint if you have one.
model_name = "google/matcha-base"

# Load the pre-trained Pix2Struct model and processor
model = Pix2StructForConditionalGeneration.from_pretrained(model_name)
processor = Pix2StructProcessor.from_pretrained(model_name)

# Function to solve handwritten math problems
def solve_math_problem(image):
    # Preprocess the image: here we render a prompt asking the model to solve the problem.
    inputs = processor(images=image, text="Solve the math problem:", return_tensors="pt")
    
    # Generate the solution
    predictions = model.generate(**inputs, max_new_tokens=100)
    
    # Decode the output
    solution = processor.decode(predictions[0], skip_special_tokens=True)
    return solution

# Gradio interface
demo = gr.Interface(
    fn=solve_math_problem,
    inputs=gr.Image(type="pil", label="Upload Handwritten Math Problem"),
    outputs=gr.Textbox(label="Solution"),
    title="Handwritten Math Problem Solver",
    description="Upload an image of a handwritten math problem, and the model will attempt to solve it.",
    examples=[
        ["example1.jpg"],  # Add example images if available
        ["example2.jpg"]
    ],
    theme="soft"
)

if __name__ == "__main__":
    demo.launch()