utils.py

from dataclasses import dataclass
from typing import List, Optional, Tuple

import base64
from io import BytesIO
from PIL import Image


from smolagents import ChatMessage

def encode_image_to_base64(image):
    """Encodes a PIL image to a base64 string."""
    buffered = BytesIO()
    image.save(buffered, format="JPEG")
    return base64.b64encode(buffered.getvalue()).decode("utf-8")

DEFAULT_SYSTEM_PROMPT = \
"""You are a smart assistant designed to answer questions about a PDF document.
    You are given relevant information in the form of PDF pages preceded by their metadata: document title, page identifier, surrounding context. 
    Use them to construct a short response to the question, and cite your sources in the following format: (document, page number).
    If it is not possible to answer using the provided pages, do not attempt to provide an answer and simply say the answer is not present within the documents.
    Give detailed and extensive answers, only containing info in the pages you are given.
    You can answer using information contained in plots and figures if necessary.
    Answer in the same language as the query."""

def _build_query(query, pages):
    messages = []
    messages.append({"type": "text", "text": "PDF pages:\n"})
    for page in pages:
        capt = page.caption
        if capt is not None:
            messages.append({
                    "type": "text",
                    "text": capt
                })
        messages.append({
                "type": "image_url",
                "image_url": {
                "url": f"data:image/jpeg;base64,{encode_image_to_base64(page.image)}"
                },
            })
    messages.append({"type": "text", "text": f"Query:\n{query}"})

    return messages

def query_openai(query, pages, api_key=None, system_prompt=DEFAULT_SYSTEM_PROMPT, model="gpt-4o-mini") -> ChatMessage:
    """Calls OpenAI's GPT-4o-mini with the query and image data."""
    if api_key and api_key.startswith("sk"):
        try:
            from openai import OpenAI
    
            client = OpenAI(api_key=api_key.strip())

            response = client.chat.completions.create(
                model=model,
                messages=[
                    {
                        "role": "system",
                        "content": system_prompt
                    },
                    {
                    "role": "user",
                    "content": _build_query(query, pages)
                    }
                ],
                max_tokens=500,
            )

            message = ChatMessage.from_dict(
                response.choices[0].message.model_dump(include={"role", "content", "tool_calls"})
            )
            message.raw = response

            return message
        
        except Exception as e:
            return "OpenAI API connection failure. Verify the provided key is correct (sk-***)."
        
    return "Enter your OpenAI API key to get a custom response"

CONTEXT_SYSTEM_PROMPT = \
"""You are a smart assistant designed to extract context of PDF pages.
    Give concise answers, only containing info in the pages you are given.
    You can answer using information contained in plots and figures if necessary."""

RAG_SYSTEM_PROMPT = \
""" You are a smart assistant designed to answer questions about a PDF document.

    You are given relevant information in the form of PDF pages preceded by their metadata: document title, page identifier, surrounding context.
    Use them to construct a response to the question, and cite your sources.
    Use the following citation format:
    "Some information from a first document [1, p.Page Number]. Some information from the same first document but at a different page [1, p.Page Number]. Some more information from another document [2, p.Page Number].
    ...
    Sources:
    [1] Document Title
    [2] Another Document Title"

    You can answer using information contained in plots and figures if necessary.
    If it is not possible to answer using the provided pages, do not attempt to provide an answer and simply say the answer is not present within the documents.
    Give detailed answers, only containing info in the pages you are given.
    Answer in the same language as the query."""

@dataclass
class Metadata:
    doc_title: str
    page_id: int
    context: Optional[str] = None

    def __str__(self):
        return f"Document: {self.doc_title}, Page ID: {self.page_id}, Context: {self.context}"
    
@dataclass
class Page:
    image: Image.Image
    metadata: Optional[Metadata] = None

    @property
    def caption(self):
        if self.metadata is None:
            return None
        return f"Document: {self.metadata.doc_title}, Context: {self.metadata.context}"
    
    def __hash__(self):
        return hash(self.image)