First commit

Dockerfile

@@ -0,0 +1,112 @@
+# Dockerfile customized for deployment on HuggingFace Spaces platform
+
+# -- The Dockerfile has been tailored specifically for use on HuggingFace.
+# -- It implies that certain modifications or optimizations have been made with HuggingFace's environment in mind.
+# -- It uses "HuggingFace Spaces" to be more specific about the target platform.
+
+# FROM pytorch/pytorch:2.2.1-cuda12.1-cudnn8-devel
+FROM pytorch/pytorch:2.4.0-cuda12.1-cudnn9-devel 
+# FOR HF
+
+USER root
+
+ENV DEBIAN_FRONTEND=noninteractive
+RUN apt-get update && apt-get install -y \
+    git \
+    cmake \
+    python3 \
+    python3-pip \
+    python3-venv \
+    python3-dev \
+    python3-numpy \
+    gcc \
+    build-essential \
+    gfortran \
+    wget \
+    curl \
+    pkg-config \
+    software-properties-common \
+    zip \
+    && apt-get clean && rm -rf /tmp/* /var/tmp/*
+
+RUN apt-get update && DEBIAN_FRONTEND=noninteractive \
+    apt-get install -y python3.10 python3-pip
+
+RUN apt-get install -y libopenblas-base libopenmpi-dev
+
+ENV TZ=Asia/Dubai
+RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
+
+
+
+RUN useradd -m -u 1000 user
+    
+RUN apt-get update && apt-get install -y sudo && \
+echo 'user ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
+
+USER user
+ENV HOME=/home/user \
+	PATH=/home/user/.local/bin:$PATH
+
+RUN mkdir $HOME/app
+RUN mkdir $HOME/app/test_images
+
+# WORKDIR $HOME/app
+
+RUN chown -R user:user $HOME/app   
+
+USER user
+WORKDIR $HOME/app
+
+RUN python -m  pip install  qwen-vl-utils 
+RUN python -m pip install --pre -U -f https://mlc.ai/wheels mlc-llm-nightly-cu122 mlc-ai-nightly-cu122
+
+RUN  python3 -m pip install chromadb db-sqlite3 auto-gptq exllama sqlalchemy  
+WORKDIR $HOME/app
+RUN git clone https://github.com/casper-hansen/AutoAWQ  
+WORKDIR $HOME/app/AutoAWQ/ 
+RUN python3 -m pip  install -e .
+WORKDIR $HOME/app
+# ENV FLASH_ATTENTION_FORCE_BUILD=TRUE
+RUN python -m pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu121
+RUN python -m pip install  accelerate diffusers datasets timm flash-attn==2.6.1 gradio
+
+RUN  python3 -m pip install --no-deps optimum  
+RUN  python3 -m pip install --no-deps autoawq>=0.1.8 
+
+#This seems to be a must : Intel Extension for PyTorch 2.4 needs to work with PyTorch 2.4.*, but PyTorch 2.2.2 is
+RUN python -m pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu121
+RUN  python3 -m pip install -U accelerate 
+RUN  python3 -m pip install -U git+https://github.com/huggingface/transformers
+
+WORKDIR $HOME/app
+COPY --chown=user:user app.py .
+COPY --chown=user:user test_images /home/user/app/test_images
+# /home/user/app/
+# chown -R user:user /home/user/.cache/
+
+ENV PYTHONUNBUFFERED=1 	GRADIO_ALLOW_FLAGGING=never 	GRADIO_NUM_PORTS=1 	GRADIO_SERVER_NAME=0.0.0.0     GRADIO_SERVER_PORT=7860 	SYSTEM=spaces
+RUN python3 -m pip  install pennylane sympy pennylane-qiskit duckdb
+WORKDIR $HOME/app
+
+EXPOSE 8097 7842 8501 8000 6666 7860
+
+CMD ["python", "app.py"]
+
+
+# ERROR! Intel® Extension for PyTorch* needs to work with PyTorch 2.4.*, but PyTorch 2.2.2 is found. Please switch to the matching version and run again.
+# ERROR! Intel® Extension for PyTorch* needs to work with PyTorch 2.4.*, but PyTorch 2.2.2 is found. Please switch to the matching version and run again.
+# `Qwen2VLRotaryEmbedding` can now be fully parameterized by passing the model config through the `config` argument. All other arguments will be removed in v4.46
+# /home/user/.local/lib/python3.10/site-packages/transformers/modeling_utils.py:4749: FutureWarning: `_is_quantized_training_enabled` is going to be deprecated in transformers 4.39.0. Please use `model.hf_quantizer.is_trainable` instead
+#   warnings.warn(
+# /home/user/.local/lib/python3.10/site-packages/accelerate/utils/imports.py:336: UserWarning: Intel Extension for PyTorch 2.4 needs to work with PyTorch 2.4.*, but PyTorch 2.2.2 is found. Please switch to the matching version and run again.
+#   warnings.warn(
+# Error loading model Qwen/Qwen2-VL-2B-Instruct-GPTQ-Int4: Found modules on cpu/disk. 
+# Using Exllama or Exllamav2 backend requires all the modules to be on GPU.You can deactivate exllama backend by setting `disable_exllama=True` in the quantization config object
+# Error loading model Qwen/Qwen2-VL-7B-Instruct: (ReadTimeoutError("HTTPSConnectionPool(host='hf.co', port=443): 
+# Read timed out. (read timeout=10)"), '(Request ID: b8269a88-9b6b-43e0-942d-1049f173dc00)')
+
+# Error loading model Qwen/Qwen2-VL-7B-Instruct: CUDA out of memory. 
+# Tried to allocate 130.00 MiB. GPU 0 has a total capacity of 14.58 GiB of which 77.62 MiB is free.
+
+# instruct: FlashAttention only supports Ampere GPUs or newer.

README.md

@@ -0,0 +1,297 @@
+---
+title: Vlms
+emoji: 🔥
+colorFrom: gray
+colorTo: green
+sdk: docker
+pinned: false
+license: mit
+---
+
+# VLM-Image-Analysis: A Vision-and-Language Modeling Framework
+
+Welcome to the Hugging Face Space for VLM-Image-Analysis. This space showcases a cutting-edge framework that combines multiple Vision-Language Models (VLMs) and a Large Language Model (LLM) to provide comprehensive image analysis and captioning.
+
+<h1 align="center">    
+  <img src="static/image.jpg" width="50%"></a>
+  <h6> (Adapted from wang2023allseeing: https://huggingface.co/datasets/OpenGVLab/CRPE?row=1) <h6>  
+</h1>
+
+This repository contains the core code for a multi-model framework that enhances image interpretation through the combined power of several Vision-and-Language Modeling (VLM) systems. VLM-Image-Analysis delivers detailed, multi-faceted analyses of images by leveraging N cutting-edge VLM models, pre-trained on a wide range of datasets to detect diverse visual cues and linguistic patterns.
+
+It is available on HF at the follwing HF space: https://huggingface.co/spaces/BoltzmannEntropy/vlms/
+
+<h1 align="center">    
+  <img src="static/003.png" width="100%"></a>  
+</h1>
+
+1. Multiple VLMs analyze the input image independently, providing diverse perspectives.
+2. A 'judge' LLM synthesizes these outputs into a comprehensive, coherent description.
+
+
+## Abstract
+
+
+This research introduces a novel technique in the realm of automatic image captioning that leverages the collaborative potential between Vision-Language Models (VLMs) and Large Language Models (LLMs). Our innovative method utilizes a two-tiered framework: initially, various VLMs provide diverse verbal descriptions of an image based on unique prompts; subsequently, these array of captions are integrated by a central 'judge' LLM to produce a cohesive and comprehensive caption that encapsulates the essence of the image. The objective of this synergistic approach is to elevate the precision, richness, and contextual appropriateness of image descriptions by pooling together diverse model capabilities. We validate the effectiveness of our dual-model strategy across a spectrum of VLMs—namely Qwen2-VL, Phi-3-vision, and Moondream2—and assess its performance on different datasets. Our empirical results indicate that this ensemble method yields image captions with substantially higher quality and depth than traditional single-model approaches, marking a significant stride in the evolution of computer vision technology.
+
+
+## Introduction
+
+The field of computer vision has undergone significant transformations in recent times, especially within the niche area of generating descriptive text for images—a task known as image captioning. Vision-Language Models (VLMs) have risen to prominence as instrumental instruments that facilitate the seamless integration of visual information with comprehension of natural language. Despite their capabilities, individual VLMs may possess unique advantages and constraints, which can sometimes result in descriptions that are either biased or lack completeness. This situation thus opens up a scope for creative approaches that can capitalize on the strengths of multiple models to produce more balanced and comprehensive image interpretations.
+
+<h1 align="center">    
+  <img src="static/005.png" width="1000%"></a>  
+</h1>
+
+To address this issue, we introduce a novel methodology that leverages the combined strengths of multiple VLMs in conjunction with a Large Language Model (LLM) acting as a judge. Our approach is designed to produce more comprehensive, accurate, and contextually rich image captions by following a two-stage process:
+
+1. **Multi-VLM Caption Generation**: In the first stage, we employ several distinct VLMs, including Qwen2-VL, Phi-3-vision, and Moondream2, to generate image descriptions. Each model is prompted with different instructions, encouraging diverse perspectives on the image content. This diversity is crucial, as it allows for a more comprehensive capture of the image's nuances and details.
+
+2. **LLM-based Caption Synthesis**: The second stage involves using a 'judge' LLM to analyse and synthesise the outputs from the first stage. This model is tasked with unifying the various descriptions into a single, coherent caption that captures the essence of all initial responses. The judge LLM's role is not merely to aggregate information but to intelligently combine and refine the insights provided by the VLMs.
+
+Our methodology leverages state-of-the-art models and techniques, including:
+
+- The Qwen2-VL family of models, which offer varying capabilities and model sizes, allowing for a range of perspectives and computational efficiencies.
+- Microsoft's Phi-3-vision model, known for its efficiency and performance in visual understanding tasks.
+- The Moondream2 model, which brings unique perspectives to image understanding, potentially capturing aspects overlooked by other models.
+
+The implementation utilises advanced libraries such as PyTorch and Hugging Face's Transformers, ensuring efficient processing and easy integration with existing deep learning workflows. We have also incorporated features to handle both individual images and batch processing from datasets or ZIP files, making our approach versatile for various research and practical applications.
+
+By combining multiple VLMs and using an LLM as a judge, we aim to mitigate individual model biases, capture a broader range of image details, and produce captions that are more informative and contextually appropriate. This approach not only improves the quality of image captioning but also opens up new avenues for exploring the synergies between different AI models in multimodal tasks.
+
+It is important to note, however, that while our method shows promise, it is not without limitations. The effectiveness of the approach may vary depending on the specific combination of models used and the nature of the images being captioned. Additionally, the computational resources required for this ensemble approach are significantly higher than those needed for single-model methods, which may limit its applicability in resource-constrained environments.
+
+In the following sections, we shall detail our methodology, present the experimental setup, and discuss the results and implications of our findings. We believe this work contributes significantly to the field of image captioning and demonstrates the potential of ensemble approaches in vision-language tasks.
+
+
+
+## Table of Contents
+
+- [VLM-Image-Analysis: A Vision-and-Language Modeling Framework](#vlm-image-analysis-a-vision-and-language-modeling-framework)
+  - [Abstract](#abstract)
+  - [Introduction](#introduction)
+  - [Table of Contents](#table-of-contents)
+  - [Base VLMs](#base-vlms)
+  - [Judge VLM](#judge-vlm)
+- [How to Use](#how-to-use)
+  - [Example Process](#example-process)
+  - [Models](#models)
+  - [Setup](#setup)
+  - [Inference](#inference)
+  - [The Gradio APP](#the-gradio-app)
+    - [1. VLM Model Selection](#1-vlm-model-selection)
+    - [2. Device Settings](#2-device-settings)
+    - [3. Image Selection](#3-image-selection)
+  - [Docker Setup for LLMs](#docker-setup-for-llms)
+    - [Base System](#base-system)
+    - [LLM-related Packages](#llm-related-packages)
+    - [Jupyter Configuration](#jupyter-configuration)
+    - [Exposed Ports](#exposed-ports)
+  - [Prompts](#prompts)
+    - [Example of Available Prompts](#example-of-available-prompts)
+  - [Citation](#citation)
+
+## Base VLMs
+
+The first phase of the framework isolates each specialized VLM model to individually analyse an input image. Each model processes the image independently, resulting in multiple descriptive outputs from different viewpoints. These models are carefully selected based on their unique strengths in visual and textual analysis, ensuring a comprehensive examination of the image.
+
+For instance, a VLM might be prompted with the following directive:
+```
+"Provide an extensive description of all elements in this image, including objects, people, and activities."
+```
+
+This phase enables the generation of varied textual descriptions and captions, offering diverse perspectives on the visual content.
+
+## Judge VLM
+
+In the second phase, the outputs from the base VLMs are passed to a synthesizing "Judge" VLM model. This model is specifically trained to merge the individual outputs into a coherent, unified conclusion that captures the overall context of the image.
+
+The Judge VLM is designed to handle different types of input data, including textual descriptions from both K and V models. Its role is to harmonise and integrate these multiple viewpoints, providing a more comprehensive synthesis of the image's content.
+
+# How to Use
+
+1. **Access the Space**: Click on the "Open in Spaces" badge at the top of this README or visit [https://huggingface.co/spaces/BoltzmannEntropy/vlms](https://huggingface.co/spaces/BoltzmannEntropy/vlms)
+
+2. **Select a Dataset**: Choose from the available Hugging Face datasets:
+   - gokaygokay/panorama_hdr_dataset
+   - OpenGVLab/CRPE
+
+3. **Select Models**: Choose which VLMs you want to use for the analysis from the available options.
+
+4. **Choose Prompts**: Select from predefined prompts or enter a custom prompt to guide the analysis.
+
+5. **Run Analysis**: Click the "Submit" button to start the image analysis process.
+
+6. **View Results**: The generated descriptions, detected text, and translations will be displayed in the interface.
+
+## Example Process
+
+For instance, when the system processes an image and generates an initial textual output from a model such as "microsoft/Phi-3-vision-128k-instruct," the Judge VLM then engages the "Qwen/Qwen2-VL-7B-Instruct" model for further analysis. The Qwen/Qwen2 model evaluates this initial interpretation and offers a synthesized response that integrates both visual and textual insights, providing a final, nuanced interpretation to the user or downstream system.
+
+## Models
+
+The framework utilises multiple advanced Vision-and-Language Models (VLMs). For more details, refer to the respective model documentation:
+- [Microsoft/Phi-3-Vision-128K-Instruct](https://huggingface.co/microsoft/Phi-3-Vision-128K-Instruct)
+- [Qwen/Qwen2-VL-7B-Instruct](https://huggingface.co/Qwen/Qwen2-VL-7B-Instruct)
+
+## Setup
+
+To set up this project, follow these steps in a terminal:
+
+1. **Clone the Repository**
+
+    - Clone the repository to your local machine:
+      ```bash
+      git clone https://github.com/your-repo/vlm-image-analysis.git
+      cd vlm-image-analysis
+      ```
+
+2. **Run the docker:**
+   
+
+After completing these steps, your setup should be complete and you can start using the project.
+
+## Inference
+
+Start the docker container: 
+```bash
+docker run --gpus all --rm -it --shm-size=8gb --memory="16g" --env="DISPLAY" -p 8077:7842 -p 7860:7860  -p 8501:8501 -v %cd%:/RAG -v %cd%:/root/sharedfolder -v %cd%/.cache:/root/.cache  lmdeploy-docker:latest bash
+```
+
+You should see something like:
+```
+==========
+== CUDA ==
+==========
+
+CUDA Version 12.1.1
+
+Container image Copyright (c) 2016-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+
+This container image and its contents are governed by the NVIDIA Deep Learning Container License.
+By pulling and using the container, you accept the terms and conditions of this license:
+https://developer.nvidia.com/ngc/nvidia-deep-learning-container-license
+
+A copy of this license is made available in this container at /NGC-DL-CONTAINER-LICENSE for your convenience.
+
+root@c6d710b08cde:/RAG/web#
+```
+
+To run the application:
+
+```bash
+root@c6d710b08cde:/RAG/web# python vlms.py
+```
+
+Which should output:
+```bash
+Running on local URL:  http://0.0.0.0:7860
+To create a public link, set `share=True` in `launch()`.
+```
+
+On initial use, the models are very slowly downloaded to a folder which is mapped externally using the docker -v command:
+```
+-v %cd%/.cache:/root/.cache
+ ```
+
+For instance:
+```
+root@c6d710b08cde:/RAG/web# ls -ls /root/.cache/huggingface/
+0 drwxr-xr-x 1 root root 512 Sep 12 06:15 models--OpenGVLab--InternVL2-1B
+0 drwxr-xr-x 1 root root 512 Sep 12 06:27 models--Qwen--Qwen2-VL-7B-Instruct
+0 drwxr-xr-x 1 root root 512 Sep 16 11:14 models--Qwen--Qwen2-VL-7B-Instruct-GPTQ-Int8
+0 drwxr-xr-x 1 root root 512 Sep 11 11:00 models--microsoft--Phi-3-vision-128k-instruct
+0 drwxr-xr-x 1 root root 512 Sep 15 06:02 models--vikhyatk--moondream2
+```
+
+## The Gradio APP
+
+The Gradio application provides a user-friendly interface for interacting with the VLM-Image-Analysis framework. It includes three main tabs: **VLM Model Selection**, **Device Settings**, and **Image Selection**. Each tab allows users to configure different aspects of the image analysis process.
+
+### 1. VLM Model Selection
+
+This tab allows users to select the Vision-and-Language Models (VLMs) they want to use, specify the directory of images to be analyzed, and choose prompts for generating textual descriptions. The interface includes:
+
+- **Model Selection**: A checkbox group where users can select from available models such as `Qwen/Qwen2-VL-7B-Instruct`, `microsoft/Phi-3-vision-128k-instruct`, `vikhyatk/moondream2`, and `Qwen/Qwen2-VL-7B-Instruct-GPTQ-Int8`.
+- **Image Directory Path**: A textbox to input the path to the directory containing images.
+- **Prompt Selection**: A checkbox group where users can select from predefined prompts that guide the analysis.
+- **Submit Button**: A button to run the inference based on the selected models, prompts, and image directory.
+
+The results of the inference are displayed in an HTML format, showing the generated descriptions, detected text, and translations.
+
+### 2. Device Settings
+
+<h1 align="center">    
+  <img src="static/004.png" width="75%"></a>  
+</h1>
+
+In this tab, users can configure settings related to the computational resources used for running the models:
+
+- **Device Map**: A radio button selection to choose the device map (`auto`, `cpu`, `cuda`). This determines whether the inference will use the GPU or CPU.
+- **Torch Dtype**: A radio button to select the data type for PyTorch (`torch.float16` or `torch.float32`). This affects the precision and performance of the model.
+- **Trust Remote Code**: A checkbox to indicate whether to trust remote code when loading models. This is relevant for models that require remote execution code.
+
+### 3. Image Selection
+
+This tab allows users to process images from the specified HF Dataset:
+```
+# List of available Hugging Face datasets
+dataset_options = [
+    "gokaygokay/panorama_hdr_dataset",  
+    "OpenGVLab/CRPE"  
+]
+```
+
+- **HF Image Dataset**: A textbox to input the path to the HF dataset containing images.
+- **Load a ZIP of Images Button**: A button to load and images from the specified ZIP file.
+
+## Docker Setup for LLMs
+
+### Base System
+
+The Docker image is based on the `pytorch/pytorch:2.2.2-cuda12.1-cudnn8-devel` image, which provides a PyTorch environment with CUDA 12.1 and cuDNN 8 for GPU acceleration. 
+
+### LLM-related Packages
+
+- **MLC LLM**: Installed from a nightly build URL, including `mlc-llm-nightly-cu122` for LLM services.
+- **AutoAWQ**: Cloned from GitHub and installed, which likely includes functionality for automatic quantization and weight compression.
+- **FlashAttention**: The package `flash-attn==2.6.1` is installed to optimize attention mechanisms in transformer models.
+- **Transformers**: The `transformers==4.33.0` package is used for working with pre-trained transformer models, along with `accelerate`, `diffusers`, `datasets`, `timm` for various model and dataset utilities.
+- **Other Libraries**: `chromadb`, `db-sqlite3`, `auto-gptq`, `exllama`, `sqlalchemy`, `optimum`, and `autoawq` for additional functionalities related to data handling, model optimization, and automatic weight quantization.
+
+### Jupyter Configuration
+
+- **Jupyter**: Jupyter Notebook and related packages are set up, including configuration through `jupyter_notebook_config.py` and a script `run_jupyter.sh`.
+
+### Exposed Ports
+
+The container exposes the following ports for accessing services:
+- `8097`, `7842`, `8501`, `8000`, `6666`, `7860`
+
+
+## Prompts
+
+The framework uses a variety of prompts to guide the Vision-and-Language Modeling (VLM) systems in generating descriptive outputs from images. Each prompt is designed to elicit detailed and specific information from the models.
+
+### Example of Available Prompts
+
+1. **Comprehensive Image Analysis**
+   - **Prompt**: 
+     ```
+     Thoroughly analyse the provided image and generate a comprehensive description of its content. This includes identifying and describing all objects, people, and activities visible in the image. Additionally, detect and extract any text present within the image using Optical Character Recognition (OCR), regardless of the language. Present the extracted text in a structured table format, including columns for the original text, its translation into English, and the language of the text.
+     ```
+   - **Rationale**: This prompt is designed to provide an exhaustive analysis of the image, encompassing both visual and textual information. It aims to capture every detail, making it suitable for applications where a complete understanding of the image is necessary, such as in detailed reporting or comprehensive data annotation.
+
+
+## Citation
+
+Please consider citing the work if you consider it useful for your research:
+
+```
+@book{judge-vlm,
+  author       = {Shlomo Kashani},
+  title        = {VLM-Image-Analysis: A Vision-and-Language Modeling Framework},
+  year         = {2024},  
+  github       = {https://huggingface.co/spaces/BoltzmannEntropy/vlms/}
+}
+```

app.py

@@ -0,0 +1,361 @@
+import os
+import base64
+import io
+import sqlite3
+import torch
+import gradio as gr
+import pandas as pd
+from PIL import Image
+import requests
+from transformers import Qwen2VLForConditionalGeneration, AutoProcessor, AutoModelForCausalLM, AutoTokenizer
+from huggingface_hub import hf_hub_download
+from datasets import load_dataset
+import traceback
+from tqdm import tqdm 
+import zipfile
+
+# Define constants for vikhyatk/moondream2 model
+MOON_DREAM_MODEL_ID = "vikhyatk/moondream2"
+MOON_DREAM_REVISION = "2024-08-26"
+
+# Define constants for the Qwen2-VL models
+QWEN2_VL_MODELS = [
+    'Qwen/Qwen2-VL-7B-Instruct',
+    'Qwen/Qwen2-VL-2B-Instruct-GPTQ-Int4',
+    'OpenGVLab/InternVL2-1B',
+    'Qwen/Qwen2-VL-72B',
+]
+
+# List of models to use (combining unique entries from available models and QWEN2_VL_MODELS)
+available_models = [
+    *QWEN2_VL_MODELS,  # Expands the QWEN2_VL_MODELS list into the available_models
+    'microsoft/Phi-3-vision-128k-instruct',
+    'vikhyatk/moondream2'
+]
+
+# List of available Hugging Face datasets
+dataset_options = [
+    "gokaygokay/panorama_hdr_dataset",  
+    "OpenGVLab/CRPE"  
+]
+
+# List of text prompts to use
+text_prompts = [
+    "Provide a detailed description of the image contents, including all visible objects, people, activities, and extract any text present within the image using Optical Character Recognition (OCR). Organize the extracted text in a structured table format with columns for original text, its translation into English, and the language it is written in.",
+    "Offer a thorough description of all elements within the image, from objects to individuals and their activities. Ensure any legible text seen in the image is extracted using Optical Character Recognition (OCR). Provide an accurate narrative that encapsulates the full content of the image.",    
+    "Create a four-sentence caption for the image. Start by specifying the style and type, such as painting, photograph, or digital art. In the next sentences, detail the contents and the composition clearly and concisely. Use language suited for prompting a text-to-image model, separating descriptive terms with commas instead of 'or'. Keep the description direct, avoiding interpretive phrases or abstract expressions",
+]
+
+# SQLite setup
+# def init_db():
+#     conn = sqlite3.connect('image_outputs.db')
+#     cursor = conn.cursor()
+#     cursor.execute('''
+#         CREATE TABLE IF NOT EXISTS image_outputs (
+#             id INTEGER PRIMARY KEY AUTOINCREMENT,
+#             image BLOB,
+#             prompt TEXT,
+#             output TEXT,
+#             model_name TEXT
+#         )
+#     ''')
+#     conn.commit()
+#     conn.close()
+
+def image_to_binary(image_path):
+    with open(image_path, 'rb') as file:
+        return file.read()
+
+# def store_in_db(image_path, prompt, output, model_name):
+#     conn = sqlite3.connect('image_outputs.db')
+#     cursor = conn.cursor()
+#     image_blob = image_to_binary(image_path)
+#     cursor.execute('''
+#         INSERT INTO image_outputs (image, prompt, output, model_name)
+#         VALUES (?, ?, ?, ?)
+#     ''', (image_blob, prompt, output, model_name))
+#     conn.commit()
+#     conn.close()
+
+# Function to encode an image to base64 for HTML display
+def encode_image(image):
+    img_buffer = io.BytesIO()
+    image.save(img_buffer, format="PNG")
+    img_str = base64.b64encode(img_buffer.getvalue()).decode("utf-8")
+    return f'<img src="data:image/png;base64,{img_str}" style="max-width:500px;"/>'
+
+# Function to load and display images from the panorama_hdr_dataset
+def load_dataset_images(dataset_name, num_images):
+    try:
+        dataset = load_dataset(dataset_name, split='train')
+        images = []
+        for i, item in enumerate(dataset[:num_images]):
+            if 'image' in item:
+                img = item['image']
+                print (type(img))
+                encoded_img = encode_image(img)
+                metadata = f"Width: {img.width}, Height: {img.height}"
+                if 'hdr' in item:
+                    metadata += f", HDR: {item['hdr']}"
+                images.append(f"<div style='display: inline-block; margin: 10px; text-align: center;'><h3>Image {i+1}</h3>{encoded_img}<p>{metadata}</p></div>")
+        if not images:
+            return "No images could be loaded from this dataset. Please check the dataset structure."
+        return "".join(images)
+    except Exception as e:
+        print(f"Error loading dataset: {e}")
+        traceback.print_exc()
+
+# Function to generate output
+def generate_output(model, processor, prompt, image, model_name, device):
+    try:
+        image_bytes = io.BytesIO()
+        image.save(image_bytes, format="PNG")
+        image_bytes = image_bytes.getvalue()
+
+        if model_name in QWEN2_VL_MODELS:
+            messages = [
+                {
+                    "role": "user",
+                    "content": [
+                        {"type": "image", "image": image_bytes},
+                        {"type": "text", "text": prompt},
+                    ]
+                }
+            ]
+            text = processor.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
+            inputs = processor(
+                text=[text],
+                images=[Image.open(io.BytesIO(image_bytes))],
+                padding=True,
+                return_tensors="pt",
+            )
+            inputs = {k: v.to(device) for k, v in inputs.items()}
+            generated_ids = model.generate(**inputs, max_new_tokens=1024)
+            generated_ids_trimmed = [out_ids[len(in_ids):] for in_ids, out_ids in zip(inputs['input_ids'], generated_ids)]
+            response_text = processor.batch_decode(generated_ids_trimmed, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
+            return response_text
+
+        elif model_name == 'microsoft/Phi-3-vision-128k-instruct':
+            messages = [{"role": "user", "content": f"<|image_1|>\n{prompt}"}]
+            prompt = processor.tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
+            inputs = processor(prompt, [image], return_tensors="pt")
+            inputs = {k: v.to(device) for k, v in inputs.items()}
+            generate_ids = model.generate(**inputs, eos_token_id=processor.tokenizer.eos_token_id, max_new_tokens=1024)
+            generate_ids = generate_ids[:, inputs['input_ids'].shape[1]:]
+            response_text = processor.batch_decode(generate_ids, skip_special_tokens=True)[0]
+            return response_text
+
+        elif model_name == 'vikhyatk/moondream2':
+            tokenizer = AutoTokenizer.from_pretrained(MOON_DREAM_MODEL_ID, revision=MOON_DREAM_REVISION)
+            enc_image = model.encode_image(image)
+            response_text = model.answer_question(enc_image, prompt, tokenizer)
+            return response_text
+    except Exception as e:
+        return f"Error during generation with model {model_name}: {e}"
+
+# Function to list and encode images from a directory
+def list_images(directory_path):
+    images = []
+    for filename in os.listdir(directory_path):
+        if filename.lower().endswith(('.png', '.jpg', '.jpeg')):
+            image_path = os.path.join(directory_path, filename)
+            encoded_img = encode_image(image_path)
+            images.append({
+                "filename": filename,
+                "image": encoded_img
+            })
+    return images
+
+# Function to extract images from a ZIP file
+# Function to extract images from a ZIP file
+def extract_images_from_zip(zip_file):
+    images = []
+    with zipfile.ZipFile(zip_file, 'r') as zip_ref:
+        for file_info in zip_ref.infolist():
+            if file_info.filename.lower().endswith(('.png', '.jpg', '.jpeg')):
+                with zip_ref.open(file_info) as file:
+                    try:
+                        img = Image.open(file)
+                        img = img.convert("RGB")  # Ensure the image is in RGB mode
+                        encoded_img = img
+                        images.append({
+                            "filename": file_info.filename,
+                            "image": encoded_img
+                        })
+                    except Exception as e:
+                        print(f"Error opening image {file_info.filename}: {e}")
+    return images
+
+# Gradio interface function for running inference
+def run_inference(model_names, dataset_input, num_images_input, prompts, device_map, torch_dtype, trust_remote_code,use_flash_attn, use_zip, zip_file):
+    data = []
+
+    torch_dtype_value = torch.float16 if torch_dtype == "torch.float16" else torch.float32
+    device_map_value = "cuda" if torch.cuda.is_available() else "cpu" if device_map == "auto" else device_map
+
+    model_processors = {}
+    for model_name in model_names:
+        try:
+            if model_name in QWEN2_VL_MODELS:
+                model = Qwen2VLForConditionalGeneration.from_pretrained(
+                    model_name, 
+                    torch_dtype=torch_dtype_value, 
+                    device_map=device_map_value
+                ).eval()
+                processor = AutoProcessor.from_pretrained(model_name)
+            elif model_name == 'microsoft/Phi-3-vision-128k-instruct':
+                model = AutoModelForCausalLM.from_pretrained(
+                    model_name, 
+                    device_map=device_map_value, 
+                    torch_dtype=torch_dtype_value, 
+                    trust_remote_code=trust_remote_code, 
+                    use_flash_attn=use_flash_attn
+                ).eval()
+                processor = AutoProcessor.from_pretrained(model_name, trust_remote_code=trust_remote_code)
+            elif model_name == 'vikhyatk/moondream2':
+                model = AutoModelForCausalLM.from_pretrained(
+                    MOON_DREAM_MODEL_ID, 
+                    trust_remote_code=True, 
+                    revision=MOON_DREAM_REVISION
+                ).eval()
+                processor = None  # No processor needed for this model
+
+            model_processors[model_name] = (model, processor)
+
+        except Exception as e:
+            print(f"Error loading model {model_name}: {e}")
+
+    try:
+        # Load images from the ZIP file if use_zip is True
+        if use_zip:
+            images = extract_images_from_zip(zip_file)
+            print ("Number of images in zip:" , len(images))
+            for img in tqdm(images):
+                try:
+                    img_data = img['image']
+                    if not isinstance(img_data, str):
+                        # Convert the Image object to a base64-encoded string
+                        img_buffer = io.BytesIO()
+                        img['image'].save(img_buffer, format="PNG")
+                        img_data = base64.b64encode(img_buffer.getvalue()).decode("utf-8")
+                            
+                    img_data=f'<img src="data:image/png;base64,{img_data}" style="max-width:500px;"/>'
+
+                    row_data = {"Image": img_data}  # Assuming encode_image is defined elsewhere
+                    for model_name in model_names:
+                        if model_name in model_processors:
+                            model, processor = model_processors[model_name]
+                            for prompt in prompts:
+                                try:
+                                    # Ensure image is defined
+                                    image = img['image']
+                                    response_text = generate_output(model, processor, prompt, image, model_name, device_map_value)
+                                    row_data[f"{model_name}_Response_{prompt}"] = response_text
+                                except Exception as e:
+                                    row_data[f"{model_name}_Response_{prompt}"] = f"Error during generation with model {model_name}: {e}"
+                                    traceback.print_exc()
+
+                    data.append(row_data)
+                except Exception as e:
+                    print(f"Error processing image {img['filename']}: {e}")
+                    traceback.print_exc()
+
+        # Load the dataset if use_zip is False
+        else:
+            dataset = load_dataset(dataset_input, split='train')
+            for i in tqdm(range(num_images_input)):
+                if dataset_input == "OpenGVLab/CRPE":
+                    image = dataset[i]['image']
+                elif dataset_input == "gokaygokay/panorama_hdr_dataset":
+                    image = dataset[i]['png_image']
+                else:
+                    image = dataset[i]['image']
+
+                encoded_img = encode_image(image)
+                row_data = {"Image": encoded_img}
+
+                for model_name in model_names:
+                    if model_name in model_processors:
+                        model, processor = model_processors[model_name]
+                        for prompt in prompts:
+                            try:
+                                response_text = generate_output(model, processor, prompt, image, model_name, device_map_value)
+                                row_data[f"{model_name}_Response_{prompt}"] = response_text
+                            except Exception as e:
+                                row_data[f"{model_name}_Response_{prompt}"] = f"Error during generation with model {model_name}: {e}"
+
+                data.append(row_data)
+
+    except Exception as e:
+        print(f"Error loading dataset: {e}")
+        traceback.print_exc()
+
+    return pd.DataFrame(data).to_html(escape=False)
+
+# Gradio UI setup
+def create_gradio_interface():
+    css = """
+      #output {
+        height: 500px;
+        overflow: auto;
+      }
+    """
+    with gr.Blocks(css=css) as demo:
+        # Title
+        gr.Markdown("# VLM-Image-Analysis: A Vision-and-Language Modeling Framework.")
+        gr.Markdown("""
+                    - Handle a batch of images from a ZIP file OR
+                    - Processes images from an HF DB 
+                    - Compatible with png, jpg, jpeg, and webp formats                                        
+                    - Compatibility with various AI models: Qwen2-VL-7B-Instruct, Qwen2-VL-2B-Instruct-GPTQ-Int4, InternVL2-1B, Qwen2-VL-72B, /Phi-3-vision-128k-instruct and moondream2""")
+
+        gr.Image(value="static/image.jpg", label="HF Image", width=300, height=300)  # Set custom width and height
+
+        with gr.Tab("VLM model and Dataset selection"):
+            gr.Markdown("### Dataset Selection: HF or from a ZIP file.")
+            with gr.Accordion("Advanced Settings", open=True):
+                with gr.Row():
+                    # with gr.Column():  
+                    use_zip_input = gr.Checkbox(label="Use ZIP File", value=False)
+                    dataset_input = gr.Dropdown(choices=dataset_options, label="Select Dataset", value=dataset_options[1], visible=True)
+                    num_images_input = gr.Radio(choices=[1, 5, 20], label="Number of Images", value=5)
+                    zip_file_input = gr.File(label="Upload ZIP File of Images", file_types=[".zip"])
+            gr.Markdown("### VLM Model Selection")
+            with gr.Row():
+                with gr.Column():  
+                    models_input = gr.CheckboxGroup(choices=available_models, label="Select Models", value=available_models[4])
+                    prompts_input = gr.CheckboxGroup(choices=text_prompts, label="Select Prompts", value=text_prompts[2])
+                    submit_btn = gr.Button("Run Inference")
+                
+            with gr.Row():        
+                output_display = gr.HTML(label="Results")
+
+        with gr.Tab("GPU Device Settings"):
+            device_map_input = gr.Radio(choices=["auto", "cpu", "cuda"], label="Device Map", value="auto")
+            torch_dtype_input = gr.Radio(choices=["torch.float16", "torch.float32"], label="Torch Dtype", value="torch.float16")
+            trust_remote_code_input = gr.Checkbox(label="Trust Remote Code", value=True)
+            use_flash_attn = gr.Checkbox(label="Use flash-attn 2 (Ampere GPUs or newer.)", value=False)
+            
+                               
+        def run_inference_wrapper(model_names, dataset_input, num_images_input, prompts, device_map, torch_dtype, trust_remote_code,use_flash_attn, use_zip, zip_file):
+            return run_inference(model_names, dataset_input, num_images_input, prompts, device_map, torch_dtype, trust_remote_code,use_flash_attn, use_zip, zip_file)
+        
+        def toggle_dataset_visibility(use_zip):
+            return gr.update(visible=not use_zip)
+        
+        submit_btn.click(
+            fn=run_inference_wrapper, 
+            inputs=[models_input, dataset_input, num_images_input, prompts_input, device_map_input, torch_dtype_input, trust_remote_code_input,use_flash_attn, use_zip_input, zip_file_input], 
+            outputs=output_display
+        )
+        
+        use_zip_input.change(
+            fn=toggle_dataset_visibility, 
+            inputs=use_zip_input, 
+            outputs=dataset_input
+        )
+
+    demo.launch(debug=True, share=False)
+
+if __name__ == "__main__":
+    create_gradio_interface()

build.bat

@@ -0,0 +1,2 @@
+docker build -t hf-docker .
+

hf-db-test.py

@@ -0,0 +1,22 @@
+from datasets import load_dataset
+import matplotlib.pyplot as plt
+
+# Specify dataset and number of images to load
+dataset_name = "OpenGVLab/CRPE"  # Can switch to "OpenGVLab/CRPE"
+num_images = 2  # Specify the number of images you want to load
+
+# Load the dataset
+dataset = load_dataset(dataset_name, split='train')
+
+# Check if the dataset is 'OpenGVLab/CRPE' or 'gokaygokay/panorama_hdr_dataset' and handle accordingly
+for i in range(num_images):
+    if dataset_name == "gokaygokay/panorama_hdr_dataset":
+        img = dataset[i]["png_image"]  # Access image for this dataset
+    elif dataset_name == "OpenGVLab/CRPE":
+        img = dataset[i]["image"]  # Access image for 'OpenGVLab/CRPE'
+    
+    # Display the image
+    plt.imshow(img)
+    plt.title(f"Image {i+1}")
+    plt.axis('off')  # Hide axes for better visualization
+    plt.show()

run-d.bat

@@ -0,0 +1 @@
+docker run --gpus all --rm -it --shm-size=8gb --memory="16g" --env="DISPLAY" -p 8077:7842 -p 7860:7860  -p 8501:8501 -v %cd%:/home/user/app -v %cd%:/home/user/sharedfolder -v %cd%/.cache:/home/user/.cache  hf-docker:latest bash