File size: 7,375 Bytes
3825412 0fb31b6 3825412 6ad3f54 3825412 bc107af 3825412 40ecef1 3825412 addf0e4 3825412 a12c70b 3825412 2e77b98 3825412 119e0d0 3825412 119e0d0 3825412 bbc7ba0 3825412 ecc3776 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 |
import gradio as gr
import os
from PIL import Image
from pymongo import MongoClient
import requests
from io import BytesIO
from dotenv import load_dotenv
instruction_beginning = """
## π Evaluation of AI Quality
\n**Background**
\nIn this task, you will evaluate the quality of image edits based on a textual instruction and two input images regarding three aspects: **Instruction-Edit Alignment**, **Visual Quality** and **Consistency**.
Each aspect should be rated on a scale from 1 to 10, where 1 indicates 'very poor' and 10 represents 'excellent'.
Please ensure you have read the detailed instructions provided in this [document](https://www.canva.com/design/DAGP0UTTygI/rYkYZtLUipuKbXPbRcj9kQ/edit?utm_content=DAGP0UTTygI&utm_campaign=designshare&utm_medium=link2&utm_source=sharebutton) before starting the labeling process.
\n**Dataset Source**
This project uses the [MagicBrush dataset](https://osu-nlp-group.github.io/MagicBrush/) (dev split), released under the CC-BY-4.0 license.
\n**Labeling**
\nPlease enter a nickname to avoid repeating image pairs. Make sure to remember this nickname for future sessions. If you choose not to enter a nickname, your ratings will not be saved.
"""
alignment_info = """
How well does the edited area align with the text instruction? (e.g. numbers, colors, and objects)
"""
quality_info = """
How realistic and aesthetically pleasing is the edited area? (e.g. color realism and overall aesthetics)
"""
consistency_info = """
How seamlessly does the edit integrate with the rest of the original image? (e.g. consistency in style, lighting, logic, and spatial coherence)
"""
overall_info = """
How do you perceive and like the edit as a whole, how well does it meet your expectations and complements the original image?
"""
# load_dotenv()
mongo_user = os.getenv('MONGO_USER')
mongo_password = os.getenv('MONGO_PASSWORD')
cluster_url = os.getenv('MONGO_CLUSTER_URL')
gradio_user = os.getenv('GRADIO_USER')
gradio_password = os.getenv('GRADIO_PASSWORD')
connection_url = f"mongodb+srv://{mongo_user}:{mongo_password}@{cluster_url}"
client = MongoClient(connection_url)
db = client["thesis"]
collection = db["labeling"]
def download_image(url):
"""Download image from a given URL."""
response = requests.get(url)
response.raise_for_status()
return Image.open(BytesIO(response.content))
def fetch_random_entry(annotator):
"""Fetch a random entry from the database that hasn't been rated by the specified annotator."""
pipeline = [
{
"$match": {
"ratings.rater": {"$ne": annotator} # exclude entries where rater is the specified annotator
}
},
{"$sample": {"size": 1}} # randomly select one entry
]
results = list(collection.aggregate(pipeline))
return results[0] if results else None
def save_rating(entry_id, turn, annotator, alignment, quality, consistency, overall):
"""Save the given ratings into the database."""
if annotator and entry_id != '' and turn != '':
rating = {
"rater": annotator,
"alignment": alignment,
"quality": quality,
"consistency": consistency,
"overall": overall
}
collection.update_one(
{"meta_information.id": int(entry_id), "meta_information.turn": int(turn)},
{"$push": {"ratings": rating}}
)
def count_labeled_images(annotator):
"""Count how many images a person has labeled based on the 'ratings' field."""
pipeline = [
{
"$match": {
"ratings.rater": annotator # where 'rater' is the given annotator
}
},
{
"$count": "labeled_images" # count the number of documents that match
}
]
result = list(collection.aggregate(pipeline))
return result[0]['labeled_images'] if result else 0
def prepare_next_image(annotator):
"""Fetch the next image and its metadata."""
entry = fetch_random_entry(annotator)
if not entry:
return None, None, None, None, "No more images to rate!", None
meta_info = entry["meta_information"]
input_image = download_image(meta_info["input_img_link"])
output_image = download_image(meta_info["output_img_link"])
instruction = meta_info["instruction"]
progress_message = f"**Rate this image edit! ({count_labeled_images(annotator)}/528 labeled)**"
return meta_info["id"], input_image, output_image, instruction, progress_message, meta_info["turn"]
def start(annotator):
return prepare_next_image(annotator)
def record_input(id, turn, annotator, alignment, quality, consistency, overall):
save_rating(id, turn, annotator, alignment, quality, consistency, overall)
img_id, img_block1, img_block2, prompt, progress_text, turn = prepare_next_image(annotator)
return img_id, img_block1, img_block2, prompt, progress_text, turn, 5, 5, 5, 5
# Gradio Interface
def create_interface():
with gr.Blocks(theme=gr.themes.Origin()) as demo:
gr.Markdown(instruction_beginning)
# annotator = gr.Textbox(label="Nickname", interactive=True)
annotator = gr.Textbox(label="Annotator Nickname")
start_btn = gr.Button("Start", variant="primary")
progress_text = gr.Markdown("Waiting to start.")
# progress_text = gr.Markdown("You have labeled **0** out of 528 potential images.")
with gr.Row():
img_block1 = gr.Image(visible=True, width=300, height=300, label="Original Image", interactive=False)
img_block2 = gr.Image(visible=True, width=300, height=300, label="Edited Image", interactive=False)
prompt = gr.Textbox(label="Instruction", visible=True, interactive=False)
img_id = gr.Textbox(visible=False)
turn = gr.Textbox(visible=False)
with gr.Row():
slider_alignment = gr.Slider(label="Instruction-Edit Alignment", minimum=0, maximum=10, step=1, value=5,
info=alignment_info)
slider_quality = gr.Slider(label="Visual Quality", minimum=0, maximum=10, step=1, value=5,
info=quality_info)
slider_consistency = gr.Slider(label="Consistency", minimum=0, maximum=10, step=1, value=5,
info=consistency_info)
slider_overall = gr.Slider(label="Overall Impression", minimum=0, maximum=10, step=1, value=5,
info=overall_info)
save_and_continue_btn = gr.Button("Save & Continue", variant="primary")
start_btn.click(
fn=start,
inputs=[annotator],
outputs=[img_id, img_block1, img_block2, prompt, progress_text, turn]
)
save_and_continue_btn.click(
fn=record_input,
inputs=[img_id, turn, annotator, slider_alignment, slider_quality, slider_consistency, slider_overall],
outputs=[img_id, img_block1, img_block2, prompt, progress_text, turn,
slider_alignment, slider_quality, slider_consistency, slider_overall]
)
return demo
if __name__ == "__main__":
demo = create_interface()
demo.queue()
#demo.launch(share=True, debug=True, auth=(gradio_user, gradio_password))
demo.launch(share=True, debug=True)
|