Update app.py

app.py

@@ -49,6 +49,11 @@ pipe_edit = StableDiffusionXLInstructPix2PixPipeline.from_single_file( edit_file
 pipe_edit.scheduler = EDMEulerScheduler(sigma_min=0.002, sigma_max=120.0, sigma_data=1.0, prediction_type="v_prediction")
 pipe_edit.to("cuda")
 
+from transformers import BlipProcessor, BlipForConditionalGeneration
+
+processor = BlipProcessor.from_pretrained("unography/blip-long-cap")
+model = BlipForConditionalGeneration.from_pretrained("unography/blip-long-cap", torch_dtype=torch.float16).to("cuda")
+
 # Generator
 @spaces.GPU(duration=30, queue=False)
 def king(type ,
@@ -66,6 +71,11 @@ def king(type ,
         progress=gr.Progress(track_tqdm=True),
     ):
     if type=="Image Editing" :
+        raw_image = Image.open(input_image).convert('RGB')
+        inputs = processor(raw_image, return_tensors="pt").to("cuda", torch.float16)
+        out = model.generate(**inputs, min_length=10, max_length=25)
+        caption = processor.decode(out[0], skip_special_tokens=True)
+        instructions = f"{instruction} {caption} {instruction}"
         if randomize_seed:
             seed = random.randint(0, 99999)
         text_cfg_scale = text_cfg_scale
@@ -74,13 +84,13 @@ def king(type ,
         steps=steps
         generator = torch.manual_seed(seed)
         output_image = pipe_edit(
-            instruction, image=input_image,
+            instructions, image=input_image,
             guidance_scale=text_cfg_scale, image_guidance_scale=image_cfg_scale,
             num_inference_steps=steps, generator=generator, output_type="latent",
         ).images         
 
         refine = refiner(
-            prompt=instruction,
+            prompt=instruction2,
             guidance_scale=guidance_scale,
             num_inference_steps=steps,
             image=output_image,