Update app.py

app.py

@@ -412,9 +412,7 @@ def generate_30(
         del processor5
         gc.collect()
         torch.cuda.empty_cache()
-        expand_prompt(prompt)
-        expand_prompt(caption)
-        expanded = expand_prompt(caption_2)
+        expanded = expand_prompt(prompt+caption+caption_2)
         expanded_1 = expanded[0]
         expanded_2 = expanded[1]
         global model
@@ -428,7 +426,6 @@ def generate_30(
         pipe.text_encoder=text_encoder_1.to(device=device, dtype=torch.bfloat16)
         pipe.text_encoder_2=text_encoder_2.to(device=device, dtype=torch.bfloat16)
         #pipe.unet=unetX.to(device=device, dtype=torch.bfloat16)
-        
         print('-- generating image --')
         sd_image = ip_model.generate(
                 pil_image_1=sd_image_a,
@@ -491,35 +488,82 @@ def generate_60(
     samples=1,
     progress=gr.Progress(track_tqdm=True)  # Add progress as a keyword argument
 ):
-    pipe.text_encoder=text_encoder_1
-    pipe.text_encoder_2=text_encoder_2
+    global captioner_2
+    captioner2=captioner_2
     seed = random.randint(0, MAX_SEED)
     generator = torch.Generator(device='cuda').manual_seed(seed)
     if latent_file is not None:  # Check if a latent file is provided
-        sd_image_a = Image.open(latent_file.name)
+        sd_image_a = Image.open(latent_file.name).convert('RGB')
+        sd_image_a.resize((height,width), Image.LANCZOS)
+        caption=[]
+        caption_2=[]
+        #caption.append(captioner(sd_image_a))
+        caption.append(captioner2(sd_image_a))
+        #caption.append(captioner_3(sd_image_a))
+        caption_2.append(captioning(sd_image_a))
         if latent_file_2 is not None:  # Check if a latent file is provided
-            sd_image_b = Image.open(latent_file_2.name)
+            sd_image_b = Image.open(latent_file_2.name).convert('RGB')
             sd_image_b.resize((height,width), Image.LANCZOS)
+            #caption.append(captioner(sd_image_b))
+            caption.append(captioner2(sd_image_b))
+            #caption.append(captioner_3(sd_image_b))
+            caption_2.append(captioning(sd_image_b))
         else:
             sd_image_b = None
         if latent_file_3 is not None:  # Check if a latent file is provided
-            sd_image_c = Image.open(latent_file_3.name)
+            sd_image_c = Image.open(latent_file_3.name).convert('RGB')
             sd_image_c.resize((height,width), Image.LANCZOS)
+            #caption.append(captioner(sd_image_c))
+            caption.append(captioner2(sd_image_c))
+            #caption.append(captioner_3(sd_image_c))
+            caption_2.append(captioning(sd_image_c))
         else:
             sd_image_c = None
         if latent_file_4 is not None:  # Check if a latent file is provided
-            sd_image_d = Image.open(latent_file_4.name)
+            sd_image_d = Image.open(latent_file_4.name).convert('RGB')
             sd_image_d.resize((height,width), Image.LANCZOS)
+            #caption.append(captioner(sd_image_d))
+            caption.append(captioner2(sd_image_d))
+            #caption.append(captioner_3(sd_image_d))
+            caption_2.append(captioning(sd_image_d))
         else:
             sd_image_d = None
         if latent_file_5 is not None:  # Check if a latent file is provided
-            sd_image_e = Image.open(latent_file_5.name)
+            sd_image_e = Image.open(latent_file_5.name).convert('RGB')
             sd_image_e.resize((height,width), Image.LANCZOS)
+            #caption.append(captioner(sd_image_e))
+            caption.append(captioner2(sd_image_e))
+            #caption.append(captioner_3(sd_image_e))
+            caption_2.append(captioning(sd_image_e))
         else:
             sd_image_e = None
         timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
         filename= f'rv_IP_{timestamp}.png'
         print("-- using image file --")
+        print(caption)
+        print(caption_2)
+        print("-- generating further caption --")
+        global model5
+        global processor5
+        del captioner2
+        del model5
+        del processor5
+        gc.collect()
+        torch.cuda.empty_cache()
+        expanded = expand_prompt(prompt+caption+caption_2)
+        expanded_1 = expanded[0]
+        expanded_2 = expanded[1]
+        global model
+        global txt_tokenizer
+        del model
+        del txt_tokenizer
+        gc.collect()
+        torch.cuda.empty_cache()
+        global text_encoder_1
+        global text_encoder_2
+        pipe.text_encoder=text_encoder_1.to(device=device, dtype=torch.bfloat16)
+        pipe.text_encoder_2=text_encoder_2.to(device=device, dtype=torch.bfloat16)
+        #pipe.unet=unetX.to(device=device, dtype=torch.bfloat16)
         print('-- generating image --')
         sd_image = ip_model.generate(
                 pil_image_1=sd_image_a,
@@ -527,7 +571,8 @@ def generate_60(
                 pil_image_3=sd_image_c,
                 pil_image_4=sd_image_d,
                 pil_image_5=sd_image_e,
-                prompt=prompt,
+                prompt=prompt+' '+expanded_1,
+                prompt_2=expanded_2,
                 negative_prompt=negative_prompt,
                 text_scale=text_scale,
                 ip_scale=ip_scale,
@@ -581,44 +626,91 @@ def generate_90(
     samples=1,
     progress=gr.Progress(track_tqdm=True)  # Add progress as a keyword argument
 ):
-    pipe.text_encoder=text_encoder_1
-    pipe.text_encoder_2=text_encoder_2
+    global captioner_2
+    captioner2=captioner_2
     seed = random.randint(0, MAX_SEED)
     generator = torch.Generator(device='cuda').manual_seed(seed)
     if latent_file is not None:  # Check if a latent file is provided
-        sd_image_a = Image.open(latent_file.name)
+        sd_image_a = Image.open(latent_file.name).convert('RGB')
+        sd_image_a.resize((height,width), Image.LANCZOS)
+        caption=[]
+        caption_2=[]
+        #caption.append(captioner(sd_image_a))
+        caption.append(captioner2(sd_image_a))
+        #caption.append(captioner_3(sd_image_a))
+        caption_2.append(captioning(sd_image_a))
         if latent_file_2 is not None:  # Check if a latent file is provided
-            sd_image_b = Image.open(latent_file_2.name)
+            sd_image_b = Image.open(latent_file_2.name).convert('RGB')
             sd_image_b.resize((height,width), Image.LANCZOS)
+            #caption.append(captioner(sd_image_b))
+            caption.append(captioner2(sd_image_b))
+            #caption.append(captioner_3(sd_image_b))
+            caption_2.append(captioning(sd_image_b))
         else:
             sd_image_b = None
         if latent_file_3 is not None:  # Check if a latent file is provided
-            sd_image_c = Image.open(latent_file_3.name)
+            sd_image_c = Image.open(latent_file_3.name).convert('RGB')
             sd_image_c.resize((height,width), Image.LANCZOS)
+            #caption.append(captioner(sd_image_c))
+            caption.append(captioner2(sd_image_c))
+            #caption.append(captioner_3(sd_image_c))
+            caption_2.append(captioning(sd_image_c))
         else:
             sd_image_c = None
         if latent_file_4 is not None:  # Check if a latent file is provided
-            sd_image_d = Image.open(latent_file_4.name)
+            sd_image_d = Image.open(latent_file_4.name).convert('RGB')
             sd_image_d.resize((height,width), Image.LANCZOS)
+            #caption.append(captioner(sd_image_d))
+            caption.append(captioner2(sd_image_d))
+            #caption.append(captioner_3(sd_image_d))
+            caption_2.append(captioning(sd_image_d))
         else:
             sd_image_d = None
         if latent_file_5 is not None:  # Check if a latent file is provided
-            sd_image_e = Image.open(latent_file_5.name)
+            sd_image_e = Image.open(latent_file_5.name).convert('RGB')
             sd_image_e.resize((height,width), Image.LANCZOS)
+            #caption.append(captioner(sd_image_e))
+            caption.append(captioner2(sd_image_e))
+            #caption.append(captioner_3(sd_image_e))
+            caption_2.append(captioning(sd_image_e))
         else:
             sd_image_e = None
         timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
         filename= f'rv_IP_{timestamp}.png'
         print("-- using image file --")
+        print(caption)
+        print(caption_2)
+        print("-- generating further caption --")
+        global model5
+        global processor5
+        del captioner2
+        del model5
+        del processor5
+        gc.collect()
+        torch.cuda.empty_cache()
+        expanded = expand_prompt(prompt+caption+caption_2)
+        expanded_1 = expanded[0]
+        expanded_2 = expanded[1]
+        global model
+        global txt_tokenizer
+        del model
+        del txt_tokenizer
+        gc.collect()
+        torch.cuda.empty_cache()
+        global text_encoder_1
+        global text_encoder_2
+        pipe.text_encoder=text_encoder_1.to(device=device, dtype=torch.bfloat16)
+        pipe.text_encoder_2=text_encoder_2.to(device=device, dtype=torch.bfloat16)
+        #pipe.unet=unetX.to(device=device, dtype=torch.bfloat16)
         print('-- generating image --')
-        #with torch.no_grad():
         sd_image = ip_model.generate(
                 pil_image_1=sd_image_a,
                 pil_image_2=sd_image_b,
                 pil_image_3=sd_image_c,
                 pil_image_4=sd_image_d,
                 pil_image_5=sd_image_e,
-                prompt=prompt,
+                prompt=prompt+' '+expanded_1,
+                prompt_2=expanded_2,
                 negative_prompt=negative_prompt,
                 text_scale=text_scale,
                 ip_scale=ip_scale,