Update pipeline_stable_diffusion_3_ipa.py

pipeline_stable_diffusion_3_ipa.py

@@ -1148,72 +1148,51 @@ class StableDiffusion3Pipeline(DiffusionPipeline, SD3LoraLoaderMixin, FromSingle
             print('Using primary image.')
             clip_image = clip_image.resize((max(clip_image.size), max(clip_image.size)))
             #clip_image_embeds_1 = self.encode_clip_image_emb(clip_image, device, dtype)
-            clip_image_embeds_1 = self.clip_image_processor(images=clip_image, return_tensors="pt").pixel_values
-            print('clip output size: ', clip_image_embeds_1.shape)
-            clip_image_embeds_1 = clip_image_embeds_1.to(device, dtype=dtype)
-            clip_image_embeds_1 = self.image_encoder(clip_image_embeds_1, output_hidden_states=True).hidden_states[-2]
+            with torch.inference_mode():
+                clip_image_embeds_1 = self.clip_image_processor(images=clip_image, return_tensors="pt").pixel_values
+                print('clip output size: ', clip_image_embeds_1.shape)
+                clip_image_embeds_1 = clip_image_embeds_1.to(device, dtype=dtype)
+                clip_image_embeds_1 = self.image_encoder(clip_image_embeds_1, output_hidden_states=True).hidden_states[-2]
             print('encoder output size: ', clip_image_embeds_1.shape)
             clip_image_embeds_1 = clip_image_embeds_1 * scale_1
             image_prompt_embeds_list.append(clip_image_embeds_1)
         if clip_image_2 != None:
             print('Using secondary image.')
             clip_image_2 = clip_image_2.resize((max(clip_image_2.size), max(clip_image_2.size)))
-            #clip_image_embeds_2 = self.encode_clip_image_emb(clip_image, device, dtype)
-            clip_image_embeds_2 = self.clip_image_processor(images=clip_image_2, return_tensors="pt").pixel_values
-            clip_image_embeds_2 = clip_image_embeds_2.to(device, dtype=dtype)
-            clip_image_embeds_2 = self.image_encoder(clip_image_embeds_2, output_hidden_states=True).hidden_states[-2]
+            with torch.inference_mode():
+                clip_image_embeds_2 = self.clip_image_processor(images=clip_image_2, return_tensors="pt").pixel_values
+                clip_image_embeds_2 = clip_image_embeds_2.to(device, dtype=dtype)
+                clip_image_embeds_2 = self.image_encoder(clip_image_embeds_2, output_hidden_states=True).hidden_states[-2]
             clip_image_embeds_2 = clip_image_embeds_2 * scale_2
             image_prompt_embeds_list.append(clip_image_embeds_2)
         if clip_image_3 != None:
             print('Using tertiary image.')
             clip_image_3 = clip_image_3.resize((max(clip_image_3.size), max(clip_image_3.size)))
-            #clip_image_embeds_3 = self.encode_clip_image_emb(clip_image, device, dtype)
-            clip_image_embeds_3 = self.clip_image_processor(images=clip_image_3, return_tensors="pt").pixel_values
-            clip_image_embeds_3 = clip_image_embeds_3.to(device, dtype=dtype)
-            clip_image_embeds_3 = self.image_encoder(clip_image_embeds_3, output_hidden_states=True).hidden_states[-2]
+            with torch.inference_mode():
+                clip_image_embeds_3 = self.clip_image_processor(images=clip_image_3, return_tensors="pt").pixel_values
+                clip_image_embeds_3 = clip_image_embeds_3.to(device, dtype=dtype)
+                clip_image_embeds_3 = self.image_encoder(clip_image_embeds_3, output_hidden_states=True).hidden_states[-2]
             clip_image_embeds_3 = clip_image_embeds_3 * scale_3
             image_prompt_embeds_list.append(clip_image_embeds_3)
         if clip_image_4 != None:
             print('Using quaternary image.')
             clip_image_4 = clip_image_4.resize((max(clip_image_4.size), max(clip_image_4.size)))
-            #clip_image_embeds_4 = self.encode_clip_image_emb(clip_image, device, dtype)
-            clip_image_embeds_4 = self.clip_image_processor(images=clip_image_4, return_tensors="pt").pixel_values
-            clip_image_embeds_4 = clip_image_embeds_4.to(device, dtype=dtype)
-            clip_image_embeds_2 = self.image_encoder(clip_image_embeds_4, output_hidden_states=True).hidden_states[-2]
+            with torch.inference_mode():
+                clip_image_embeds_4 = self.clip_image_processor(images=clip_image_4, return_tensors="pt").pixel_values
+                clip_image_embeds_4 = clip_image_embeds_4.to(device, dtype=dtype)
+                clip_image_embeds_2 = self.image_encoder(clip_image_embeds_4, output_hidden_states=True).hidden_states[-2]
             clip_image_embeds_4 = clip_image_embeds_4 * scale_4
             image_prompt_embeds_list.append(clip_image_embeds_4)
         if clip_image_5 != None:
             print('Using quinary image.')
             clip_image_5 = clip_image_5.resize((max(clip_image_5.size), max(clip_image_5.size)))
-            #clip_image_embeds_5 = self.encode_clip_image_emb(clip_image, device, dtype)
-            clip_image_embeds_5 = self.clip_image_processor(images=clip_image_5, return_tensors="pt").pixel_values
-            clip_image_embeds_5 = clip_image_embeds_5.to(device, dtype=dtype)
-            clip_image_embeds_5 = self.image_encoder(clip_image_embeds_5, output_hidden_states=True).hidden_states[-2]
+            with torch.inference_mode():
+                clip_image_embeds_5 = self.clip_image_processor(images=clip_image_5, return_tensors="pt").pixel_values
+                clip_image_embeds_5 = clip_image_embeds_5.to(device, dtype=dtype)
+                clip_image_embeds_5 = self.image_encoder(clip_image_embeds_5, output_hidden_states=True).hidden_states[-2]
             clip_image_embeds_5 = clip_image_embeds_5 * scale_5
             image_prompt_embeds_list.append(clip_image_embeds_5)
             
-        # Concatenate the image embeddings
-       ## clip_image_embeds = torch.mean(torch.stack(image_prompt_embeds_list), dim=0)
-
-        
-       # clip_image_embeds = torch.cat(image_prompt_embeds_list, dim=0).mean(dim=0) #.unsqueeze(0)
-        #bs_embed, seq_len = clip_image_embeds.shape
-        #clip_image_embeds = clip_image_embeds.view(bs_embed, seq_len)  # Simplified reshape
-
-        # experimental way
-        #clip_image_embeds = torch.cat([torch.zeros_like(torch.stack(image_prompt_embeds_list)), torch.stack(image_prompt_embeds_list)], dim=0).mean(dim=0)
-        # FAILS clip_image_embeds = torch.cat(torch.stack(image_prompt_embeds_list), dim=0).mean(dim=0)
-        # FAILS TIMESTEPS clip_image_embeds = torch.cat(image_prompt_embeds_list, dim=0).mean(dim=0)
-
-        clip_image_embeds_stack_list = torch.stack(image_prompt_embeds_list).mean(dim=0)
-        print('stacked with mean dim 0 shape: ', clip_image_embeds_stack_list.shape)
-        zeroes_tensor = torch.zeros_like(clip_image_embeds_stack_list)
-        print('zeros shape: ', zeroes_tensor.shape)
-        clip_image_embeds = torch.cat([zeroes_tensor, clip_image_embeds_stack_list], dim=0)
-        print('embeds shape old: ', clip_image_embeds.shape)
-
-
-
         clip_image_embeds_cat_list = torch.cat(image_prompt_embeds_list).mean(dim=0)
         print('catted embeds list with mean: ',clip_image_embeds_cat_list.shape)
         seq_len, _ = clip_image_embeds_cat_list.shape
@@ -1224,45 +1203,7 @@ class StableDiffusion3Pipeline(DiffusionPipeline, SD3LoraLoaderMixin, FromSingle
         zeros_tensor = torch.zeros_like(clip_image_embeds_view)
         print('zeros: ',zeros_tensor.shape)
         clip_image_embeds = torch.cat([zeros_tensor, clip_image_embeds_view], dim=0)
-        print('embeds shape new: ', clip_image_embeds.shape)
-
-        # 1. Stack the image embeddings
-        stacked_image_embeds = torch.cat(image_prompt_embeds_list, dim=1)
-        print('shape 1: ', stacked_image_embeds.shape)
-        # 2. Calculate the mean of the stacked embeddings
-        average_image_embed = torch.mean(stacked_image_embeds, dim=0) #.unsqueeze(0)  # Add batch dimension after averaging
-        print('shape 2: ', average_image_embed.shape)
-        # 3. Create a tensor of zeros with the same shape as the averaged embedding
-        zeros_tensor = torch.zeros_like(average_image_embed)
-        #print('shape 3: ', zeros_tensor.shape)
-        zeros_tensor_repeat = zeros_tensor.repeat(1, 1, 1)
-        print('shape 3.1: ', zeros_tensor_repeat.shape)
-        clip_image_embeds_repeat = average_image_embed.repeat(1, 1, 1)
-        print('shape 3.5: ', clip_image_embeds_repeat.shape)
-        # 4. Concatenate the zeros and the average embedding
-        clip_image_embeds_cat = torch.cat([zeros_tensor, average_image_embed], dim=0)
-        print('shape 4: ', clip_image_embeds_cat.shape)
-        clip_image_embeds_cat_repeat = clip_image_embeds_cat.repeat(1, 1, 1)
-        print('shape 4.1: ', clip_image_embeds_cat_repeat.shape)
-        clip_image_embeds_repeat_cat = torch.cat([zeros_tensor_repeat, clip_image_embeds_repeat], dim=0)
-        print('shape 4a: ', clip_image_embeds_repeat_cat.shape)
-        clip_image_embeds_repeat_cat_1 = torch.cat([zeros_tensor_repeat, clip_image_embeds_repeat], dim=1)
-        print('shape 4b: ', clip_image_embeds_repeat_cat_1.shape)
-        #clip_image_embeds = clip_image_embeds_repeat_cat
-        '''
-        #clip_image_embeds = clip_image_embeds.unsqueeze(0) # Add a dimension at the beginning so now you have [1, 2*seq_len_img, embed_dim_img]
-        print('shape 5: ', clip_image_embeds.shape)
-
-        bs_embed, seq_len, _ = clip_image_embeds.shape
-
-        clip_image_embedsa = clip_image_embeds.view(bs_embed, 1, -1)
-        print('shape 7: ', clip_image_embedsa.shape)
-        clip_image_embedsb = clip_image_embeds.view(seq_len, -1)
-        print('shape 8: ', clip_image_embedsb.shape)
-        
-        clip_image_embeds = clip_image_embedsb
-        '''
-        #clip_image_embeds = torch.cat([torch.stack(image_prompt_embeds_list)], dim=0).mean(dim=0)
+        print('embeds shape: ', clip_image_embeds.shape)
         
         # 4. Prepare timesteps
         timesteps, num_inference_steps = retrieve_timesteps(self.scheduler, num_inference_steps, device, timesteps)