Update app.py

app.py

@@ -16,7 +16,10 @@ import time
 import os
 from ip_adapter import IPAdapterXL
 from image_gen_aux import UpscaleWithModel
-from huggingface_hub import snapshot_download
+from huggingface_hub import hf_hub_download
+from models.transformer_sd3 import SD3Transformer2DModel
+from pipeline_stable_diffusion_3_ipa import StableDiffusion3Pipeline
+from PIL import Image
 
 FTP_HOST = "1ink.us"
 FTP_USER = "ford442"
@@ -35,6 +38,9 @@ torch.set_float32_matmul_precision("highest")
 
 hftoken = os.getenv("HF_AUTH_TOKEN") 
 
+image_encoder_path = "google/siglip-so400m-patch14-384"
+ipadapter_path = hf_hub_download(repo_id="InstantX/SD3.5-Large-IP-Adapter", filename="ip-adapter.bin")
+
 def upload_to_ftp(filename):
     try:
         transport = paramiko.Transport((FTP_HOST, 22))
@@ -57,7 +63,13 @@ checkpoint = "microsoft/Phi-3.5-mini-instruct"
 vae = AutoencoderKL.from_pretrained("ford442/sdxl-vae-bf16")
 #vae = AutoencoderKL.from_pretrained("ford442/sdxl-vae-bf16")
 
-pipe = StableDiffusion3Pipeline.from_pretrained("ford442/stable-diffusion-3.5-medium-bf16").to(device=torch.device("cuda:0"), dtype=torch.bfloat16)
+transformer = SD3Transformer2DModel.from_pretrained(
+    model_path, 
+    subfolder="transformer", 
+    torch_dtype=torch.bfloat16
+)
+
+pipe = StableDiffusion3Pipeline.from_pretrained("ford442/stable-diffusion-3.5-medium-bf16", transformer=transformer).to(device=torch.device("cuda:0"), dtype=torch.bfloat16)
 #pipe = StableDiffusion3Pipeline.from_pretrained("ford442/stable-diffusion-3.5-medium-bf16").to(torch.device("cuda:0"))
 #pipe = StableDiffusion3Pipeline.from_pretrained("ford442/RealVis_Medium_1.0b_bf16", torch_dtype=torch.bfloat16)
 #pipe = StableDiffusion3Pipeline.from_pretrained("stabilityai/stable-diffusion-3.5-medium", token=hftoken, torch_dtype=torch.float32, device_map='balanced')
@@ -89,26 +101,14 @@ model = AutoModelForCausalLM.from_pretrained(checkpoint, device_map='balanced')
 
 
 
-repo_id = "ford442/SDXL-IP_ADAPTER"
-subfolder = "image_encoder"
-subfolder2 = "ip_adapter"
-
-# Download the entire repository
-local_repo_path = snapshot_download(repo_id=repo_id, repo_type="model")
+pipe.init_ipadapter(
+    ip_adapter_path=ipadapter_path, 
+    image_encoder_path=image_encoder_path, 
+    nb_token=64, 
+)
 
-# Construct the paths to the subfolders
-local_folder = os.path.join(local_repo_path, subfolder)
-local_folder2 = os.path.join(local_repo_path, subfolder2) # Path to the ip_adapter dir
 
-print(f"Image encoder downloaded to: {local_folder}")
-print(f"IP Adapter files downloaded to: {local_folder2}")
 
-# Construct the path to the ip-adapter_sdxl.bin file
-#ip_ckpt = os.path.join(local_folder2, "ip-adapter_sdxl.bin") # Correct path
-ip_ckpt = os.path.join(local_folder2, "ip-adapter_sdxl_vit-h.bin") # Correct path
-
-print(f"IP Adapter checkpoint path: {ip_ckpt}")
-ip_model = IPAdapterXL(pipe, local_folder, ip_ckpt, device)
 
 upscaler_2 = UpscaleWithModel.from_pretrained("Kim2091/ClearRealityV1").to(torch.device("cuda:0"))
 
@@ -221,14 +221,14 @@ def infer(
         print("-- using image file --")
         print('-- generating image --')
         #with torch.no_grad():
-        sd_image = ip_model.generate(
-            prompt=enhanced_prompt,
-            pil_image=sd_image_a,
-            num_samples=1,
-            num_inference_steps=num_inference_steps,
-            guidance_scale=guidance_scale,
-            seed=seed
-        )
+        result = pipe(
+            clip_image=image,
+            prompt=prompt,
+            ipadapter_scale=scale,
+            width=width,
+            height=height,
+            generator=torch.Generator().manual_seed(seed)
+        ).images[0]
         rv_path = f"sd35_{seed}.png"
         sd_image[0].save(rv_path,optimize=False,compress_level=0)
         upload_to_ftp(rv_path)