add diffusers weights (#1)

- add diffusers weights + code example (d19a8536869c33313ca169e1819b1040944a0eac)


Co-authored-by: YiYi Xu <[email protected]>

README.md

@@ -91,6 +91,52 @@ Which should give you an image like below:
 
 ![An adorable fluffy pastel creature](sample_result.png)
 
+### Using Controlnets in Diffusers
+
+Make sure you upgrade to the latest version of diffusers: `pip install -U diffusers`. And then you can run:
+
+```python
+import torch
+from diffusers import StableDiffusion3ControlNetPipeline, SD3ControlNetModel
+from diffusers.utils import load_image
+from diffusers.image_processor import VaeImageProcessor
+
+class SD3CannyImageProcessor(VaeImageProcessor):
+    def __init__(self):
+        super().__init__(do_normalize=False)
+    def preprocess(self, image, **kwargs):
+        image = super().preprocess(image, **kwargs)
+        image = image * 255 * 0.5 + 0.5
+        return image
+    def postprocess(self, image, do_denormalize=True, **kwargs):
+        do_denormalize = [True] * image.shape[0]
+        image = super().postprocess(image, **kwargs, do_denormalize=do_denormalize)
+        return image
+
+controlnet = SD3ControlNetModel.from_pretrained("stabilityai/stable-diffusion-3.5-large-controlnet-canny", torch_dtype=torch.float16)
+pipe = StableDiffusion3ControlNetPipeline.from_pretrained(
+    "stabilityai/stable-diffusion-3.5-large",
+    controlnet=controlnet,
+    torch_dtype=torch.float16
+).to("cuda")
+pipe.image_processor = SD3CannyImageProcessor()
+
+control_image = load_image("https://huggingface.co/datasets/diffusers/diffusers-images-docs/resolve/main/canny.png")
+prompt =  "A Night time photo taken by Leica M11, portrait of a Japanese woman in a kimono, looking at the camera, Cherry blossoms"
+
+generator = torch.Generator(device="cpu").manual_seed(0)
+image = pipe(
+    prompt, 
+    control_image=control_image, 
+    controlnet_conditioning_scale=1.0,
+    guidance_scale=3.5,
+    num_inference_steps=60,
+    generator=generator,
+    max_sequence_length=77,
+).images[0]
+image.save(f'canny-8b.jpg')
+
+```
 
 ### Preprocessing
 

config.json

@@ -0,0 +1,21 @@
+{
+  "_class_name": "SD3ControlNetModel",
+  "_diffusers_version": "0.32.0.dev0",
+  "attention_head_dim": 64,
+  "caption_projection_dim": 2048,
+  "dual_attention_layers": [],
+  "extra_conditioning_channels": 0,
+  "force_zeros_for_pooled_projection": false,
+  "in_channels": 16,
+  "joint_attention_dim": null,
+  "num_attention_heads": 38,
+  "num_layers": 19,
+  "out_channels": 16,
+  "patch_size": 2,
+  "pooled_projection_dim": 2048,
+  "pos_embed_max_size": null,
+  "pos_embed_type": null,
+  "qk_norm": null,
+  "sample_size": 128,
+  "use_pos_embed": false
+}

diffusion_pytorch_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5f2608a8af55f223250398e04e3c497aa194ddbcb70049f556194c4a2d333865
+size 8614110992