woshin/FLUX.1-Canny-dev-redux

Introduction

This model is fine-tuned based on black-forest-labs/FLUX.1-canny-dev.
The model enhances the performance when used with black-forest-labs/FLUX.1-Redux-dev.

Examples

shoe/clothe generate demo

texture generate demo

finetuned vs original

Inference

import os
import torch
from diffusers import FluxPriorReduxPipeline, FluxPipeline, FluxControlPipeline, FluxTransformer2DModel
from diffusers.utils import load_image
from PIL import Image

pipe_prior_redux = FluxPriorReduxPipeline.from_pretrained(
  "black-forest-labs/FLUX.1-Redux-dev", 
  torch_dtype=torch.bfloat16).to("cuda")

pipe = FluxControlPipeline.from_pretrained(
  "black-forest-labs/FLUX.1-Canny-dev", 
  torch_dtype=torch.bfloat16
).to("cuda")

transformer = FluxTransformer2DModel.from_pretrained(
  "woshin/FLUX.1-Canny-dev-redux",
  subfolder="transformer",
  torch_dtype=torch.bfloat16
).to("cuda")

pipe.transformer = transformer


def inference_control_redux(style_image, control_image, save_image, size=(1024, 1024)):
  style_image = load_image(style_image).resize(size)
  control_image = load_image(control_image).resize(size)
  pipe_prior_redux_out = pipe_prior_redux(style_image, return_dict=False)
  prompt_embeds, pooled_prompt_embeds = pipe_prior_redux_out[0], pipe_prior_redux_out[1]
  image = pipe(
    control_image=control_image,
    height=size[1],
    width=size[0],
    num_inference_steps=50,
    guidance_scale=30.0,
    prompt_embeds=prompt_embeds,
    pooled_prompt_embeds=pooled_prompt_embeds,
  ).images[0]

  dst = Image.new('RGB', (size[0] * 3, size[1]))
  dst.paste(style_image, (0, 0))
  dst.paste(control_image, (size[0], 0))
  dst.paste(image, (size[0] * 2, 0))
  dst.save(save_image)

style_image = "style.png" # download from this repo
control_image = "control.png"
save_image = "save.png"
inference_control_redux(style_image, control_image, save_image)