Deadmon commited on
Commit
aa14900
·
verified ·
1 Parent(s): 8d02f67

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +119 -99
app.py CHANGED
@@ -1,36 +1,86 @@
1
  import torch
2
  import spaces
3
- from diffusers import StableDiffusionPipeline, DDIMScheduler, AutoencoderKL
4
- from transformers import AutoFeatureExtractor
5
- from diffusers.pipelines.stable_diffusion.safety_checker import StableDiffusionSafetyChecker
6
- from ip_adapter.ip_adapter_faceid import IPAdapterFaceID, IPAdapterFaceIDPlus
7
  from huggingface_hub import hf_hub_download
8
  from insightface.app import FaceAnalysis
9
- from insightface.utils import face_align
10
  import gradio as gr
11
  import cv2
12
 
13
- base_model_paths = {
14
- "RealisticVisionV4": "SG161222/Realistic_Vision_V4.0_noVAE",
15
- "RealisticVisionV6": "SG161222/Realistic_Vision_V6.0_B1_noVAE",
16
- "Deliberate": "Yntec/Deliberate",
17
- "DeliberateV2": "Yntec/Deliberate2",
18
- "Dreamshaper8": "Lykon/dreamshaper-8",
19
- "EpicRealism": "emilianJR/epiCRealism"
20
  }
21
 
22
-
23
- vae_model_path = "stabilityai/sd-vae-ft-mse"
24
- image_encoder_path = "laion/CLIP-ViT-H-14-laion2B-s32B-b79K"
25
- ip_ckpt = hf_hub_download(repo_id="h94/IP-Adapter-FaceID", filename="ip-adapter-faceid_sd15.bin", repo_type="model")
26
- ip_plus_ckpt = hf_hub_download(repo_id="h94/IP-Adapter-FaceID", filename="ip-adapter-faceid-plusv2_sd15.bin", repo_type="model")
27
-
28
- safety_model_id = "CompVis/stable-diffusion-safety-checker"
29
- safety_feature_extractor = AutoFeatureExtractor.from_pretrained(safety_model_id)
30
- safety_checker = StableDiffusionSafetyChecker.from_pretrained(safety_model_id)
31
-
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
32
  device = "cuda"
33
 
 
34
  noise_scheduler = DDIMScheduler(
35
  num_train_timesteps=1000,
36
  beta_start=0.00085,
@@ -40,70 +90,54 @@ noise_scheduler = DDIMScheduler(
40
  set_alpha_to_one=False,
41
  steps_offset=1,
42
  )
43
- vae = AutoencoderKL.from_pretrained(vae_model_path).to(dtype=torch.float16)
44
 
45
- def load_model(base_model_path):
46
- pipe = StableDiffusionPipeline.from_pretrained(
47
- base_model_path,
 
48
  torch_dtype=torch.float16,
49
  scheduler=noise_scheduler,
50
- vae=vae,
51
- feature_extractor=safety_feature_extractor,
52
- safety_checker=None # <--- Disable safety checker
53
- ).to(device)
54
- return pipe
55
 
56
- ip_model = None
57
- ip_model_plus = None
 
 
58
 
59
- app = FaceAnalysis(name="buffalo_l", providers=['CPUExecutionProvider'])
60
- app.prepare(ctx_id=0, det_size=(640, 640))
 
61
 
62
- cv2.setNumThreads(1)
 
 
 
 
63
 
64
- @spaces.GPU(enable_queue=True)
65
- def generate_image(images, prompt, negative_prompt, preserve_face_structure, face_strength, likeness_strength, nfaa_negative_prompt, base_model, num_inference_steps, guidance_scale, width, height, progress=gr.Progress(track_tqdm=True)):
66
- global ip_model, ip_model_plus
67
- base_model_path = base_model_paths[base_model]
68
- pipe = load_model(base_model_path)
69
- ip_model = IPAdapterFaceID(pipe, ip_ckpt, device)
70
- ip_model_plus = IPAdapterFaceIDPlus(pipe, image_encoder_path, ip_plus_ckpt, device)
71
 
72
  faceid_all_embeds = []
73
- first_iteration = True
74
  for image in images:
75
  face = cv2.imread(image)
76
  faces = app.get(face)
77
  faceid_embed = torch.from_numpy(faces[0].normed_embedding).unsqueeze(0)
78
  faceid_all_embeds.append(faceid_embed)
79
- if(first_iteration and preserve_face_structure):
80
- face_image = face_align.norm_crop(face, landmark=faces[0].kps, image_size=224) # you can also segment the face
81
- first_iteration = False
82
-
83
  average_embedding = torch.mean(torch.stack(faceid_all_embeds, dim=0), dim=0)
84
-
85
- total_negative_prompt = f"{negative_prompt} {nfaa_negative_prompt}"
86
-
87
- if(not preserve_face_structure):
88
- print("Generating normal")
89
- image = ip_model.generate(
90
- prompt=prompt, negative_prompt=total_negative_prompt, faceid_embeds=average_embedding,
91
- scale=likeness_strength, width=width, height=height, num_inference_steps=num_inference_steps, guidance_scale=guidance_scale
92
- )
93
- else:
94
- print("Generating plus")
95
- image = ip_model_plus.generate(
96
- prompt=prompt, negative_prompt=total_negative_prompt, faceid_embeds=average_embedding,
97
- scale=likeness_strength, face_image=face_image, shortcut=True, s_scale=face_strength, width=width, height=height, num_inference_steps=num_inference_steps, guidance_scale=guidance_scale
98
- )
99
- print(image)
100
- return image
101
 
102
- def change_style(style):
103
- if style == "Photorealistic":
104
- return(gr.update(value=True), gr.update(value=1.3), gr.update(value=1.0))
105
- else:
106
- return(gr.update(value=True), gr.update(value=0.1), gr.update(value=0.8))
 
 
 
 
107
 
108
  def swap_to_gallery(images):
109
  return gr.update(value=images, visible=True), gr.update(visible=True), gr.update(visible=False)
@@ -113,48 +147,34 @@ def remove_back_to_files():
113
 
114
  css = '''
115
  h1{margin-bottom: 0 !important}
116
- footer{display:none !important}
117
  '''
118
 
119
  with gr.Blocks(css=css) as demo:
120
- gr.Markdown("")
121
- gr.Markdown("")
122
  with gr.Row():
123
  with gr.Column():
124
- files = gr.Files(
125
- label="Drag 1 or more photos of your face",
126
- file_types=["image"]
127
- )
128
- uploaded_files = gr.Gallery(label="Your images", visible=False, columns=5, rows=1, height=125)
129
  with gr.Column(visible=False) as clear_button:
130
- remove_and_reupload = gr.ClearButton(value="Remove and upload new ones", components=files, size="sm")
131
- prompt = gr.Textbox(label="Prompt",
132
- info="Try something like 'a photo of a man/woman/person'",
133
- placeholder="A photo of a [man/woman/person]...")
134
  negative_prompt = gr.Textbox(label="Negative Prompt", placeholder="low quality")
135
- style = gr.Radio(label="Generation type", info="For stylized try prompts like 'a watercolor painting of a woman'", choices=["Photorealistic", "Stylized"], value="Photorealistic")
136
- base_model = gr.Dropdown(label="Base Model", choices=list(base_model_paths.keys()), value="Realistic_Vision_V4.0_noVAE")
137
- submit = gr.Button("Submit")
138
- with gr.Accordion(open=False, label="Advanced Options"):
139
- preserve = gr.Checkbox(label="Preserve Face Structure", info="Higher quality, less versatility (the face structure of your first photo will be preserved). Unchecking this will use the v1 model.", value=True)
140
- face_strength = gr.Slider(label="Face Structure strength", info="Only applied if preserve face structure is checked", value=1.3, step=0.1, minimum=0, maximum=3)
141
- likeness_strength = gr.Slider(label="Face Embed strength", value=1.0, step=0.1, minimum=0, maximum=5)
142
- nfaa_negative_prompts = gr.Textbox(label="Appended Negative Prompts", info="Negative prompts to steer generations towards safe for all audiences outputs", value="naked, bikini, skimpy, scanty, bare skin, lingerie, swimsuit, exposed, see-through")
143
  num_inference_steps = gr.Slider(label="Number of Inference Steps", value=30, step=1, minimum=10, maximum=100)
144
  guidance_scale = gr.Slider(label="Guidance Scale", value=7.5, step=0.1, minimum=1, maximum=20)
145
  width = gr.Slider(label="Width", value=512, step=64, minimum=256, maximum=1024)
146
  height = gr.Slider(label="Height", value=512, step=64, minimum=256, maximum=1024)
 
147
  with gr.Column():
148
  gallery = gr.Gallery(label="Generated Images")
149
- style.change(fn=change_style,
150
- inputs=style,
151
- outputs=[preserve, face_strength, likeness_strength])
152
  files.upload(fn=swap_to_gallery, inputs=files, outputs=[uploaded_files, clear_button, files])
153
  remove_and_reupload.click(fn=remove_back_to_files, outputs=[uploaded_files, clear_button, files])
154
- submit.click(fn=generate_image,
155
- inputs=[files,prompt,negative_prompt,preserve, face_strength, likeness_strength, nfaa_negative_prompts, base_model, num_inference_steps, guidance_scale, width, height],
156
- outputs=gallery)
157
-
158
- gr.Markdown("")
159
-
160
  demo.launch()
 
1
  import torch
2
  import spaces
3
+ from diffusers import DDIMScheduler, StableDiffusionXLPipeline
4
+ import ipown
 
 
5
  from huggingface_hub import hf_hub_download
6
  from insightface.app import FaceAnalysis
 
7
  import gradio as gr
8
  import cv2
9
 
10
+ # List of models for switching
11
+ model_options = {
12
+ "CyberRealistic": "John6666/cyberrealistic-pony-v61-sdxl",
13
+ "StallionDreams": "John6666/stallion-dreams-pony-realistic-v1-sdxl",
14
+ "PonyRealism": "John6666/pony-realism-v21main-sdxl"
 
 
15
  }
16
 
17
+ # Define styles with prompts and negative prompts
18
+ style_list = [
19
+ {
20
+ "name": "(No style)",
21
+ "prompt": "{prompt}",
22
+ "negative_prompt": "longbody, lowres, bad anatomy, bad hands, missing fingers, extra digit, fewer digits, cropped, worst quality, low quality",
23
+ },
24
+ {
25
+ "name": "Cinematic",
26
+ "prompt": "cinematic still {prompt} . emotional, harmonious, vignette, highly detailed, high budget, bokeh, cinemascope, moody, epic, gorgeous, film grain, grainy",
27
+ "negative_prompt": "anime, cartoon, graphic, text, painting, crayon, graphite, abstract, glitch, deformed, mutated, ugly, disfigured",
28
+ },
29
+ {
30
+ "name": "3D Model",
31
+ "prompt": "professional 3d model {prompt} . octane render, highly detailed, volumetric, dramatic lighting",
32
+ "negative_prompt": "ugly, deformed, noisy, low poly, blurry, painting",
33
+ },
34
+ {
35
+ "name": "Anime",
36
+ "prompt": "anime artwork {prompt} . anime style, key visual, vibrant, studio anime, highly detailed",
37
+ "negative_prompt": "photo, deformed, black and white, realism, disfigured, low contrast",
38
+ },
39
+ {
40
+ "name": "Digital Art",
41
+ "prompt": "concept art {prompt} . digital artwork, illustrative, painterly, matte painting, highly detailed",
42
+ "negative_prompt": "photo, photorealistic, realism, ugly",
43
+ },
44
+ {
45
+ "name": "Photographic",
46
+ "prompt": "cinematic photo {prompt} . 35mm photograph, film, bokeh, professional, 4k, highly detailed",
47
+ "negative_prompt": "drawing, painting, crayon, sketch, graphite, impressionist, noisy, blurry, soft, deformed, ugly",
48
+ },
49
+ {
50
+ "name": "Pixel art",
51
+ "prompt": "pixel-art {prompt} . low-res, blocky, pixel art style, 8-bit graphics",
52
+ "negative_prompt": "sloppy, messy, blurry, noisy, highly detailed, ultra textured, photo, realistic",
53
+ },
54
+ {
55
+ "name": "Fantasy art",
56
+ "prompt": "ethereal fantasy concept art of {prompt} . magnificent, celestial, ethereal, painterly, epic, majestic, magical, fantasy art, cover art, dreamy",
57
+ "negative_prompt": "photographic, realistic, realism, 35mm film, dslr, cropped, frame, text, deformed, glitch, noise, noisy, off-center, deformed, cross-eyed, closed eyes, bad anatomy, ugly, disfigured, sloppy, duplicate, mutated, black and white",
58
+ },
59
+ {
60
+ "name": "Neonpunk",
61
+ "prompt": "neonpunk style {prompt} . cyberpunk, vaporwave, neon, vibes, vibrant, stunningly beautiful, crisp, detailed, sleek, ultramodern, magenta highlights, dark purple shadows, high contrast, cinematic, ultra detailed, intricate, professional",
62
+ "negative_prompt": "painting, drawing, illustration, glitch, deformed, mutated, cross-eyed, ugly, disfigured",
63
+ },
64
+ {
65
+ "name": "Manga",
66
+ "prompt": "manga style {prompt} . vibrant, high-energy, detailed, iconic, Japanese comic style",
67
+ "negative_prompt": "ugly, deformed, noisy, blurry, low contrast, realism, photorealistic, Western comic style",
68
+ },
69
+ ]
70
+
71
+ styles = {k["name"]: (k["prompt"], k["negative_prompt"]) for k in style_list}
72
+ STYLE_NAMES = list(styles.keys())
73
+ DEFAULT_STYLE_NAME = "(No style)"
74
+
75
+ def apply_style(style_name: str, positive: str, negative: str = "") -> tuple[str, str]:
76
+ p, n = styles.get(style_name, styles[DEFAULT_STYLE_NAME])
77
+ return p.replace("{prompt}", positive), n + negative
78
+
79
+ # Download the necessary model component
80
+ ip_ckpt = hf_hub_download(repo_id="h94/IP-Adapter-FaceID", filename="ip-adapter-faceid_sdxl.bin", repo_type="model")
81
  device = "cuda"
82
 
83
+ # Configure the noise scheduler
84
  noise_scheduler = DDIMScheduler(
85
  num_train_timesteps=1000,
86
  beta_start=0.00085,
 
90
  set_alpha_to_one=False,
91
  steps_offset=1,
92
  )
 
93
 
94
+ # Function to initialize the pipeline with a selected model
95
+ def get_pipeline(model_path):
96
+ return StableDiffusionXLPipeline.from_pretrained(
97
+ model_path,
98
  torch_dtype=torch.float16,
99
  scheduler=noise_scheduler,
100
+ use_safetensors=True,
101
+ )
 
 
 
102
 
103
+ # Initialize with a default model
104
+ current_model = model_options["PonyRealism"]
105
+ pipe = get_pipeline(current_model)
106
+ ip_model = ipown.IPAdapterFaceIDXL(pipe, ip_ckpt, device)
107
 
108
+ @spaces.GPU()
109
+ def generate_image(images, model_choice, style_name, prompt, negative_prompt, face_strength, likeness_strength, num_inference_steps, guidance_scale, width, height):
110
+ global current_model, pipe, ip_model
111
 
112
+ # Update the model if the choice has changed
113
+ if model_options[model_choice] != current_model:
114
+ current_model = model_options[model_choice]
115
+ pipe = get_pipeline(current_model)
116
+ ip_model = ipown.IPAdapterFaceIDXL(pipe, ip_ckpt, device)
117
 
118
+ torch.cuda.empty_cache()
119
+
120
+ app = FaceAnalysis(name="buffalo_l", providers=['CUDAExecutionProvider', 'CPUExecutionProvider'])
121
+ app.prepare(ctx_id=0, det_size=(512, 512))
 
 
 
122
 
123
  faceid_all_embeds = []
 
124
  for image in images:
125
  face = cv2.imread(image)
126
  faces = app.get(face)
127
  faceid_embed = torch.from_numpy(faces[0].normed_embedding).unsqueeze(0)
128
  faceid_all_embeds.append(faceid_embed)
129
+
 
 
 
130
  average_embedding = torch.mean(torch.stack(faceid_all_embeds, dim=0), dim=0)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
131
 
132
+ # Apply style to the prompt and negative prompt
133
+ prompt, negative_prompt = apply_style(style_name, prompt, negative_prompt)
134
+
135
+ image = ip_model.generate(
136
+ prompt=prompt, negative_prompt=negative_prompt, faceid_embeds=average_embedding,
137
+ scale=likeness_strength, width=width, height=height, guidance_scale=guidance_scale, num_inference_steps=num_inference_steps
138
+ )
139
+
140
+ return image
141
 
142
  def swap_to_gallery(images):
143
  return gr.update(value=images, visible=True), gr.update(visible=True), gr.update(visible=False)
 
147
 
148
  css = '''
149
  h1{margin-bottom: 0 !important}
 
150
  '''
151
 
152
  with gr.Blocks(css=css) as demo:
153
+ gr.Markdown("# IP-Adapter-FaceID SDXL demo")
154
+ gr.Markdown("A simple Demo for the [h94/IP-Adapter-FaceID SDXL model](https://huggingface.co/h94/IP-Adapter-FaceID).")
155
  with gr.Row():
156
  with gr.Column():
157
+ model_dropdown = gr.Dropdown(label="Select Model", choices=list(model_options.keys()), value="PonyRealism")
158
+ style_dropdown = gr.Dropdown(label="Select Style", choices=STYLE_NAMES, value=DEFAULT_STYLE_NAME)
159
+ files = gr.Files(label="Drag 1 or more photos of your face", file_types=["image"])
160
+ uploaded_files = gr.Gallery(label="Your images", visible=False, columns=5, rows=1, height=250)
 
161
  with gr.Column(visible=False) as clear_button:
162
+ remove_and_reupload = gr.ClearButton(value="Remove files and upload new ones", components=files, size="sm")
163
+ prompt = gr.Textbox(label="Prompt", placeholder="A photo of a man/woman/person ...")
 
 
164
  negative_prompt = gr.Textbox(label="Negative Prompt", placeholder="low quality")
165
+ face_strength = gr.Slider(label="Face Strength", value=7.5, step=0.1, minimum=0, maximum=15)
166
+ likeness_strength = gr.Slider(label="Likeness Strength", value=1.0, step=0.1, minimum=0, maximum=5)
167
+ with gr.Accordion("Advanced Options", open=False):
 
 
 
 
 
168
  num_inference_steps = gr.Slider(label="Number of Inference Steps", value=30, step=1, minimum=10, maximum=100)
169
  guidance_scale = gr.Slider(label="Guidance Scale", value=7.5, step=0.1, minimum=1, maximum=20)
170
  width = gr.Slider(label="Width", value=512, step=64, minimum=256, maximum=1024)
171
  height = gr.Slider(label="Height", value=512, step=64, minimum=256, maximum=1024)
172
+ submit = gr.Button("Submit", variant="primary")
173
  with gr.Column():
174
  gallery = gr.Gallery(label="Generated Images")
175
+
 
 
176
  files.upload(fn=swap_to_gallery, inputs=files, outputs=[uploaded_files, clear_button, files])
177
  remove_and_reupload.click(fn=remove_back_to_files, outputs=[uploaded_files, clear_button, files])
178
+ submit.click(fn=generate_image, inputs=[files, model_dropdown, style_dropdown, prompt, negative_prompt, face_strength, likeness_strength, num_inference_steps, guidance_scale, width, height], outputs=gallery)
179
+
 
 
 
 
180
  demo.launch()