init update

.gitattributes

@@ -33,3 +33,39 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+ckpts/sd3_medium_adapter.pth filter=lfs diff=lfs merge=lfs -text
+ckpts/flux_adapter.pth filter=lfs diff=lfs merge=lfs -text
+ckpts/sd3.5_medium_adapter.pth filter=lfs diff=lfs merge=lfs -text
+examples/input_imgs/style_imgs/misc-stainedglass.png filter=lfs diff=lfs merge=lfs -text
+examples/output_imgs_flux/subject_imgs_wo_bg/bear_plushie_13.png filter=lfs diff=lfs merge=lfs -text
+examples/output_imgs_sd3_medium/style_imgs/bridge_artstyle-psychedelic.png filter=lfs diff=lfs merge=lfs -text
+examples/output_imgs_sd3_medium/subject_imgs_wo_bg/bear_plushie_13.png filter=lfs diff=lfs merge=lfs -text
+examples/input_imgs/subject_imgs_w_bg/cat.png filter=lfs diff=lfs merge=lfs -text
+examples/output_imgs_flux/style_imgs/bicycle_misc-stainedglass.png filter=lfs diff=lfs merge=lfs -text
+examples/output_imgs_sd3.5_medium/style_imgs/bicycle_misc-stainedglass.png filter=lfs diff=lfs merge=lfs -text
+examples/output_imgs_sd3.5_medium/subject_imgs_wo_bg/teapot_0.png filter=lfs diff=lfs merge=lfs -text
+examples/input_imgs/subject_imgs_w_bg/dog7.png filter=lfs diff=lfs merge=lfs -text
+examples/output_imgs_flux/editing/1.png filter=lfs diff=lfs merge=lfs -text
+examples/output_imgs_flux/pixel_control_imgs/depth_3.png filter=lfs diff=lfs merge=lfs -text
+examples/input_imgs/style_imgs/artstyle-psychedelic.png filter=lfs diff=lfs merge=lfs -text
+examples/output_imgs_flux/style_imgs/bridge_artstyle-psychedelic.png filter=lfs diff=lfs merge=lfs -text
+examples/output_imgs_flux/understand/depth_0.png filter=lfs diff=lfs merge=lfs -text
+examples/output_imgs_sd3.5_medium/subject_imgs_w_bg/cat_8.png filter=lfs diff=lfs merge=lfs -text
+examples/output_imgs_sd3.5_medium/subject_imgs_w_bg/dog7_2.png filter=lfs diff=lfs merge=lfs -text
+examples/output_imgs_flux/subject_imgs_w_bg/cat_8.png filter=lfs diff=lfs merge=lfs -text
+examples/output_imgs_sd3_medium/style_imgs/bicycle_misc-stainedglass.png filter=lfs diff=lfs merge=lfs -text
+examples/input_imgs/editing/0.png filter=lfs diff=lfs merge=lfs -text
+examples/input_imgs/subject_imgs_wo_bg/bear_plushie.png filter=lfs diff=lfs merge=lfs -text
+examples/input_imgs/subject_imgs_wo_bg/teapot.png filter=lfs diff=lfs merge=lfs -text
+examples/input_imgs/understand/0.png filter=lfs diff=lfs merge=lfs -text
+examples/output_imgs_flux/editing/0.png filter=lfs diff=lfs merge=lfs -text
+examples/output_imgs_sd3.5_medium/subject_imgs_wo_bg/bear_plushie_13.png filter=lfs diff=lfs merge=lfs -text
+examples/output_imgs_sd3_medium/pixel_control_imgs/depth_3.png filter=lfs diff=lfs merge=lfs -text
+examples/output_imgs_flux/subject_imgs_w_bg/dog7_2.png filter=lfs diff=lfs merge=lfs -text
+examples/output_imgs_flux/subject_imgs_wo_bg/teapot_0.png filter=lfs diff=lfs merge=lfs -text
+examples/output_imgs_sd3.5_medium/pixel_control_imgs/depth_3.png filter=lfs diff=lfs merge=lfs -text
+examples/output_imgs_sd3_medium/subject_imgs_wo_bg/teapot_0.png filter=lfs diff=lfs merge=lfs -text
+examples/input_imgs/editing/1.png filter=lfs diff=lfs merge=lfs -text
+examples/input_imgs/pixel_control_imgs/depth_3.png filter=lfs diff=lfs merge=lfs -text
+examples/output_imgs_sd3.5_medium/style_imgs/bridge_artstyle-psychedelic.png filter=lfs diff=lfs merge=lfs -text
+examples/output_imgs_sd3.5_medium/understand/depth_0.png filter=lfs diff=lfs merge=lfs -text

LICENSE

@@ -0,0 +1,5 @@
+MIT License
+Copyright 2025 AIDC-AI
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

NOTICE

@@ -0,0 +1,6 @@
+Copyright (C) 2025 AIDC-AI
+Licensed under the MIT License (the "License").
+The models were trained based on the following models, and if you use our model in conjunction with the model(s) below, you must review the license(s) of the models and comply with all of their terms respectively:
+1. stable-diffusion-3.5-medium (https://huggingface.co/stabilityai/stable-diffusion-3.5-medium), license: STABILITY AI COMMUNITY LICENSE AGREEMENT (https://huggingface.co/stabilityai/stable-diffusion-3.5-medium/blob/main/LICENSE.md). This Stability AI Model is licensed under the Stability AI Community License, Copyright © Stability AI Ltd. All Rights Reserved.
+2. stable-diffusion-3-medium (https://huggingface.co/stabilityai/stable-diffusion-3-medium), license: STABILITY AI COMMUNITY LICENSE AGREEMENT (https://huggingface.co/stabilityai/stable-diffusion-3-medium/blob/main/LICENSE.md). This Stability AI Model is licensed under the Stability AI Community License, Copyright © Stability AI Ltd. All Rights Reserved.
+3. flux (https://github.com/black-forest-labs/flux), License: FLUX.1 [dev] Non-Commercial License v1.1.1 (https://github.com/black-forest-labs/flux/blob/main/model_licenses/LICENSE-FLUX1-dev). The FLUX.1 [dev] Model is licensed by Black Forest Labs. Inc. under the FLUX.1 [dev] Non-Commercial License. Copyright Black Forest Labs. Inc.  IN NO EVENT SHALL BLACK FOREST LABS, INC. BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH USE OF THIS MODEL.

README.md

@@ -1,3 +1,109 @@
----
-license: mit
----
+# UNIC-Adapter: Unified Image-Instruction Adapter for Multimodal Image Generation
+[![arXiv](https://img.shields.io/badge/arXiv-Paper-A42C25.svg)](https://arxiv.org/abs/2412.18928)
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+[![Github](https://img.shields.io/badge/GitHub-AIDC--AI/UNIC-Adapter-blue?style=flat&logo=github)](https://github.com/AIDC-AI/UNIC-Adapter)  
+
+UNIC-Adapter is a unified image-instruction adapter that integrates multimodal instructions for controllable image generation. This repository contains the official implementation for the CVPR 2025 paper "UNIC-Adapter: Unified Image-instruction Adapter with Multi-modal Transformer for Image Generation".
+
+In this repository, we release a model based on SD3 Medium, which supports the tasks described in our paper. In addition, we also provide two further models: one built on SD3.5 Medium, which is capable of traditional computer vision perception tasks, and another on FLUX.1-dev, which supports both instruction-based image editing and traditional computer vision perception tasks.
+
+## Generated samples
+
+### Pixel-level Control
+*(Left: Condition image, Center left: SD3 Medium with UNIC-Adapter, Center right: SD3.5 Medium with UNIC-Adapter, Right: FLUX.1-dev with UNIC-Adapter)*
+
+<div float="left">
+  <img src='./examples/input_imgs/pixel_control_imgs/depth_3.png' width='24%'/>
+  <img src='./examples/output_imgs_sd3_medium/pixel_control_imgs/depth_3.png' width='24%' />
+  <img src='./examples/output_imgs_sd3.5_medium/pixel_control_imgs/depth_3.png' width='24%' />
+  <img src='./examples/output_imgs_flux/pixel_control_imgs/depth_3.png' width='24%' />
+</div>
+
+### Subject-driven Generation
+*(Left: Condition image, Center left: SD3 Medium with UNIC-Adapter, Center right: SD3.5 Medium with UNIC-Adapter, Right: FLUX.1-dev with UNIC-Adapter)*
+
+<div float="left">
+  <img src='./examples/input_imgs/subject_imgs_wo_bg/bear_plushie.png' width='24%'/>
+  <img src='./examples/output_imgs_sd3_medium/subject_imgs_wo_bg/bear_plushie_13.png' width='24%' />
+  <img src='./examples/output_imgs_sd3.5_medium/subject_imgs_wo_bg/bear_plushie_13.png' width='24%' />
+  <img src='./examples/output_imgs_flux/subject_imgs_wo_bg/bear_plushie_13.png' width='24%' />
+</div>
+
+<div float="left">
+  <img src='./examples/input_imgs/subject_imgs_wo_bg/teapot.png' width='24%'/>
+  <img src='./examples/output_imgs_sd3_medium/subject_imgs_wo_bg/teapot_0.png' width='24%' />
+  <img src='./examples/output_imgs_sd3.5_medium/subject_imgs_wo_bg/teapot_0.png' width='24%' />
+  <img src='./examples/output_imgs_flux/subject_imgs_wo_bg/teapot_0.png' width='24%' />
+</div>
+
+*(Left: condition image, Center: SD 3.5 medium with UNIC-Adapter, Right: FLUX.1-dev with UNIC-Adapter)*
+<div float="left">
+  <img src='./examples/input_imgs/subject_imgs_w_bg/cat.png' width='24%'/>
+  <img src='./examples/output_imgs_sd3.5_medium/subject_imgs_w_bg/cat_8.png' width='24%' />
+  <img src='./examples/output_imgs_flux/subject_imgs_w_bg/cat_8.png' width='24%' />
+</div>
+
+<div float="left">
+  <img src='./examples/input_imgs/subject_imgs_w_bg/dog7.png' width='24%'/>
+  <img src='./examples/output_imgs_sd3.5_medium/subject_imgs_w_bg/dog7_2.png' width='24%' />
+  <img src='./examples/output_imgs_flux/subject_imgs_w_bg/dog7_2.png' width='24%' />
+</div>
+
+### Style-driven Generation
+*(Left: Condition image, Center left: SD3 Medium with UNIC-Adapter, Center right: SD3.5 Medium with UNIC-Adapter, Right: FLUX.1-dev with UNIC-Adapter)*
+
+<div float="left">
+  <img src='./examples/input_imgs/style_imgs/misc-stainedglass.png' width='24%'/>
+  <img src='./examples/output_imgs_sd3_medium/style_imgs/bicycle_misc-stainedglass.png' width='24%' />
+  <img src='./examples/output_imgs_sd3.5_medium/style_imgs/bicycle_misc-stainedglass.png' width='24%' />
+  <img src='./examples/output_imgs_flux/style_imgs/bicycle_misc-stainedglass.png' width='24%' />
+</div>
+
+<div float="left">
+  <img src='./examples/input_imgs/style_imgs/artstyle-psychedelic.png' width='24%'/>
+  <img src='./examples/output_imgs_sd3_medium/style_imgs/bridge_artstyle-psychedelic.png' width='24%' />
+  <img src='./examples/output_imgs_sd3.5_medium/style_imgs/bridge_artstyle-psychedelic.png' width='24%' />
+  <img src='./examples/output_imgs_flux/style_imgs/bridge_artstyle-psychedelic.png' width='24%' />
+</div>
+
+### Image Understanding 
+*(Left: Source image, Center: SD3.5 Medium with UNIC-Adapter, Right: FLUX.1-dev with UNIC-Adapter)*
+
+<div float="left">
+  <img src='./examples/input_imgs/understand/0.png' width='32%'/>
+  <img src='./examples/output_imgs_sd3.5_medium/understand/depth_0.png' width='32%'/>
+  <img src='./examples/output_imgs_flux/understand/depth_0.png' width='32%' />
+</div>
+
+
+### Image Editing
+*(Left: Source image, Right: FLUX.1-dev with UNIC-Adapter)*
+
+<div float="left">
+  <img src='./examples/input_imgs/editing/0.png' width='32%'/>
+  <img src='./examples/output_imgs_flux/editing/0.png' width='32%' />
+</div>
+<div float="left">
+  <img src='./examples/input_imgs/editing/1.png' width='32%'/>
+  <img src='./examples/output_imgs_flux/editing/1.png' width='32%' />
+</div>
+
+## License
+This project is licensed under the MIT License (SPDX-License-Identifier: MIT). 
+The models cannot be used independently. If you use our model in conjunction with the Flux model, you must review the [FLUX.1 [dev] Non-Commercial License](https://github.com/black-forest-labs/flux/blob/main/model_licenses/LICENSE-FLUX1-dev) of the Flux model and comply with all of its terms; If you use our model in conjunction with the stable-diffusion-3-medium model, then you must review the [STABILITY AI COMMUNITY LICENSE AGREEMENT](https://huggingface.co/stabilityai/stable-diffusion-3-medium/blob/main/LICENSE.md) of the SD3 model and comply with all of its terms; If you use our model in conjunction with the stable-diffusion-3.5-medium model, then you must review the [STABILITY AI COMMUNITY LICENSE AGREEMENT](https://huggingface.co/stabilityai/stable-diffusion-3.5-medium/blob/main/LICENSE.md) of the SD3.5 model and comply with all of its terms.
+
+## Citation
+If you find this repo is helpful for your research, please cite our paper:
+```bibtex
+@inproceedings{duan2025unic,
+  title={UNIC-Adapter: Unified Image-instruction Adapter with Multi-modal Transformer for Image Generation},
+  author={Duan, Lunhao and Zhao, Shanshan and Yan, Wenjun and Li, Yinglun and Chen, Qing-Guo and Xu, Zhao and Luo, Weihua and Zhang, Kaifu and Gong, Mingming and Xia, Gui-Song},
+  booktitle={Proceedings of the Computer Vision and Pattern Recognition Conference},
+  pages={7963--7973},
+  year={2025}
+}
+```
+
+## Disclaimer
+We used compliance checking algorithms during the training process, to ensure the compliance of the trained model(s) to the best of our ability. Due to complex data and the diversity of language model usage scenarios, we cannot guarantee that the model is completely free of copyright issues or improper content. If you believe anything infringes on your rights or generates improper content, please contact us, and we will promptly address the matter.
+

ckpts/flux_adapter.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:38adc64c23538ee4f4c34ba7daa1529073d3ce716180eae2fbb230a906042830
+size 11569600681

ckpts/sd3.5_medium_adapter.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bdcd67ceaaafc3f7e1c69c8198996b85e065357e1464e0d507bc6184ada3aa07
+size 3268194086

ckpts/sd3_medium_adapter.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5b9ac27eb691f0dee8744b4f5d529f3f4937ca5bf068837d9206c04c7745f7eb
+size 2498739674