We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Error when using blockwise scales with sd_xl_offset_example-lora_1.0.safetensors
### Describe the bug

When loading the [sd_xl_offset_example-lora_1.0.safetensors](https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0/blob/main/sd_xl_offset_example-lora_1.0.safetensors) from the [official repository](https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0/) and setting the scales by blocks, it throws this error:

`RuntimeError: No LoRA weight found for module down_blocks.0.resnets.0.conv1`

Loading the same lora with just a float e.g.: `pipe.set_adapters(["offset"], [1.0])` doesn't raise the error.

### Reproduction

```python
import torch

from diffusers import StableDiffusionXLPipeline


pipe = StableDiffusionXLPipeline.from_pretrained(
    "stabilityai/stable-diffusion-xl-base-1.0", torch_dtype=torch.float16, variant="fp16"
).to("cuda")

pipe.load_lora_weights(
    "stabilityai/stable-diffusion-xl-base-1.0",
    weight_name="sd_xl_offset_example-lora_1.0.safetensors",
    adapter_name="offset",
)
offset_scales = {
    "unet": {
        "down": {"block_1": [1.0, 1.0], "block_2": [1.0, 1.0]},
        "mid": 0,
        "up": {"block_0": [1.0, 1.0, 1.0], "block_1": [1.0, 1.0, 1.0]},
    },
    "text_encoder": 1.0,
    "text_encoder_2": 1.0,
}
pipe.set_adapters(["offset"], [offset_scales])

prompt = "high quality photo of a panther in the jungle, night, dark, low light, dim"
generator = torch.Generator(device="cpu").manual_seed(899041154)

image = pipe(
    prompt=prompt,
    negative_prompt="",
    guidance_scale=6.5,
    num_inference_steps=25,
    generator=generator,
).images[0]
image.save("result.png")
```

### Logs

_No response_

### System Info

- `diffusers` version: 0.28.0.dev0
- Platform: Linux-6.8.7-arch1-1-x86_64-with-glibc2.39
- Python version: 3.11.8
- PyTorch version (GPU?): 2.2.2+cu121 (True)
- Huggingface_hub version: 0.20.3
- Transformers version: 4.40.1
- Accelerate version: 0.30.0
- xFormers version: not installed

### Who can help?

@sayakpaul @UmerHA 

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `src/diffusers/utils/peft_utils.py`
Content:
```
1 # Copyright 2024 The HuggingFace Team. All rights reserved.
2 #
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
6 #
7 #     http://www.apache.org/licenses/LICENSE-2.0
8 #
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 # See the License for the specific language governing permissions and
13 # limitations under the License.
14 """
15 PEFT utilities: Utilities related to peft library
16 """
17 
18 import collections
19 import importlib
20 from typing import Optional
21 
22 from packaging import version
23 
24 from .import_utils import is_peft_available, is_torch_available
25 
26 
27 if is_torch_available():
28     import torch
29 
30 
31 def recurse_remove_peft_layers(model):
32     r"""
33     Recursively replace all instances of `LoraLayer` with corresponding new layers in `model`.
34     """
35     from peft.tuners.tuners_utils import BaseTunerLayer
36 
37     has_base_layer_pattern = False
38     for module in model.modules():
39         if isinstance(module, BaseTunerLayer):
40             has_base_layer_pattern = hasattr(module, "base_layer")
41             break
42 
43     if has_base_layer_pattern:
44         from peft.utils import _get_submodules
45 
46         key_list = [key for key, _ in model.named_modules() if "lora" not in key]
47         for key in key_list:
48             try:
49                 parent, target, target_name = _get_submodules(model, key)
50             except AttributeError:
51                 continue
52             if hasattr(target, "base_layer"):
53                 setattr(parent, target_name, target.get_base_layer())
54     else:
55         # This is for backwards compatibility with PEFT <= 0.6.2.
56         # TODO can be removed once that PEFT version is no longer supported.
57         from peft.tuners.lora import LoraLayer
58 
59         for name, module in model.named_children():
60             if len(list(module.children())) > 0:
61                 ## compound module, go inside it
62                 recurse_remove_peft_layers(module)
63 
64             module_replaced = False
65 
66             if isinstance(module, LoraLayer) and isinstance(module, torch.nn.Linear):
67                 new_module = torch.nn.Linear(
68                     module.in_features,
69                     module.out_features,
70                     bias=module.bias is not None,
71                 ).to(module.weight.device)
72                 new_module.weight = module.weight
73                 if module.bias is not None:
74                     new_module.bias = module.bias
75 
76                 module_replaced = True
77             elif isinstance(module, LoraLayer) and isinstance(module, torch.nn.Conv2d):
78                 new_module = torch.nn.Conv2d(
79                     module.in_channels,
80                     module.out_channels,
81                     module.kernel_size,
82                     module.stride,
83                     module.padding,
84                     module.dilation,
85                     module.groups,
86                 ).to(module.weight.device)
87 
88                 new_module.weight = module.weight
89                 if module.bias is not None:
90                     new_module.bias = module.bias
91 
92                 module_replaced = True
93 
94             if module_replaced:
95                 setattr(model, name, new_module)
96                 del module
97 
98                 if torch.cuda.is_available():
99                     torch.cuda.empty_cache()
100     return model
101 
102 
103 def scale_lora_layers(model, weight):
104     """
105     Adjust the weightage given to the LoRA layers of the model.
106 
107     Args:
108         model (`torch.nn.Module`):
109             The model to scale.
110         weight (`float`):
111             The weight to be given to the LoRA layers.
112     """
113     from peft.tuners.tuners_utils import BaseTunerLayer
114 
115     if weight == 1.0:
116         return
117 
118     for module in model.modules():
119         if isinstance(module, BaseTunerLayer):
120             module.scale_layer(weight)
121 
122 
123 def unscale_lora_layers(model, weight: Optional[float] = None):
124     """
125     Removes the previously passed weight given to the LoRA layers of the model.
126 
127     Args:
128         model (`torch.nn.Module`):
129             The model to scale.
130         weight (`float`, *optional*):
131             The weight to be given to the LoRA layers. If no scale is passed the scale of the lora layer will be
132             re-initialized to the correct value. If 0.0 is passed, we will re-initialize the scale with the correct
133             value.
134     """
135     from peft.tuners.tuners_utils import BaseTunerLayer
136 
137     if weight == 1.0:
138         return
139 
140     for module in model.modules():
141         if isinstance(module, BaseTunerLayer):
142             if weight is not None and weight != 0:
143                 module.unscale_layer(weight)
144             elif weight is not None and weight == 0:
145                 for adapter_name in module.active_adapters:
146                     # if weight == 0 unscale should re-set the scale to the original value.
147                     module.set_scale(adapter_name, 1.0)
148 
149 
150 def get_peft_kwargs(rank_dict, network_alpha_dict, peft_state_dict, is_unet=True):
151     rank_pattern = {}
152     alpha_pattern = {}
153     r = lora_alpha = list(rank_dict.values())[0]
154 
155     if len(set(rank_dict.values())) > 1:
156         # get the rank occuring the most number of times
157         r = collections.Counter(rank_dict.values()).most_common()[0][0]
158 
159         # for modules with rank different from the most occuring rank, add it to the `rank_pattern`
160         rank_pattern = dict(filter(lambda x: x[1] != r, rank_dict.items()))
161         rank_pattern = {k.split(".lora_B.")[0]: v for k, v in rank_pattern.items()}
162 
163     if network_alpha_dict is not None and len(network_alpha_dict) > 0:
164         if len(set(network_alpha_dict.values())) > 1:
165             # get the alpha occuring the most number of times
166             lora_alpha = collections.Counter(network_alpha_dict.values()).most_common()[0][0]
167 
168             # for modules with alpha different from the most occuring alpha, add it to the `alpha_pattern`
169             alpha_pattern = dict(filter(lambda x: x[1] != lora_alpha, network_alpha_dict.items()))
170             if is_unet:
171                 alpha_pattern = {
172                     ".".join(k.split(".lora_A.")[0].split(".")).replace(".alpha", ""): v
173                     for k, v in alpha_pattern.items()
174                 }
175             else:
176                 alpha_pattern = {".".join(k.split(".down.")[0].split(".")[:-1]): v for k, v in alpha_pattern.items()}
177         else:
178             lora_alpha = set(network_alpha_dict.values()).pop()
179 
180     # layer names without the Diffusers specific
181     target_modules = list({name.split(".lora")[0] for name in peft_state_dict.keys()})
182     use_dora = any("lora_magnitude_vector" in k for k in peft_state_dict)
183 
184     lora_config_kwargs = {
185         "r": r,
186         "lora_alpha": lora_alpha,
187         "rank_pattern": rank_pattern,
188         "alpha_pattern": alpha_pattern,
189         "target_modules": target_modules,
190         "use_dora": use_dora,
191     }
192     return lora_config_kwargs
193 
194 
195 def get_adapter_name(model):
196     from peft.tuners.tuners_utils import BaseTunerLayer
197 
198     for module in model.modules():
199         if isinstance(module, BaseTunerLayer):
200             return f"default_{len(module.r)}"
201     return "default_0"
202 
203 
204 def set_adapter_layers(model, enabled=True):
205     from peft.tuners.tuners_utils import BaseTunerLayer
206 
207     for module in model.modules():
208         if isinstance(module, BaseTunerLayer):
209             # The recent version of PEFT needs to call `enable_adapters` instead
210             if hasattr(module, "enable_adapters"):
211                 module.enable_adapters(enabled=enabled)
212             else:
213                 module.disable_adapters = not enabled
214 
215 
216 def delete_adapter_layers(model, adapter_name):
217     from peft.tuners.tuners_utils import BaseTunerLayer
218 
219     for module in model.modules():
220         if isinstance(module, BaseTunerLayer):
221             if hasattr(module, "delete_adapter"):
222                 module.delete_adapter(adapter_name)
223             else:
224                 raise ValueError(
225                     "The version of PEFT you are using is not compatible, please use a version that is greater than 0.6.1"
226                 )
227 
228     # For transformers integration - we need to pop the adapter from the config
229     if getattr(model, "_hf_peft_config_loaded", False) and hasattr(model, "peft_config"):
230         model.peft_config.pop(adapter_name, None)
231         # In case all adapters are deleted, we need to delete the config
232         # and make sure to set the flag to False
233         if len(model.peft_config) == 0:
234             del model.peft_config
235             model._hf_peft_config_loaded = None
236 
237 
238 def set_weights_and_activate_adapters(model, adapter_names, weights):
239     from peft.tuners.tuners_utils import BaseTunerLayer
240 
241     def get_module_weight(weight_for_adapter, module_name):
242         if not isinstance(weight_for_adapter, dict):
243             # If weight_for_adapter is a single number, always return it.
244             return weight_for_adapter
245 
246         for layer_name, weight_ in weight_for_adapter.items():
247             if layer_name in module_name:
248                 return weight_
249         raise RuntimeError(f"No LoRA weight found for module {module_name}.")
250 
251     # iterate over each adapter, make it active and set the corresponding scaling weight
252     for adapter_name, weight in zip(adapter_names, weights):
253         for module_name, module in model.named_modules():
254             if isinstance(module, BaseTunerLayer):
255                 # For backward compatbility with previous PEFT versions
256                 if hasattr(module, "set_adapter"):
257                     module.set_adapter(adapter_name)
258                 else:
259                     module.active_adapter = adapter_name
260                 module.set_scale(adapter_name, get_module_weight(weight, module_name))
261 
262     # set multiple active adapters
263     for module in model.modules():
264         if isinstance(module, BaseTunerLayer):
265             # For backward compatbility with previous PEFT versions
266             if hasattr(module, "set_adapter"):
267                 module.set_adapter(adapter_names)
268             else:
269                 module.active_adapter = adapter_names
270 
271 
272 def check_peft_version(min_version: str) -> None:
273     r"""
274     Checks if the version of PEFT is compatible.
275 
276     Args:
277         version (`str`):
278             The version of PEFT to check against.
279     """
280     if not is_peft_available():
281         raise ValueError("PEFT is not installed. Please install it with `pip install peft`")
282 
283     is_peft_version_compatible = version.parse(importlib.metadata.version("peft")) > version.parse(min_version)
284 
285     if not is_peft_version_compatible:
286         raise ValueError(
287             f"The version of PEFT you are using is not compatible, please use a version that is greater"
288             f" than {min_version}"
289         )
290
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/src/diffusers/utils/peft_utils.py b/src/diffusers/utils/peft_utils.py
--- a/src/diffusers/utils/peft_utils.py
+++ b/src/diffusers/utils/peft_utils.py
@@ -246,7 +246,13 @@
         for layer_name, weight_ in weight_for_adapter.items():
             if layer_name in module_name:
                 return weight_
-        raise RuntimeError(f"No LoRA weight found for module {module_name}.")
+
+        parts = module_name.split(".")
+        # e.g. key = "down_blocks.1.attentions.0"
+        key = f"{parts[0]}.{parts[1]}.attentions.{parts[3]}"
+        block_weight = weight_for_adapter.get(key, 1.0)
+
+        return block_weight
 
     # iterate over each adapter, make it active and set the corresponding scaling weight
     for adapter_name, weight in zip(adapter_names, weights):

{"golden_diff": "diff --git a/src/diffusers/utils/peft_utils.py b/src/diffusers/utils/peft_utils.py\n--- a/src/diffusers/utils/peft_utils.py\n+++ b/src/diffusers/utils/peft_utils.py\n@@ -246,7 +246,13 @@\n         for layer_name, weight_ in weight_for_adapter.items():\n             if layer_name in module_name:\n                 return weight_\n-        raise RuntimeError(f\"No LoRA weight found for module {module_name}.\")\n+\n+        parts = module_name.split(\".\")\n+        # e.g. key = \"down_blocks.1.attentions.0\"\n+        key = f\"{parts[0]}.{parts[1]}.attentions.{parts[3]}\"\n+        block_weight = weight_for_adapter.get(key, 1.0)\n+\n+        return block_weight\n \n     # iterate over each adapter, make it active and set the corresponding scaling weight\n     for adapter_name, weight in zip(adapter_names, weights):\n", "issue": "Error when using blockwise scales with sd_xl_offset_example-lora_1.0.safetensors\n### Describe the bug\n\nWhen loading the [sd_xl_offset_example-lora_1.0.safetensors](https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0/blob/main/sd_xl_offset_example-lora_1.0.safetensors) from the [official repository](https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0/) and setting the scales by blocks, it throws this error:\r\n\r\n`RuntimeError: No LoRA weight found for module down_blocks.0.resnets.0.conv1`\r\n\r\nLoading the same lora with just a float e.g.: `pipe.set_adapters([\"offset\"], [1.0])` doesn't raise the error.\n\n### Reproduction\n\n```python\r\nimport torch\r\n\r\nfrom diffusers import StableDiffusionXLPipeline\r\n\r\n\r\npipe = StableDiffusionXLPipeline.from_pretrained(\r\n    \"stabilityai/stable-diffusion-xl-base-1.0\", torch_dtype=torch.float16, variant=\"fp16\"\r\n).to(\"cuda\")\r\n\r\npipe.load_lora_weights(\r\n    \"stabilityai/stable-diffusion-xl-base-1.0\",\r\n    weight_name=\"sd_xl_offset_example-lora_1.0.safetensors\",\r\n    adapter_name=\"offset\",\r\n)\r\noffset_scales = {\r\n    \"unet\": {\r\n        \"down\": {\"block_1\": [1.0, 1.0], \"block_2\": [1.0, 1.0]},\r\n        \"mid\": 0,\r\n        \"up\": {\"block_0\": [1.0, 1.0, 1.0], \"block_1\": [1.0, 1.0, 1.0]},\r\n    },\r\n    \"text_encoder\": 1.0,\r\n    \"text_encoder_2\": 1.0,\r\n}\r\npipe.set_adapters([\"offset\"], [offset_scales])\r\n\r\nprompt = \"high quality photo of a panther in the jungle, night, dark, low light, dim\"\r\ngenerator = torch.Generator(device=\"cpu\").manual_seed(899041154)\r\n\r\nimage = pipe(\r\n    prompt=prompt,\r\n    negative_prompt=\"\",\r\n    guidance_scale=6.5,\r\n    num_inference_steps=25,\r\n    generator=generator,\r\n).images[0]\r\nimage.save(\"result.png\")\r\n```\n\n### Logs\n\n_No response_\n\n### System Info\n\n- `diffusers` version: 0.28.0.dev0\r\n- Platform: Linux-6.8.7-arch1-1-x86_64-with-glibc2.39\r\n- Python version: 3.11.8\r\n- PyTorch version (GPU?): 2.2.2+cu121 (True)\r\n- Huggingface_hub version: 0.20.3\r\n- Transformers version: 4.40.1\r\n- Accelerate version: 0.30.0\r\n- xFormers version: not installed\n\n### Who can help?\n\n@sayakpaul @UmerHA \n", "before_files": [{"content": "# Copyright 2024 The HuggingFace Team. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\"\"\"\nPEFT utilities: Utilities related to peft library\n\"\"\"\n\nimport collections\nimport importlib\nfrom typing import Optional\n\nfrom packaging import version\n\nfrom .import_utils import is_peft_available, is_torch_available\n\n\nif is_torch_available():\n    import torch\n\n\ndef recurse_remove_peft_layers(model):\n    r\"\"\"\n    Recursively replace all instances of `LoraLayer` with corresponding new layers in `model`.\n    \"\"\"\n    from peft.tuners.tuners_utils import BaseTunerLayer\n\n    has_base_layer_pattern = False\n    for module in model.modules():\n        if isinstance(module, BaseTunerLayer):\n            has_base_layer_pattern = hasattr(module, \"base_layer\")\n            break\n\n    if has_base_layer_pattern:\n        from peft.utils import _get_submodules\n\n        key_list = [key for key, _ in model.named_modules() if \"lora\" not in key]\n        for key in key_list:\n            try:\n                parent, target, target_name = _get_submodules(model, key)\n            except AttributeError:\n                continue\n            if hasattr(target, \"base_layer\"):\n                setattr(parent, target_name, target.get_base_layer())\n    else:\n        # This is for backwards compatibility with PEFT <= 0.6.2.\n        # TODO can be removed once that PEFT version is no longer supported.\n        from peft.tuners.lora import LoraLayer\n\n        for name, module in model.named_children():\n            if len(list(module.children())) > 0:\n                ## compound module, go inside it\n                recurse_remove_peft_layers(module)\n\n            module_replaced = False\n\n            if isinstance(module, LoraLayer) and isinstance(module, torch.nn.Linear):\n                new_module = torch.nn.Linear(\n                    module.in_features,\n                    module.out_features,\n                    bias=module.bias is not None,\n                ).to(module.weight.device)\n                new_module.weight = module.weight\n                if module.bias is not None:\n                    new_module.bias = module.bias\n\n                module_replaced = True\n            elif isinstance(module, LoraLayer) and isinstance(module, torch.nn.Conv2d):\n                new_module = torch.nn.Conv2d(\n                    module.in_channels,\n                    module.out_channels,\n                    module.kernel_size,\n                    module.stride,\n                    module.padding,\n                    module.dilation,\n                    module.groups,\n                ).to(module.weight.device)\n\n                new_module.weight = module.weight\n                if module.bias is not None:\n                    new_module.bias = module.bias\n\n                module_replaced = True\n\n            if module_replaced:\n                setattr(model, name, new_module)\n                del module\n\n                if torch.cuda.is_available():\n                    torch.cuda.empty_cache()\n    return model\n\n\ndef scale_lora_layers(model, weight):\n    \"\"\"\n    Adjust the weightage given to the LoRA layers of the model.\n\n    Args:\n        model (`torch.nn.Module`):\n            The model to scale.\n        weight (`float`):\n            The weight to be given to the LoRA layers.\n    \"\"\"\n    from peft.tuners.tuners_utils import BaseTunerLayer\n\n    if weight == 1.0:\n        return\n\n    for module in model.modules():\n        if isinstance(module, BaseTunerLayer):\n            module.scale_layer(weight)\n\n\ndef unscale_lora_layers(model, weight: Optional[float] = None):\n    \"\"\"\n    Removes the previously passed weight given to the LoRA layers of the model.\n\n    Args:\n        model (`torch.nn.Module`):\n            The model to scale.\n        weight (`float`, *optional*):\n            The weight to be given to the LoRA layers. If no scale is passed the scale of the lora layer will be\n            re-initialized to the correct value. If 0.0 is passed, we will re-initialize the scale with the correct\n            value.\n    \"\"\"\n    from peft.tuners.tuners_utils import BaseTunerLayer\n\n    if weight == 1.0:\n        return\n\n    for module in model.modules():\n        if isinstance(module, BaseTunerLayer):\n            if weight is not None and weight != 0:\n                module.unscale_layer(weight)\n            elif weight is not None and weight == 0:\n                for adapter_name in module.active_adapters:\n                    # if weight == 0 unscale should re-set the scale to the original value.\n                    module.set_scale(adapter_name, 1.0)\n\n\ndef get_peft_kwargs(rank_dict, network_alpha_dict, peft_state_dict, is_unet=True):\n    rank_pattern = {}\n    alpha_pattern = {}\n    r = lora_alpha = list(rank_dict.values())[0]\n\n    if len(set(rank_dict.values())) > 1:\n        # get the rank occuring the most number of times\n        r = collections.Counter(rank_dict.values()).most_common()[0][0]\n\n        # for modules with rank different from the most occuring rank, add it to the `rank_pattern`\n        rank_pattern = dict(filter(lambda x: x[1] != r, rank_dict.items()))\n        rank_pattern = {k.split(\".lora_B.\")[0]: v for k, v in rank_pattern.items()}\n\n    if network_alpha_dict is not None and len(network_alpha_dict) > 0:\n        if len(set(network_alpha_dict.values())) > 1:\n            # get the alpha occuring the most number of times\n            lora_alpha = collections.Counter(network_alpha_dict.values()).most_common()[0][0]\n\n            # for modules with alpha different from the most occuring alpha, add it to the `alpha_pattern`\n            alpha_pattern = dict(filter(lambda x: x[1] != lora_alpha, network_alpha_dict.items()))\n            if is_unet:\n                alpha_pattern = {\n                    \".\".join(k.split(\".lora_A.\")[0].split(\".\")).replace(\".alpha\", \"\"): v\n                    for k, v in alpha_pattern.items()\n                }\n            else:\n                alpha_pattern = {\".\".join(k.split(\".down.\")[0].split(\".\")[:-1]): v for k, v in alpha_pattern.items()}\n        else:\n            lora_alpha = set(network_alpha_dict.values()).pop()\n\n    # layer names without the Diffusers specific\n    target_modules = list({name.split(\".lora\")[0] for name in peft_state_dict.keys()})\n    use_dora = any(\"lora_magnitude_vector\" in k for k in peft_state_dict)\n\n    lora_config_kwargs = {\n        \"r\": r,\n        \"lora_alpha\": lora_alpha,\n        \"rank_pattern\": rank_pattern,\n        \"alpha_pattern\": alpha_pattern,\n        \"target_modules\": target_modules,\n        \"use_dora\": use_dora,\n    }\n    return lora_config_kwargs\n\n\ndef get_adapter_name(model):\n    from peft.tuners.tuners_utils import BaseTunerLayer\n\n    for module in model.modules():\n        if isinstance(module, BaseTunerLayer):\n            return f\"default_{len(module.r)}\"\n    return \"default_0\"\n\n\ndef set_adapter_layers(model, enabled=True):\n    from peft.tuners.tuners_utils import BaseTunerLayer\n\n    for module in model.modules():\n        if isinstance(module, BaseTunerLayer):\n            # The recent version of PEFT needs to call `enable_adapters` instead\n            if hasattr(module, \"enable_adapters\"):\n                module.enable_adapters(enabled=enabled)\n            else:\n                module.disable_adapters = not enabled\n\n\ndef delete_adapter_layers(model, adapter_name):\n    from peft.tuners.tuners_utils import BaseTunerLayer\n\n    for module in model.modules():\n        if isinstance(module, BaseTunerLayer):\n            if hasattr(module, \"delete_adapter\"):\n                module.delete_adapter(adapter_name)\n            else:\n                raise ValueError(\n                    \"The version of PEFT you are using is not compatible, please use a version that is greater than 0.6.1\"\n                )\n\n    # For transformers integration - we need to pop the adapter from the config\n    if getattr(model, \"_hf_peft_config_loaded\", False) and hasattr(model, \"peft_config\"):\n        model.peft_config.pop(adapter_name, None)\n        # In case all adapters are deleted, we need to delete the config\n        # and make sure to set the flag to False\n        if len(model.peft_config) == 0:\n            del model.peft_config\n            model._hf_peft_config_loaded = None\n\n\ndef set_weights_and_activate_adapters(model, adapter_names, weights):\n    from peft.tuners.tuners_utils import BaseTunerLayer\n\n    def get_module_weight(weight_for_adapter, module_name):\n        if not isinstance(weight_for_adapter, dict):\n            # If weight_for_adapter is a single number, always return it.\n            return weight_for_adapter\n\n        for layer_name, weight_ in weight_for_adapter.items():\n            if layer_name in module_name:\n                return weight_\n        raise RuntimeError(f\"No LoRA weight found for module {module_name}.\")\n\n    # iterate over each adapter, make it active and set the corresponding scaling weight\n    for adapter_name, weight in zip(adapter_names, weights):\n        for module_name, module in model.named_modules():\n            if isinstance(module, BaseTunerLayer):\n                # For backward compatbility with previous PEFT versions\n                if hasattr(module, \"set_adapter\"):\n                    module.set_adapter(adapter_name)\n                else:\n                    module.active_adapter = adapter_name\n                module.set_scale(adapter_name, get_module_weight(weight, module_name))\n\n    # set multiple active adapters\n    for module in model.modules():\n        if isinstance(module, BaseTunerLayer):\n            # For backward compatbility with previous PEFT versions\n            if hasattr(module, \"set_adapter\"):\n                module.set_adapter(adapter_names)\n            else:\n                module.active_adapter = adapter_names\n\n\ndef check_peft_version(min_version: str) -> None:\n    r\"\"\"\n    Checks if the version of PEFT is compatible.\n\n    Args:\n        version (`str`):\n            The version of PEFT to check against.\n    \"\"\"\n    if not is_peft_available():\n        raise ValueError(\"PEFT is not installed. Please install it with `pip install peft`\")\n\n    is_peft_version_compatible = version.parse(importlib.metadata.version(\"peft\")) > version.parse(min_version)\n\n    if not is_peft_version_compatible:\n        raise ValueError(\n            f\"The version of PEFT you are using is not compatible, please use a version that is greater\"\n            f\" than {min_version}\"\n        )\n", "path": "src/diffusers/utils/peft_utils.py"}], "after_files": [{"content": "# Copyright 2024 The HuggingFace Team. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\"\"\"\nPEFT utilities: Utilities related to peft library\n\"\"\"\n\nimport collections\nimport importlib\nfrom typing import Optional\n\nfrom packaging import version\n\nfrom .import_utils import is_peft_available, is_torch_available\n\n\nif is_torch_available():\n    import torch\n\n\ndef recurse_remove_peft_layers(model):\n    r\"\"\"\n    Recursively replace all instances of `LoraLayer` with corresponding new layers in `model`.\n    \"\"\"\n    from peft.tuners.tuners_utils import BaseTunerLayer\n\n    has_base_layer_pattern = False\n    for module in model.modules():\n        if isinstance(module, BaseTunerLayer):\n            has_base_layer_pattern = hasattr(module, \"base_layer\")\n            break\n\n    if has_base_layer_pattern:\n        from peft.utils import _get_submodules\n\n        key_list = [key for key, _ in model.named_modules() if \"lora\" not in key]\n        for key in key_list:\n            try:\n                parent, target, target_name = _get_submodules(model, key)\n            except AttributeError:\n                continue\n            if hasattr(target, \"base_layer\"):\n                setattr(parent, target_name, target.get_base_layer())\n    else:\n        # This is for backwards compatibility with PEFT <= 0.6.2.\n        # TODO can be removed once that PEFT version is no longer supported.\n        from peft.tuners.lora import LoraLayer\n\n        for name, module in model.named_children():\n            if len(list(module.children())) > 0:\n                ## compound module, go inside it\n                recurse_remove_peft_layers(module)\n\n            module_replaced = False\n\n            if isinstance(module, LoraLayer) and isinstance(module, torch.nn.Linear):\n                new_module = torch.nn.Linear(\n                    module.in_features,\n                    module.out_features,\n                    bias=module.bias is not None,\n                ).to(module.weight.device)\n                new_module.weight = module.weight\n                if module.bias is not None:\n                    new_module.bias = module.bias\n\n                module_replaced = True\n            elif isinstance(module, LoraLayer) and isinstance(module, torch.nn.Conv2d):\n                new_module = torch.nn.Conv2d(\n                    module.in_channels,\n                    module.out_channels,\n                    module.kernel_size,\n                    module.stride,\n                    module.padding,\n                    module.dilation,\n                    module.groups,\n                ).to(module.weight.device)\n\n                new_module.weight = module.weight\n                if module.bias is not None:\n                    new_module.bias = module.bias\n\n                module_replaced = True\n\n            if module_replaced:\n                setattr(model, name, new_module)\n                del module\n\n                if torch.cuda.is_available():\n                    torch.cuda.empty_cache()\n    return model\n\n\ndef scale_lora_layers(model, weight):\n    \"\"\"\n    Adjust the weightage given to the LoRA layers of the model.\n\n    Args:\n        model (`torch.nn.Module`):\n            The model to scale.\n        weight (`float`):\n            The weight to be given to the LoRA layers.\n    \"\"\"\n    from peft.tuners.tuners_utils import BaseTunerLayer\n\n    if weight == 1.0:\n        return\n\n    for module in model.modules():\n        if isinstance(module, BaseTunerLayer):\n            module.scale_layer(weight)\n\n\ndef unscale_lora_layers(model, weight: Optional[float] = None):\n    \"\"\"\n    Removes the previously passed weight given to the LoRA layers of the model.\n\n    Args:\n        model (`torch.nn.Module`):\n            The model to scale.\n        weight (`float`, *optional*):\n            The weight to be given to the LoRA layers. If no scale is passed the scale of the lora layer will be\n            re-initialized to the correct value. If 0.0 is passed, we will re-initialize the scale with the correct\n            value.\n    \"\"\"\n    from peft.tuners.tuners_utils import BaseTunerLayer\n\n    if weight == 1.0:\n        return\n\n    for module in model.modules():\n        if isinstance(module, BaseTunerLayer):\n            if weight is not None and weight != 0:\n                module.unscale_layer(weight)\n            elif weight is not None and weight == 0:\n                for adapter_name in module.active_adapters:\n                    # if weight == 0 unscale should re-set the scale to the original value.\n                    module.set_scale(adapter_name, 1.0)\n\n\ndef get_peft_kwargs(rank_dict, network_alpha_dict, peft_state_dict, is_unet=True):\n    rank_pattern = {}\n    alpha_pattern = {}\n    r = lora_alpha = list(rank_dict.values())[0]\n\n    if len(set(rank_dict.values())) > 1:\n        # get the rank occuring the most number of times\n        r = collections.Counter(rank_dict.values()).most_common()[0][0]\n\n        # for modules with rank different from the most occuring rank, add it to the `rank_pattern`\n        rank_pattern = dict(filter(lambda x: x[1] != r, rank_dict.items()))\n        rank_pattern = {k.split(\".lora_B.\")[0]: v for k, v in rank_pattern.items()}\n\n    if network_alpha_dict is not None and len(network_alpha_dict) > 0:\n        if len(set(network_alpha_dict.values())) > 1:\n            # get the alpha occuring the most number of times\n            lora_alpha = collections.Counter(network_alpha_dict.values()).most_common()[0][0]\n\n            # for modules with alpha different from the most occuring alpha, add it to the `alpha_pattern`\n            alpha_pattern = dict(filter(lambda x: x[1] != lora_alpha, network_alpha_dict.items()))\n            if is_unet:\n                alpha_pattern = {\n                    \".\".join(k.split(\".lora_A.\")[0].split(\".\")).replace(\".alpha\", \"\"): v\n                    for k, v in alpha_pattern.items()\n                }\n            else:\n                alpha_pattern = {\".\".join(k.split(\".down.\")[0].split(\".\")[:-1]): v for k, v in alpha_pattern.items()}\n        else:\n            lora_alpha = set(network_alpha_dict.values()).pop()\n\n    # layer names without the Diffusers specific\n    target_modules = list({name.split(\".lora\")[0] for name in peft_state_dict.keys()})\n    use_dora = any(\"lora_magnitude_vector\" in k for k in peft_state_dict)\n\n    lora_config_kwargs = {\n        \"r\": r,\n        \"lora_alpha\": lora_alpha,\n        \"rank_pattern\": rank_pattern,\n        \"alpha_pattern\": alpha_pattern,\n        \"target_modules\": target_modules,\n        \"use_dora\": use_dora,\n    }\n    return lora_config_kwargs\n\n\ndef get_adapter_name(model):\n    from peft.tuners.tuners_utils import BaseTunerLayer\n\n    for module in model.modules():\n        if isinstance(module, BaseTunerLayer):\n            return f\"default_{len(module.r)}\"\n    return \"default_0\"\n\n\ndef set_adapter_layers(model, enabled=True):\n    from peft.tuners.tuners_utils import BaseTunerLayer\n\n    for module in model.modules():\n        if isinstance(module, BaseTunerLayer):\n            # The recent version of PEFT needs to call `enable_adapters` instead\n            if hasattr(module, \"enable_adapters\"):\n                module.enable_adapters(enabled=enabled)\n            else:\n                module.disable_adapters = not enabled\n\n\ndef delete_adapter_layers(model, adapter_name):\n    from peft.tuners.tuners_utils import BaseTunerLayer\n\n    for module in model.modules():\n        if isinstance(module, BaseTunerLayer):\n            if hasattr(module, \"delete_adapter\"):\n                module.delete_adapter(adapter_name)\n            else:\n                raise ValueError(\n                    \"The version of PEFT you are using is not compatible, please use a version that is greater than 0.6.1\"\n                )\n\n    # For transformers integration - we need to pop the adapter from the config\n    if getattr(model, \"_hf_peft_config_loaded\", False) and hasattr(model, \"peft_config\"):\n        model.peft_config.pop(adapter_name, None)\n        # In case all adapters are deleted, we need to delete the config\n        # and make sure to set the flag to False\n        if len(model.peft_config) == 0:\n            del model.peft_config\n            model._hf_peft_config_loaded = None\n\n\ndef set_weights_and_activate_adapters(model, adapter_names, weights):\n    from peft.tuners.tuners_utils import BaseTunerLayer\n\n    def get_module_weight(weight_for_adapter, module_name):\n        if not isinstance(weight_for_adapter, dict):\n            # If weight_for_adapter is a single number, always return it.\n            return weight_for_adapter\n\n        for layer_name, weight_ in weight_for_adapter.items():\n            if layer_name in module_name:\n                return weight_\n\n        parts = module_name.split(\".\")\n        # e.g. key = \"down_blocks.1.attentions.0\"\n        key = f\"{parts[0]}.{parts[1]}.attentions.{parts[3]}\"\n        block_weight = weight_for_adapter.get(key, 1.0)\n\n        return block_weight\n\n    # iterate over each adapter, make it active and set the corresponding scaling weight\n    for adapter_name, weight in zip(adapter_names, weights):\n        for module_name, module in model.named_modules():\n            if isinstance(module, BaseTunerLayer):\n                # For backward compatbility with previous PEFT versions\n                if hasattr(module, \"set_adapter\"):\n                    module.set_adapter(adapter_name)\n                else:\n                    module.active_adapter = adapter_name\n                module.set_scale(adapter_name, get_module_weight(weight, module_name))\n\n    # set multiple active adapters\n    for module in model.modules():\n        if isinstance(module, BaseTunerLayer):\n            # For backward compatbility with previous PEFT versions\n            if hasattr(module, \"set_adapter\"):\n                module.set_adapter(adapter_names)\n            else:\n                module.active_adapter = adapter_names\n\n\ndef check_peft_version(min_version: str) -> None:\n    r\"\"\"\n    Checks if the version of PEFT is compatible.\n\n    Args:\n        version (`str`):\n            The version of PEFT to check against.\n    \"\"\"\n    if not is_peft_available():\n        raise ValueError(\"PEFT is not installed. Please install it with `pip install peft`\")\n\n    is_peft_version_compatible = version.parse(importlib.metadata.version(\"peft\")) > version.parse(min_version)\n\n    if not is_peft_version_compatible:\n        raise ValueError(\n            f\"The version of PEFT you are using is not compatible, please use a version that is greater\"\n            f\" than {min_version}\"\n        )\n", "path": "src/diffusers/utils/peft_utils.py"}]}