# coding=utf-8
# Copyright 2023-present the HuggingFace Inc. team.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import inspect
import json
import os
from dataclasses import asdict, dataclass, field
from typing import Dict, Optional, Union

from huggingface_hub import hf_hub_download
from transformers.utils import PushToHubMixin

from .utils import CONFIG_NAME, PeftType, TaskType


@dataclass
class PeftConfigMixin(PushToHubMixin):
    r"""
    This is the base configuration class for PEFT adapter models. It contains all the methods that are common to all
    PEFT adapter models. This class inherits from [`~transformers.utils.PushToHubMixin`] which contains the methods to
    push your model to the Hub. The method `save_pretrained` will save the configuration of your adapter model in a
    directory. The method `from_pretrained` will load the configuration of your adapter model from a directory.

    Args:
        peft_type (Union[[`~peft.utils.config.PeftType`], `str`]): The type of Peft method to use.
    """
    peft_type: Optional[PeftType] = field(default=None, metadata={"help": "The type of PEFT model."})
    auto_mapping: Optional[dict] = field(
        default=None, metadata={"help": "An auto mapping dict to help retrieve the base model class if needed."}
    )

    def to_dict(self) -> Dict:
        return asdict(self)

    def save_pretrained(self, save_directory: str, **kwargs) -> None:
        r"""
        This method saves the configuration of your adapter model in a directory.

        Args:
            save_directory (`str`):
                The directory where the configuration will be saved.
            kwargs (additional keyword arguments, *optional*):
                Additional keyword arguments passed along to the [`~transformers.utils.PushToHubMixin.push_to_hub`]
                method.
        """
        if os.path.isfile(save_directory):
            raise AssertionError(f"Provided path ({save_directory}) should be a directory, not a file")

        os.makedirs(save_directory, exist_ok=True)
        auto_mapping_dict = kwargs.pop("auto_mapping_dict", None)

        output_dict = asdict(self)
        # converting set type to list
        for key, value in output_dict.items():
            if isinstance(value, set):
                output_dict[key] = list(value)

        output_path = os.path.join(save_directory, CONFIG_NAME)

        # Add auto mapping details for custom models.
        if auto_mapping_dict is not None:
            output_dict["auto_mapping"] = auto_mapping_dict

        # save it
        with open(output_path, "w") as writer:
            writer.write(json.dumps(output_dict, indent=2, sort_keys=True))

    @classmethod
    def from_pretrained(cls, pretrained_model_name_or_path: str, subfolder: Optional[str] = None, **kwargs):
        r"""
        This method loads the configuration of your adapter model from a directory.

        Args:
            pretrained_model_name_or_path (`str`):
                The directory or the Hub repository id where the configuration is saved.
            kwargs (additional keyword arguments, *optional*):
                Additional keyword arguments passed along to the child class initialization.
        """
        # Avoid circular dependency .. TODO: fix this with a larger refactor
        from peft.mapping import PEFT_TYPE_TO_CONFIG_MAPPING

        path = (
            os.path.join(pretrained_model_name_or_path, subfolder)
            if subfolder is not None
            else pretrained_model_name_or_path
        )

        hf_hub_download_kwargs, class_kwargs, _ = cls._split_kwargs(kwargs)

        if os.path.isfile(os.path.join(path, CONFIG_NAME)):
            config_file = os.path.join(path, CONFIG_NAME)
        else:
            try:
                config_file = hf_hub_download(
                    pretrained_model_name_or_path, CONFIG_NAME, subfolder=subfolder, **hf_hub_download_kwargs
                )
            except Exception:
                raise ValueError(f"Can't find '{CONFIG_NAME}' at '{pretrained_model_name_or_path}'")

        loaded_attributes = cls.from_json_file(config_file)

        # TODO: this hack is needed to fix the following issue (on commit 702f937):
        # if someone saves a default config and loads it back with `PeftConfig` class it yields to
        # not loading the correct config class.

        # from peft import AdaLoraConfig, PeftConfig
        # peft_config = AdaLoraConfig()
        # print(peft_config)
        # >>> AdaLoraConfig(peft_type=<PeftType.ADALORA: 'ADALORA'>, auto_mapping=None, base_model_name_or_path=None,
        # revision=None, task_type=None, inference_mode=False, r=8, target_modules=None, lora_alpha=8, lora_dropout=0.0, ...
        #
        # peft_config.save_pretrained("./test_config")
        # peft_config = PeftConfig.from_pretrained("./test_config")
        # print(peft_config)
        # >>> PeftConfig(peft_type='ADALORA', auto_mapping=None, base_model_name_or_path=None, revision=None, task_type=None, inference_mode=False)
        if "peft_type" in loaded_attributes:
            peft_type = loaded_attributes["peft_type"]
            config_cls = PEFT_TYPE_TO_CONFIG_MAPPING[peft_type]
        else:
            config_cls = cls

        config = config_cls(**class_kwargs)

        for key, value in loaded_attributes.items():
            if hasattr(config, key):
                setattr(config, key, value)

        return config

    @classmethod
    def from_json_file(cls, path_json_file: str, **kwargs):
        r"""
        Loads a configuration file from a json file.

        Args:
            path_json_file (`str`):
                The path to the json file.
        """
        with open(path_json_file, "r") as file:
            json_object = json.load(file)

        return json_object

    @classmethod
    def _split_kwargs(cls, kwargs):
        hf_hub_download_kwargs = {}
        class_kwargs = {}
        other_kwargs = {}

        for key, value in kwargs.items():
            if key in inspect.signature(hf_hub_download).parameters:
                hf_hub_download_kwargs[key] = value
            elif key in list(cls.__annotations__):
                class_kwargs[key] = value
            else:
                other_kwargs[key] = value

        return hf_hub_download_kwargs, class_kwargs, other_kwargs

    @classmethod
    def _get_peft_type(
        cls,
        model_id: str,
        **hf_hub_download_kwargs,
    ):
        subfolder = hf_hub_download_kwargs.get("subfolder", None)

        path = os.path.join(model_id, subfolder) if subfolder is not None else model_id

        if os.path.isfile(os.path.join(path, CONFIG_NAME)):
            config_file = os.path.join(path, CONFIG_NAME)
        else:
            try:
                config_file = hf_hub_download(
                    model_id,
                    CONFIG_NAME,
                    **hf_hub_download_kwargs,
                )
            except Exception:
                raise ValueError(f"Can't find '{CONFIG_NAME}' at '{model_id}'")

        loaded_attributes = cls.from_json_file(config_file)
        return loaded_attributes["peft_type"]

    @property
    def is_prompt_learning(self) -> bool:
        r"""
        Utility method to check if the configuration is for prompt learning.
        """
        return False

    @property
    def is_adaption_prompt(self) -> bool:
        """Return True if this is an adaption prompt config."""
        return False


@dataclass
class PeftConfig(PeftConfigMixin):
    """
    This is the base configuration class to store the configuration of a [`PeftModel`].

    Args:
        peft_type (Union[[`~peft.utils.config.PeftType`], `str`]): The type of Peft method to use.
        task_type (Union[[`~peft.utils.config.TaskType`], `str`]): The type of task to perform.
        inference_mode (`bool`, defaults to `False`): Whether to use the Peft model in inference mode.
    """

    base_model_name_or_path: str = field(default=None, metadata={"help": "The name of the base model to use."})
    revision: str = field(default=None, metadata={"help": "The specific model version to use."})
    peft_type: Union[str, PeftType] = field(default=None, metadata={"help": "Peft type"})
    task_type: Union[str, TaskType] = field(default=None, metadata={"help": "Task type"})
    inference_mode: bool = field(default=False, metadata={"help": "Whether to use inference mode"})


@dataclass
class PromptLearningConfig(PeftConfig):
    """
    This is the base configuration class to store the configuration of [`PrefixTuning`], [`PromptEncoder`], or
    [`PromptTuning`].

    Args:
        num_virtual_tokens (`int`): The number of virtual tokens to use.
        token_dim (`int`): The hidden embedding dimension of the base transformer model.
        num_transformer_submodules (`int`): The number of transformer submodules in the base transformer model.
        num_attention_heads (`int`): The number of attention heads in the base transformer model.
        num_layers (`int`): The number of layers in the base transformer model.
    """

    num_virtual_tokens: int = field(default=None, metadata={"help": "Number of virtual tokens"})
    token_dim: int = field(
        default=None, metadata={"help": "The hidden embedding dimension of the base transformer model"}
    )
    num_transformer_submodules: Optional[int] = field(
        default=None, metadata={"help": "Number of transformer submodules"}
    )
    num_attention_heads: Optional[int] = field(default=None, metadata={"help": "Number of attention heads"})
    num_layers: Optional[int] = field(default=None, metadata={"help": "Number of transformer layers"})

    @property
    def is_prompt_learning(self) -> bool:
        r"""
        Utility method to check if the configuration is for prompt learning.
        """
        return True

src/peft/config.py
# coding=utf-8
# Copyright 2023-present the HuggingFace Inc. team.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import inspect
import json
import os
from dataclasses import asdict, dataclass, field
from typing import Dict, Optional, Union

from huggingface_hub import hf_hub_download
from transformers.utils import PushToHubMixin

from .utils import CONFIG_NAME, PeftType, TaskType


@dataclass
class PeftConfigMixin(PushToHubMixin):
    r"""
    This is the base configuration class for PEFT adapter models. It contains all the methods that are common to all
    PEFT adapter models. This class inherits from [`~transformers.utils.PushToHubMixin`] which contains the methods to
    push your model to the Hub. The method `save_pretrained` will save the configuration of your adapter model in a
    directory. The method `from_pretrained` will load the configuration of your adapter model from a directory.

    Args:
        peft_type (Union[[`~peft.utils.config.PeftType`], `str`]): The type of Peft method to use.
    """
    peft_type: Optional[PeftType] = field(default=None, metadata={"help": "The type of PEFT model."})
    auto_mapping: Optional[dict] = field(
        default=None, metadata={"help": "An auto mapping dict to help retrieve the base model class if needed."}
    )

    def to_dict(self) -> Dict:
        return asdict(self)

    def save_pretrained(self, save_directory: str, **kwargs) -> None:
        r"""
        This method saves the configuration of your adapter model in a directory.

        Args:
            save_directory (`str`):
                The directory where the configuration will be saved.
            kwargs (additional keyword arguments, *optional*):
                Additional keyword arguments passed along to the [`~transformers.utils.PushToHubMixin.push_to_hub`]
                method.
        """
        if os.path.isfile(save_directory):
            raise AssertionError(f"Provided path ({save_directory}) should be a directory, not a file")

        os.makedirs(save_directory, exist_ok=True)
        auto_mapping_dict = kwargs.pop("auto_mapping_dict", None)

        output_dict = asdict(self)
        # converting set type to list
        for key, value in output_dict.items():
            if isinstance(value, set):
                output_dict[key] = list(value)

        output_path = os.path.join(save_directory, CONFIG_NAME)

        # Add auto mapping details for custom models.
        if auto_mapping_dict is not None:
            output_dict["auto_mapping"] = auto_mapping_dict

        # save it
        with open(output_path, "w") as writer:
            writer.write(json.dumps(output_dict, indent=2, sort_keys=True))

    @classmethod
    def from_pretrained(cls, pretrained_model_name_or_path: str, subfolder: Optional[str] = None, **kwargs):
        r"""
        This method loads the configuration of your adapter model from a directory.

        Args:
            pretrained_model_name_or_path (`str`):
                The directory or the Hub repository id where the configuration is saved.
            kwargs (additional keyword arguments, *optional*):
                Additional keyword arguments passed along to the child class initialization.
        """
        # Avoid circular dependency .. TODO: fix this with a larger refactor
        from peft.mapping import PEFT_TYPE_TO_CONFIG_MAPPING

        path = (
            os.path.join(pretrained_model_name_or_path, subfolder)
            if subfolder is not None
            else pretrained_model_name_or_path
        )

        hf_hub_download_kwargs, class_kwargs, _ = cls._split_kwargs(kwargs)

        if os.path.isfile(os.path.join(path, CONFIG_NAME)):
            config_file = os.path.join(path, CONFIG_NAME)
        else:
            try:
                config_file = hf_hub_download(
                    pretrained_model_name_or_path, CONFIG_NAME, subfolder=subfolder, **hf_hub_download_kwargs
                )
            except Exception:
                raise ValueError(f"Can't find '{CONFIG_NAME}' at '{pretrained_model_name_or_path}'")

        loaded_attributes = cls.from_json_file(config_file)

        # TODO: this hack is needed to fix the following issue (on commit 702f937):
        # if someone saves a default config and loads it back with `PeftConfig` class it yields to
        # not loading the correct config class.

        # from peft import AdaLoraConfig, PeftConfig
        # peft_config = AdaLoraConfig()
        # print(peft_config)
        # >>> AdaLoraConfig(peft_type=<PeftType.ADALORA: 'ADALORA'>, auto_mapping=None, base_model_name_or_path=None,
        # revision=None, task_type=None, inference_mode=False, r=8, target_modules=None, lora_alpha=8, lora_dropout=0.0, ...
        #
        # peft_config.save_pretrained("./test_config")
        # peft_config = PeftConfig.from_pretrained("./test_config")
        # print(peft_config)
        # >>> PeftConfig(peft_type='ADALORA', auto_mapping=None, base_model_name_or_path=None, revision=None, task_type=None, inference_mode=False)
        if "peft_type" in loaded_attributes:
            peft_type = loaded_attributes["peft_type"]
            config_cls = PEFT_TYPE_TO_CONFIG_MAPPING[peft_type]
        else:
            config_cls = cls

        kwargs = {**class_kwargs, **loaded_attributes}
        config = config_cls(**kwargs)
        return config

    @classmethod
    def from_json_file(cls, path_json_file: str, **kwargs):
        r"""
        Loads a configuration file from a json file.

        Args:
            path_json_file (`str`):
                The path to the json file.
        """
        with open(path_json_file, "r") as file:
            json_object = json.load(file)

        return json_object

    @classmethod
    def _split_kwargs(cls, kwargs):
        hf_hub_download_kwargs = {}
        class_kwargs = {}
        other_kwargs = {}

        for key, value in kwargs.items():
            if key in inspect.signature(hf_hub_download).parameters:
                hf_hub_download_kwargs[key] = value
            elif key in list(cls.__annotations__):
                class_kwargs[key] = value
            else:
                other_kwargs[key] = value

        return hf_hub_download_kwargs, class_kwargs, other_kwargs

    @classmethod
    def _get_peft_type(
        cls,
        model_id: str,
        **hf_hub_download_kwargs,
    ):
        subfolder = hf_hub_download_kwargs.get("subfolder", None)

        path = os.path.join(model_id, subfolder) if subfolder is not None else model_id

        if os.path.isfile(os.path.join(path, CONFIG_NAME)):
            config_file = os.path.join(path, CONFIG_NAME)
        else:
            try:
                config_file = hf_hub_download(
                    model_id,
                    CONFIG_NAME,
                    **hf_hub_download_kwargs,
                )
            except Exception:
                raise ValueError(f"Can't find '{CONFIG_NAME}' at '{model_id}'")

        loaded_attributes = cls.from_json_file(config_file)
        return loaded_attributes["peft_type"]

    @property
    def is_prompt_learning(self) -> bool:
        r"""
        Utility method to check if the configuration is for prompt learning.
        """
        return False

    @property
    def is_adaption_prompt(self) -> bool:
        """Return True if this is an adaption prompt config."""
        return False


@dataclass
class PeftConfig(PeftConfigMixin):
    """
    This is the base configuration class to store the configuration of a [`PeftModel`].

    Args:
        peft_type (Union[[`~peft.utils.config.PeftType`], `str`]): The type of Peft method to use.
        task_type (Union[[`~peft.utils.config.TaskType`], `str`]): The type of task to perform.
        inference_mode (`bool`, defaults to `False`): Whether to use the Peft model in inference mode.
    """

    base_model_name_or_path: str = field(default=None, metadata={"help": "The name of the base model to use."})
    revision: str = field(default=None, metadata={"help": "The specific model version to use."})
    peft_type: Union[str, PeftType] = field(default=None, metadata={"help": "Peft type"})
    task_type: Union[str, TaskType] = field(default=None, metadata={"help": "Task type"})
    inference_mode: bool = field(default=False, metadata={"help": "Whether to use inference mode"})


@dataclass
class PromptLearningConfig(PeftConfig):
    """
    This is the base configuration class to store the configuration of [`PrefixTuning`], [`PromptEncoder`], or
    [`PromptTuning`].

    Args:
        num_virtual_tokens (`int`): The number of virtual tokens to use.
        token_dim (`int`): The hidden embedding dimension of the base transformer model.
        num_transformer_submodules (`int`): The number of transformer submodules in the base transformer model.
        num_attention_heads (`int`): The number of attention heads in the base transformer model.
        num_layers (`int`): The number of layers in the base transformer model.
    """

    num_virtual_tokens: int = field(default=None, metadata={"help": "Number of virtual tokens"})
    token_dim: int = field(
        default=None, metadata={"help": "The hidden embedding dimension of the base transformer model"}
    )
    num_transformer_submodules: Optional[int] = field(
        default=None, metadata={"help": "Number of transformer submodules"}
    )
    num_attention_heads: Optional[int] = field(default=None, metadata={"help": "Number of attention heads"})
    num_layers: Optional[int] = field(default=None, metadata={"help": "Number of transformer layers"})

    @property
    def is_prompt_learning(self) -> bool:
        r"""
        Utility method to check if the configuration is for prompt learning.
        """
        return True

src/peft/config.py
diff --git a/src/peft/config.py b/src/peft/config.py
index e7ef124d20..63f4ad6eea 100644
--- a/src/peft/config.py
+++ b/src/peft/config.py
@@ -130,12 +130,8 @@ def from_pretrained(cls, pretrained_model_name_or_path: str, subfolder: Optional
         else:
             config_cls = cls
 
-        config = config_cls(**class_kwargs)
-
-        for key, value in loaded_attributes.items():
-            if hasattr(config, key):
-                setattr(config, key, value)
-
+        kwargs = {**class_kwargs, **loaded_attributes}
+        config = config_cls(**kwargs)
         return config
 
     @classmethod
diff --git a/tests/test_config.py b/tests/test_config.py
index ca0ef0e3bd..2e38cba657 100644
--- a/tests/test_config.py
+++ b/tests/test_config.py
@@ -20,11 +20,15 @@
 import warnings
 
 import pytest
+from parameterized import parameterized
 
 from peft import (
+    AdaLoraConfig,
     AdaptionPromptConfig,
     IA3Config,
+    LoHaConfig,
     LoraConfig,
+    MultitaskPromptTuningConfig,
     PeftConfig,
     PrefixTuningConfig,
     PromptEncoder,
@@ -35,20 +39,22 @@
 
 PEFT_MODELS_TO_TEST = [("lewtun/tiny-random-OPTForCausalLM-delta", "v1")]
 
-
-class PeftConfigTestMixin:
-    all_config_classes = (
-        LoraConfig,
-        PromptEncoderConfig,
-        PrefixTuningConfig,
-        PromptTuningConfig,
-        AdaptionPromptConfig,
-        IA3Config,
-    )
+ALL_CONFIG_CLASSES = (
+    AdaptionPromptConfig,
+    AdaLoraConfig,
+    IA3Config,
+    LoHaConfig,
+    LoraConfig,
+    MultitaskPromptTuningConfig,
+    PrefixTuningConfig,
+    PromptEncoderConfig,
+    PromptTuningConfig,
+)
 
 
-class PeftConfigTester(unittest.TestCase, PeftConfigTestMixin):
-    def test_methods(self):
+class PeftConfigTester(unittest.TestCase):
+    @parameterized.expand(ALL_CONFIG_CLASSES)
+    def test_methods(self, config_class):
         r"""
         Test if all configs have the expected methods. Here we test
         - to_dict
@@ -57,109 +63,107 @@ def test_methods(self):
         - from_json_file
         """
         # test if all configs have the expected methods
-        for config_class in self.all_config_classes:
-            config = config_class()
-            self.assertTrue(hasattr(config, "to_dict"))
-            self.assertTrue(hasattr(config, "save_pretrained"))
-            self.assertTrue(hasattr(config, "from_pretrained"))
-            self.assertTrue(hasattr(config, "from_json_file"))
-
-    def test_task_type(self):
-        for config_class in self.all_config_classes:
-            # assert this will not fail
-            _ = config_class(task_type="test")
-
-    def test_from_pretrained(self):
+        config = config_class()
+        self.assertTrue(hasattr(config, "to_dict"))
+        self.assertTrue(hasattr(config, "save_pretrained"))
+        self.assertTrue(hasattr(config, "from_pretrained"))
+        self.assertTrue(hasattr(config, "from_json_file"))
+
+    @parameterized.expand(ALL_CONFIG_CLASSES)
+    def test_task_type(self, config_class):
+        config_class(task_type="test")
+
+    @parameterized.expand(ALL_CONFIG_CLASSES)
+    def test_from_pretrained(self, config_class):
         r"""
         Test if the config is correctly loaded using:
         - from_pretrained
         """
-        for config_class in self.all_config_classes:
-            for model_name, revision in PEFT_MODELS_TO_TEST:
-                # Test we can load config from delta
-                _ = config_class.from_pretrained(model_name, revision=revision)
+        for model_name, revision in PEFT_MODELS_TO_TEST:
+            # Test we can load config from delta
+            config_class.from_pretrained(model_name, revision=revision)
 
-    def test_save_pretrained(self):
+    @parameterized.expand(ALL_CONFIG_CLASSES)
+    def test_save_pretrained(self, config_class):
         r"""
         Test if the config is correctly saved and loaded using
         - save_pretrained
         """
-        for config_class in self.all_config_classes:
-            config = config_class()
-            with tempfile.TemporaryDirectory() as tmp_dirname:
-                config.save_pretrained(tmp_dirname)
+        config = config_class()
+        with tempfile.TemporaryDirectory() as tmp_dirname:
+            config.save_pretrained(tmp_dirname)
 
-                config_from_pretrained = config_class.from_pretrained(tmp_dirname)
-                self.assertEqual(config.to_dict(), config_from_pretrained.to_dict())
+            config_from_pretrained = config_class.from_pretrained(tmp_dirname)
+            self.assertEqual(config.to_dict(), config_from_pretrained.to_dict())
 
-    def test_from_json_file(self):
-        for config_class in self.all_config_classes:
-            config = config_class()
-            with tempfile.TemporaryDirectory() as tmp_dirname:
-                config.save_pretrained(tmp_dirname)
+    @parameterized.expand(ALL_CONFIG_CLASSES)
+    def test_from_json_file(self, config_class):
+        config = config_class()
+        with tempfile.TemporaryDirectory() as tmp_dirname:
+            config.save_pretrained(tmp_dirname)
 
-                config_from_json = config_class.from_json_file(os.path.join(tmp_dirname, "adapter_config.json"))
-                self.assertEqual(config.to_dict(), config_from_json)
+            config_from_json = config_class.from_json_file(os.path.join(tmp_dirname, "adapter_config.json"))
+            self.assertEqual(config.to_dict(), config_from_json)
 
-    def test_to_dict(self):
+    @parameterized.expand(ALL_CONFIG_CLASSES)
+    def test_to_dict(self, config_class):
         r"""
         Test if the config can be correctly converted to a dict using:
         - to_dict
         """
-        for config_class in self.all_config_classes:
-            config = config_class()
-            self.assertTrue(isinstance(config.to_dict(), dict))
+        config = config_class()
+        self.assertTrue(isinstance(config.to_dict(), dict))
 
-    def test_from_pretrained_cache_dir(self):
+    @parameterized.expand(ALL_CONFIG_CLASSES)
+    def test_from_pretrained_cache_dir(self, config_class):
         r"""
         Test if the config is correctly loaded with extra kwargs
         """
         with tempfile.TemporaryDirectory() as tmp_dirname:
-            for config_class in self.all_config_classes:
-                for model_name, revision in PEFT_MODELS_TO_TEST:
-                    # Test we can load config from delta
-                    _ = config_class.from_pretrained(model_name, revision=revision, cache_dir=tmp_dirname)
+            for model_name, revision in PEFT_MODELS_TO_TEST:
+                # Test we can load config from delta
+                config_class.from_pretrained(model_name, revision=revision, cache_dir=tmp_dirname)
 
     def test_from_pretrained_cache_dir_remote(self):
         r"""
         Test if the config is correctly loaded with a checkpoint from the hub
         """
         with tempfile.TemporaryDirectory() as tmp_dirname:
-            _ = PeftConfig.from_pretrained("ybelkada/test-st-lora", cache_dir=tmp_dirname)
+            PeftConfig.from_pretrained("ybelkada/test-st-lora", cache_dir=tmp_dirname)
             self.assertTrue("models--ybelkada--test-st-lora" in os.listdir(tmp_dirname))
 
-    def test_set_attributes(self):
+    @parameterized.expand(ALL_CONFIG_CLASSES)
+    def test_set_attributes(self, config_class):
         # manually set attributes and check if they are correctly written
-        for config_class in self.all_config_classes:
-            config = config_class(peft_type="test")
+        config = config_class(peft_type="test")
 
-            # save pretrained
-            with tempfile.TemporaryDirectory() as tmp_dirname:
-                config.save_pretrained(tmp_dirname)
+        # save pretrained
+        with tempfile.TemporaryDirectory() as tmp_dirname:
+            config.save_pretrained(tmp_dirname)
 
-                config_from_pretrained = config_class.from_pretrained(tmp_dirname)
-                self.assertEqual(config.to_dict(), config_from_pretrained.to_dict())
+            config_from_pretrained = config_class.from_pretrained(tmp_dirname)
+            self.assertEqual(config.to_dict(), config_from_pretrained.to_dict())
 
-    def test_config_copy(self):
+    @parameterized.expand(ALL_CONFIG_CLASSES)
+    def test_config_copy(self, config_class):
         # see https://github.com/huggingface/peft/issues/424
-        for config_class in self.all_config_classes:
-            config = config_class()
-            copied = copy.copy(config)
-            self.assertEqual(config.to_dict(), copied.to_dict())
+        config = config_class()
+        copied = copy.copy(config)
+        self.assertEqual(config.to_dict(), copied.to_dict())
 
-    def test_config_deepcopy(self):
+    @parameterized.expand(ALL_CONFIG_CLASSES)
+    def test_config_deepcopy(self, config_class):
         # see https://github.com/huggingface/peft/issues/424
-        for config_class in self.all_config_classes:
-            config = config_class()
-            copied = copy.deepcopy(config)
-            self.assertEqual(config.to_dict(), copied.to_dict())
+        config = config_class()
+        copied = copy.deepcopy(config)
+        self.assertEqual(config.to_dict(), copied.to_dict())
 
-    def test_config_pickle_roundtrip(self):
+    @parameterized.expand(ALL_CONFIG_CLASSES)
+    def test_config_pickle_roundtrip(self, config_class):
         # see https://github.com/huggingface/peft/issues/424
-        for config_class in self.all_config_classes:
-            config = config_class()
-            copied = pickle.loads(pickle.dumps(config))
-            self.assertEqual(config.to_dict(), copied.to_dict())
+        config = config_class()
+        copied = pickle.loads(pickle.dumps(config))
+        self.assertEqual(config.to_dict(), copied.to_dict())
 
     def test_prompt_encoder_warning_num_layers(self):
         # This test checks that if a prompt encoder config is created with an argument that is ignored, there should be
@@ -182,3 +186,15 @@ def test_prompt_encoder_warning_num_layers(self):
             PromptEncoder(config)
         expected_msg = "for MLP, the argument `encoder_num_layers` is ignored. Exactly 2 MLP layers are used."
         assert str(record.list[0].message) == expected_msg
+
+    @parameterized.expand([LoHaConfig, LoraConfig, IA3Config])
+    def test_save_pretrained_with_target_modules(self, config_class):
+        # See #1041, #1045
+        config = config_class(target_modules=["a", "list"])
+        with tempfile.TemporaryDirectory() as tmp_dirname:
+            config.save_pretrained(tmp_dirname)
+
+            config_from_pretrained = config_class.from_pretrained(tmp_dirname)
+            self.assertEqual(config.to_dict(), config_from_pretrained.to_dict())
+            # explicit test that target_modules should be converted to set
+            self.assertTrue(isinstance(config_from_pretrained.target_modules, set))

add_weighted_adapter() is unusable, throws error: "Invalid type <class 'list'> found in target_modules"
### System Info

- `transformers` version: 4.34.0
- Platform: Linux-6.5.6-arch2-1-x86_64-with-glibc2.35
- Python version: 3.10.12
- Huggingface_hub version: 0.17.3
- Safetensors version: 0.4.0
- Accelerate version: 0.23.0
- Accelerate config:    not found
- PyTorch version (GPU?): 2.1.0+cu121 (True)
- Tensorflow version (GPU?): not installed (NA)
- Flax version (CPU?/GPU?/TPU?): not installed (NA)
- Jax version: not installed
- JaxLib version: not installed
- Using GPU in script?: false
- Using distributed or parallel set-up in script?: false

### Who can help?

@pacman100 @you

### Information

- [ ] The official example scripts
- [X] My own modified scripts

### Tasks

- [ ] An officially supported task in the `examples` folder
- [X] My own task or dataset (give details below)

### Reproduction

Load the model:
```
model = AutoModelForCausalLM.from_pretrained(
        "RWKV/rwkv-4-169m-pile"
        device_map="auto"
    )
```
Convert it into a PeftModel (to make the `add_weighted_adapter()` method available):
```
model = PeftModel.from_pretrained(
        model, f"{adapter_dir}/{adapter}", adapter_name='adapter_1'
    )
```
Load the 2nd adapter:
```
model.load_adapter(
        f"{adapter_dir}/{adapter}", adapter_name='adapter_2'
    )
```
Attempt to merge the two adapters:
```
model.add_weighted_adapter(
        adapters=['adapter_1', 'adapter_2'],
        weights=[0.5, 0.5],
        adapter_name="combined",
        combination_type="svd",
    )
```
This will result in the error message:
```
Invalid type <class 'list'> found in target_modules
```

### Expected behavior

I have 2 fully trained LoRA adapters. The only configuration difference between them is the rank and alpha used for each.

I would like to merge the two adapters into a new adapter, and set that adapter to the "active" one.

However, trying to use the `add_weighted_adapter()` method always results in the following error:
```
Invalid type <class 'list'> found in target_modules
```

I would expect the `add_weighted_adapter()` method to accept a list, for the "target_modules" argument.

If you can provide any advice, I would greatly appreciate it. I suspect that this is either unsupported and/or not fully-implemented; or, it has something to do with the way I'm attaching adapters. I've tried a bunch of alternate configurations, but I'm not having luck.

Thanks in advance for any help you might provide.
