code: def _check_loading(
      self, model_cls, config0, config1, input, *, is_commutative
  ):
      # Check that we can load two adapters into the same model
      # Note that we save the adapters using a normal PeftModel because PeftMixModel doesn't support saving yet
      atol = 1e-5
      rtol = 1e-5
      seed0 = 0
      seed1 = 1
  
      with tempfile.TemporaryDirectory() as tmp_dirname:
          # SAVING
          # adapter 0: note that we set mixed=False because mixed models don't support saving (yet)
          peft_model_0 = self._get_model(
              model_cls, config0, "adapter0", seed=seed0, mixed=False
          )
          output_config0 = peft_model_0(input)
          peft_model_0.save_pretrained(
              os.path.join(tmp_dirname, "adapter0")
          )
  
          # adapter 1: note that we set mixed=False because mixed models don't support saving (yet)
          peft_model_1 = self._get_model(
              model_cls, config1, "adapter1", seed=seed1, mixed=False
          )
          output_config1 = peft_model_1(input)
          peft_model_1.save_pretrained(
              os.path.join(tmp_dirname, "adapter1")
          )
  
          # adapter 0 + 1
          peft_model_01 = self._get_model(
              model_cls, config0, "adapter0", seed=seed0
          )
          torch.manual_seed(seed1)
          peft_model_01.add_adapter("adapter1", config1)
          peft_model_01.set_adapter(["adapter0", "adapter1"])
          output_mixed_01 = peft_model_01(input)
  
          # adapter 1 + 0
          peft_model_10 = self._get_model(
              model_cls, config1, "adapter1", seed=seed1
          )
          torch.manual_seed(seed0)
          peft_model_10.add_adapter("adapter0", config0)
          peft_model_10.set_adapter(["adapter1", "adapter0"])
          output_mixed_10 = peft_model_10(input)
  
          # LOADING
          # adapter 0
          base_model = self._get_model(model_cls)
          # Notes:
          # Path is tmp_dirname/adapter0/adapter0 because non-default adapters are saved in a subfolder.
          # As a sanity check, we should set a completely different seed here. That way, we ensure that the the
          # weights are not just randomly initialized exactly to the same values as before.
          torch.manual_seed(123456)
          peft_model_loaded0 = PeftMixedModel.from_pretrained(
              base_model,
              os.path.join(tmp_dirname, "adapter0", "adapter0"),
              "adapter0",
          )
          output_loaded0 = peft_model_loaded0(input)
          self.assertTrue(
              torch.allclose(
                  output_config0, output_loaded0, atol=atol, rtol=rtol
              )
          )
  
          # adapter 1
          base_model = self._get_model(model_cls)
          torch.manual_seed(
              654321
          )  # setting a completely different seed here should not affect the result
          peft_model_loaded1 = PeftMixedModel.from_pretrained(
              base_model,
              os.path.join(tmp_dirname, "adapter1", "adapter1"),
              "adapter1",
          )
          output_loaded1 = peft_model_loaded1(input)
          self.assertTrue(
              torch.allclose(
                  output_config1, output_loaded1, atol=atol, rtol=rtol
              )
          )
  
          # adapter 0 + 1
          base_model = self._get_model(model_cls)
          torch.manual_seed(
              97531
          )  # setting a completely different seed here should not affect the result
          peft_model_loaded_01 = PeftMixedModel.from_pretrained(
              base_model,
              os.path.join(tmp_dirname, "adapter0", "adapter0"),
              "adapter0",
          )
          peft_model_loaded_01.load_adapter(
              os.path.join(tmp_dirname, "adapter1", "adapter1"),
              "adapter1",
          )
          # at this point, "adapter0" should still be active
          self.assertEqual(
              peft_model_loaded_01.active_adapters, ["adapter0"]
          )
          output_loaded01_0 = peft_model_loaded_01(input)
          self.assertTrue(
              torch.allclose(
                  output_config0,
                  output_loaded01_0,
                  atol=atol,
                  rtol=rtol,
              )
          )
          # activate adapter1
          peft_model_loaded_01.set_adapter(["adapter1"])
          self.assertEqual(
              peft_model_loaded_01.active_adapters, ["adapter1"]
          )
          output_loaded01_1 = peft_model_loaded_01(input)
          self.assertTrue(
              torch.allclose(
                  output_config1,
                  output_loaded01_1,
                  atol=atol,
                  rtol=rtol,
              )
          )
          # activate both adapters
          peft_model_loaded_01.set_adapter(["adapter0", "adapter1"])
          output_loaded01 = peft_model_loaded_01(input)
          self.assertTrue(
              torch.allclose(
                  output_mixed_01, output_loaded01, atol=atol, rtol=rtol
              )
          )
  
          # adapter 1 + 0
          base_model = self._get_model(model_cls)
          torch.manual_seed(
              445566
          )  # setting a completely different seed here should not affect the result
          peft_model_loaded_10 = PeftMixedModel.from_pretrained(
              base_model,
              os.path.join(tmp_dirname, "adapter1", "adapter1"),
              "adapter1",
          )
          peft_model_loaded_10.load_adapter(
              os.path.join(tmp_dirname, "adapter0", "adapter0"),
              "adapter0",
          )
          # at this point, "adapter1" should still be active
          self.assertEqual(
              peft_model_loaded_10.active_adapters, ["adapter1"]
          )
          output_loaded10_1 = peft_model_loaded_10(input)
          self.assertTrue(
              torch.allclose(
                  output_config1,
                  output_loaded10_1,
                  atol=atol,
                  rtol=rtol,
              )
          )
          # activate adapter1
          peft_model_loaded_10.set_adapter(["adapter0"])
          self.assertEqual(
              peft_model_loaded_10.active_adapters, ["adapter0"]
          )
          output_loaded10_0 = peft_model_loaded_10(input)
          self.assertTrue(
              torch.allclose(
                  output_config0,
                  output_loaded10_0,
                  atol=atol,
                  rtol=rtol,
              )
          )
          # activate both adapters
          peft_model_loaded_10.set_adapter(["adapter1", "adapter0"])
          output_loaded10 = peft_model_loaded_10(input)
          self.assertTrue(
              torch.allclose(
                  output_mixed_10, output_loaded10, atol=atol, rtol=rtol
              )
          )
  
          if is_commutative:
              self.assertTrue(
                  torch.allclose(
                      output_loaded01,
                      output_loaded10,
                      atol=atol,
                      rtol=rtol,
                  )
              )
              self.assertTrue(
                  torch.allclose(
                      output_loaded10,
                      output_mixed_01,
                      atol=atol,
                      rtol=rtol,
                  )
              )
  
imports: import copy
  import itertools
  import os
  import re
  import tempfile
  import unittest
  import torch
  from parameterized import parameterized
  from torch import nn
  from transformers import AutoModelForCausalLM
  from peft import AdaLoraConfig, LoHaConfig, LoKrConfig, LoraConfig, OFTConfig, PeftMixedModel, PrefixTuningConfig, get_peft_model
  from peft.tuners.tuners_utils import BaseTunerLayer
  from peft.utils import infer_device
label_desc: Use image node_id 6 for calling the TestMixedAdapterTypes obj's underlying member method code with example usage: obj._check_loading(model_cls, config0, config1, input) without return types