code: def test_train_adapter_fusion(self):
      if not self.do_run_train_tests:
          self.skipTest(
              "Skipping training tests. Set `do_run_train_tests=True` to run them."
          )
      if self.config_class not in ADAPTER_MODEL_MAPPING:
          self.skipTest("Does not support flex heads.")
      model = AutoAdapterModel.from_config(self.config())
      self.add_head(model, "head")
  
      # add the adapters to be fused
      model.add_adapter("a")
      model.add_adapter("b")
      model.add_adapter("c")
  
      self.assertIn("a", model.adapters_config.adapters)
      self.assertIn("b", model.adapters_config.adapters)
      self.assertIn("c", model.adapters_config.adapters)
  
      # setup fusion
      adapter_setup = Fuse("a", "b", "c")
      model.add_adapter_fusion(adapter_setup)
      model.train_adapter_fusion(adapter_setup)
      model.set_active_adapters(adapter_setup)
      self.assertEqual(adapter_setup, model.active_adapters)
  
      # all weights of the adapters should be frozen (test for one)
      for k, v in self.filter_parameters(
          model, ["adapters.a."]
      ).items():
          self.assertFalse(v.requires_grad, k)
      # all weights of the fusion layer should be activated
      for k, v in self.filter_parameters(
          model, ["adapter_fusion_layer"]
      ).items():
          self.assertTrue(v.requires_grad, k)
      # weights of the model should be frozen (check on some examples)
      for k, v in self.filter_parameters(
          model, ["encoder.layer.0.attention"]
      ).items():
          self.assertFalse(v.requires_grad, k)
  
      state_dict_pre = copy.deepcopy(model.state_dict())
  
      # Since our config has a value matrix, make sure it is regularized.
      # We do this by patching the fusion regularization function.
      regularization_called = False
      orig_fusion_regularization_loss = (
          model.base_model.get_fusion_regularization_loss
      )
  
      def patched_fusion_reg_loss():
          nonlocal regularization_called
          regularization_called = True
          return orig_fusion_regularization_loss()
  
      model.base_model.get_fusion_regularization_loss = (
          patched_fusion_reg_loss
      )
  
      self.trainings_run(model)
      self.assertTrue(regularization_called)
  
      def has_tied_embeddings(k):
          tied_embeddings = (
              hasattr(model.config, "tie_word_embeddings")
              and model.config.tie_word_embeddings
          )
          is_tied_layer = (
              isinstance(model.heads["head"], CausalLMHead)
              and "heads.{}.{}.weight".format(
                  "head", len(model.heads["head"]._modules) - 1
              )
              in k
          )
          return tied_embeddings and is_tied_layer
  
      # check that the adapters have changed, but the base model has not
      adapters_with_change, base_with_change = False, False
      for (k1, v1), (k2, v2) in zip(
          state_dict_pre.items(), model.state_dict().items()
      ):
          if (
              "adapter_fusion_layer" in k1
              or "classifier" in k1
              or "classification_head" in k1
              or "score" in k1
              or "head" in k1
          ) and not has_tied_embeddings(k1):
              adapters_with_change |= not torch.equal(v1, v2)
          else:
              base_with_change |= not torch.equal(v1, v2)
      self.assertTrue(adapters_with_change)
      self.assertFalse(base_with_change)
  
imports: import copy
  import tempfile
  import torch
  import adapters
  from adapters import ADAPTER_CONFIG_MAP, ADAPTER_MODEL_MAPPING, AutoAdapterModel, BatchSplit, DoubleSeqBnConfig, DoubleSeqBnInvConfig, Fuse, InvertibleAdaptersMixin, InvertibleAdaptersWrapperMixin, MAMConfig, SeqBnConfig, SeqBnInvConfig
  from adapters.heads.language_modeling import CausalLMHead
  from transformers import MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING, CLIPConfig
  from transformers.testing_utils import require_torch, torch_device
  from .base import AdapterMethodBaseTestMixin, create_twin_models
label_desc: Use image node_id 22 for calling the BottleneckAdapterTestMixin obj's underlying member method code with example usage: obj.test_train_adapter_fusion() and returns: orig_fusion_regularization_loss, unknown