code: def network_apply_weights(
      self: Union[
          torch.nn.Conv2d,
          torch.nn.Linear,
          torch.nn.GroupNorm,
          torch.nn.LayerNorm,
          torch.nn.MultiheadAttention,
      ]
  ):
      """
      Applies the currently selected set of networks to the weights of torch layer self.
      If weights already have this particular set of networks applied, does nothing.
      If not, restores orginal weights from backup and alters weights according to networks.
      """
  
      network_layer_name = getattr(self, "network_layer_name", None)
      if network_layer_name is None:
          return
  
      current_names = getattr(self, "network_current_names", ())
      wanted_names = tuple(
          (x.name, x.te_multiplier, x.unet_multiplier, x.dyn_dim)
          for x in loaded_networks
      )
  
      weights_backup = getattr(self, "network_weights_backup", None)
      if weights_backup is None and wanted_names != ():
          if current_names != ():
              raise RuntimeError(
                  "no backup weights found and current weights are not unchanged"
              )
  
          if isinstance(self, torch.nn.MultiheadAttention):
              weights_backup = (
                  self.in_proj_weight.to(devices.cpu, copy=True),
                  self.out_proj.weight.to(devices.cpu, copy=True),
              )
          else:
              weights_backup = self.weight.to(devices.cpu, copy=True)
  
          self.network_weights_backup = weights_backup
  
      bias_backup = getattr(self, "network_bias_backup", None)
      if bias_backup is None:
          if (
              isinstance(self, torch.nn.MultiheadAttention)
              and self.out_proj.bias is not None
          ):
              bias_backup = self.out_proj.bias.to(
                  devices.cpu, copy=True
              )
          elif getattr(self, "bias", None) is not None:
              bias_backup = self.bias.to(devices.cpu, copy=True)
          else:
              bias_backup = None
          self.network_bias_backup = bias_backup
  
      if current_names != wanted_names:
          network_restore_weights_from_backup(self)
  
          for net in loaded_networks:
              module = net.modules.get(network_layer_name, None)
              if module is not None and hasattr(self, "weight"):
                  try:
                      with torch.no_grad():
                          updown, ex_bias = module.calc_updown(
                              self.weight
                          )
  
                          if (
                              len(self.weight.shape) == 4
                              and self.weight.shape[1] == 9
                          ):
                              # inpainting model. zero pad updown to make channel[1]  4 to 9
                              updown = torch.nn.functional.pad(
                                  updown, (0, 0, 0, 0, 0, 5)
                              )
  
                          self.weight += updown
                          if ex_bias is not None and hasattr(
                              self, "bias"
                          ):
                              if self.bias is None:
                                  self.bias = torch.nn.Parameter(
                                      ex_bias
                                  )
                              else:
                                  self.bias += ex_bias
                  except RuntimeError as e:
                      logging.debug(
                          f"Network {net.name} layer {network_layer_name}: {e}"
                      )
                      extra_network_lora.errors[net.name] = (
                          extra_network_lora.errors.get(net.name, 0) + 1
                      )
  
                  continue
  
              module_q = net.modules.get(
                  network_layer_name + "_q_proj", None
              )
              module_k = net.modules.get(
                  network_layer_name + "_k_proj", None
              )
              module_v = net.modules.get(
                  network_layer_name + "_v_proj", None
              )
              module_out = net.modules.get(
                  network_layer_name + "_out_proj", None
              )
  
              if (
                  isinstance(self, torch.nn.MultiheadAttention)
                  and module_q
                  and module_k
                  and module_v
                  and module_out
              ):
                  try:
                      with torch.no_grad():
                          updown_q, _ = module_q.calc_updown(
                              self.in_proj_weight
                          )
                          updown_k, _ = module_k.calc_updown(
                              self.in_proj_weight
                          )
                          updown_v, _ = module_v.calc_updown(
                              self.in_proj_weight
                          )
                          updown_qkv = torch.vstack(
                              [updown_q, updown_k, updown_v]
                          )
                          updown_out, ex_bias = module_out.calc_updown(
                              self.out_proj.weight
                          )
  
                          self.in_proj_weight += updown_qkv
                          self.out_proj.weight += updown_out
                      if ex_bias is not None:
                          if self.out_proj.bias is None:
                              self.out_proj.bias = torch.nn.Parameter(
                                  ex_bias
                              )
                          else:
                              self.out_proj.bias += ex_bias
  
                  except RuntimeError as e:
                      logging.debug(
                          f"Network {net.name} layer {network_layer_name}: {e}"
                      )
                      extra_network_lora.errors[net.name] = (
                          extra_network_lora.errors.get(net.name, 0) + 1
                      )
  
                  continue
  
              if module is None:
                  continue
  
              logging.debug(
                  f"Network {net.name} layer {network_layer_name}: couldn't find supported operation"
              )
              extra_network_lora.errors[net.name] = (
                  extra_network_lora.errors.get(net.name, 0) + 1
              )
  
          self.network_current_names = wanted_names
  
imports: import logging
  import os
  import re
  import lora_patches
  import network
  import network_lora
  import network_glora
  import network_hada
  import network_ia3
  import network_lokr
  import network_full
  import network_norm
  import network_oft
  import torch
  from typing import Union
  from modules import shared, devices, sd_models, errors, scripts, sd_hijack
  import modules.textual_inversion.textual_inversion
  from lora_logger import logger
label_desc: Use image node_id 7 for calling a global function with example usage: network_apply_weights() without return types