code: def get_lr_heads_param_groups(
      model: nn.Module,
      model_config: AttrDict,
      optimizer_config: AttrDict,
      optimizer_schedulers: Dict[str, ParamScheduler],
  ):
      """
      Parameter group that can be used in conjunction with linear evaluation
      to have different heads used different learning rates
      """
  
      regex_head_index = re.compile(r"heads\.([0-9]*)")
  
      parameter_groups = []
      lr_heads = optimizer_config.linear_eval_heads
      default_weight_decay = optimizer_config.weight_decay
  
      for name, module in model.named_modules():
          if "heads" not in name:
              continue
  
          match = regex_head_index.match(name)
          if match is not None:
              head_index = int(match.group(1))
              lr_decay = lr_heads[head_index]["lr"]
              weight_decay = lr_heads[head_index].get(
                  "weight_decay", default_weight_decay
              )
              regul_bn = lr_heads[head_index].get(
                  "regularize_bn", optimizer_config["regularize_bn"]
              )
          else:
              lr_decay = 1.0
              weight_decay = default_weight_decay
              regul_bn = optimizer_config["regularize_bn"]
  
          unregularized_params = []
          regularized_params = []
  
          if isinstance(module, nn.Linear) or isinstance(
              module, _CONV_TYPES
          ):
              regularized_params.append(module.weight)
              if module.bias is not None:
                  if optimizer_config["regularize_bias"]:
                      regularized_params.append(module.bias)
                  else:
                      unregularized_params.append(module.bias)
          elif isinstance(module, _NORM_TYPES):
              if module.weight is not None:
                  if regul_bn:
                      regularized_params.append(module.weight)
                  else:
                      unregularized_params.append(module.weight)
              if module.bias is not None:
                  if regul_bn and optimizer_config["regularize_bias"]:
                      regularized_params.append(module.bias)
                  else:
                      unregularized_params.append(module.bias)
  
          update_interval = optimizer_schedulers["lr"].update_interval
          lr = LayerDecayScheduler(
              optimizer_schedulers["lr"], lr_decay, update_interval
          )
  
          if unregularized_params:
              parameter_groups.append(
                  {
                      "params": unregularized_params,
                      "lr": lr,
                      "weight_decay": 0.0,
                  }
              )
          if regularized_params:
              parameter_groups.append(
                  {
                      "params": regularized_params,
                      "lr": lr,
                      "weight_decay": weight_decay,
                  }
              )
  
      return parameter_groups
  
file_path: vissl/vissl/optimizers/param_groups/layer_decay.py
imports: import re
  from typing import Dict
  import torch.nn
  from fvcore.common.param_scheduler import ParamScheduler
  from vissl.config import AttrDict
  from vissl.optimizers.param_groups.registry import register_param_group_constructor
  from vissl.optimizers.param_scheduler.layer_decay_scheduler import LayerDecayScheduler
  from vissl.utils.misc import is_apex_available
label_desc: Use image node_id 6 for calling a global function with example usage: get_lr_heads_param_groups(model, model_config, optimizer_config, optimizer_schedulers) and returns: parameter_groups