class_docstr: ## L2 Weight decay
docstr: ### Initialize weight decay
  
  * `weight_decay` is the decay coefficient
  * `weight_decouple` is a flag indicating whether to add the weight decay to the gradient or directly
  decay from the parameter. If added to the  gradient it will go through the normal optimizer update.
  * `absolute` this flag indicates whether the weight decay coefficient is absolute. This is applicable
  when the decay is performed directly on the parameter. If this is false the actual decay is
  `weight_decay`
  * `learning_rate`.
docstr_tok: ["#","#","#","Initialize","weight","decay","*","`","weight_decay","`","is","the","decay","coefficient","*","`","weight_decouple","`","is","a","flag","indicating","whether","to","add","the","weight","decay","to","the","gradient","or","directly","decay","from","the","parameter",".","If","added","to","the","gradient","it","will","go","through","the","normal","optimizer","update",".","*","`","absolute","`","this","flag","indicates","whether","the","weight","decay","coefficient","is","absolute",".","This","is","applicable","when","the","decay","is","performed","directly","on","the","parameter",".","If","this","is","false","the","actual","decay","is","`","weight_decay","`","*","`","learning_rate","`","."]
code: def __init__(
      self,
      weight_decay: float = 0.0,
      weight_decouple: bool = True,
      absolute: bool = False,
  ):
      """
      ### Initialize weight decay
  
      * `weight_decay` is the decay coefficient
      * `weight_decouple` is a flag indicating whether to add the weight decay to the gradient or directly
      decay from the parameter. If added to the  gradient it will go through the normal optimizer update.
      * `absolute` this flag indicates whether the weight decay coefficient is absolute. This is applicable
      when the decay is performed directly on the parameter. If this is false the actual decay is
      `weight_decay`
      * `learning_rate`.
      """
      # Check hyper-parameters
      if not 0.0 <= weight_decay:
          raise ValueError(
              f"Invalid weight_decay value: {weight_decay}"
          )
  
      self.absolute = absolute
      self.weight_decouple = weight_decouple
      self.weight_decay = weight_decay
  
file_path: annotated_deep_learning_paper_implementations/labml_nn/optimizers/__init__.py
imports: from typing import Dict, Tuple, Any
  import torch
  from torch import nn
  from torch.optim.optimizer import Optimizer
label_desc: Use image node_id 1 to create a new WeightDecay object  with example: obj = WeightDecay(weight_decay, weight_decouple, absolute)