code: def _apply_penalties(extra_out, args):
      """Based on `args`, optionally adds regularization penalty terms for
      activation regularization, temporal activation regularization and/or hidden
      state norm stabilization.
  
      Args:
          extra_out[*]:
              dropped: Post-dropout activations.
              hiddens: All hidden states for a batch of sequences.
              raw: Pre-dropout activations.
  
      Returns:
          The penalty term associated with all of the enabled regularizations.
  
      See:
          Regularizing and Optimizing LSTM Language Models (Merity et al., 2017)
          Regularizing RNNs by Stabilizing Activations (Krueger & Memsevic, 2016)
      """
      penalty = 0
  
      # Activation regularization.
      if args.activation_regularization:
          penalty += (
              args.activation_regularization_amount
              * extra_out["dropped"].pow(2).mean()
          )
  
      # Temporal activation regularization (slowness)
      if args.temporal_activation_regularization:
          raw = extra_out["raw"]
          penalty += (
              args.temporal_activation_regularization_amount
              * (raw[1:] - raw[:-1]).pow(2).mean()
          )
  
      # Norm stabilizer regularization
      if args.norm_stabilizer_regularization:
          penalty += (
              args.norm_stabilizer_regularization_amount
              * (
                  extra_out["hiddens"].norm(dim=-1)
                  - args.norm_stabilizer_fixed_point
              )
              .pow(2)
              .mean()
          )
  
      return penalty
  
imports: import contextlib
  import glob
  import math
  import os
  import numpy
  import scipy.signal
  from tensorboard import TensorBoard
  import torch
  from torch import nn
  import torch.nn.parallel
  from torch.autograd import Variable
  import models
  import utils
label_desc: Use image node_id 1 for calling a global function with example usage: _apply_penalties(extra_out, args) and returns: penalty