10601-hws/HW 6/hw6/python/cnn_lenet.py
def sgd_momentum(w_rate, b_rate, mu, decay, params, param_winc, param_grad):
  """Update the parameters with sgd with momentum

  Args:
    w_rate: sgd rate for updating w
    b_rate: sgd rate for updating b
    mu: momentum
    decay: weight decay of w
    params: original weight parameters
    param_winc: buffer to store history gradient accumulation
    param_grad: gradient of parameter

  Returns:
    params_: updated parameters
    param_winc_: gradient buffer of previous step
  """
  params_ = copy.deepcopy(params)
  param_winc_ = copy.deepcopy(param_winc)

  for layerNumber in param_winc_:

    param_winc_[layerNumber]['w'] = (mu * param_winc_[layerNumber]['w']) + (w_rate * (param_grad[layerNumber]['w']  + (decay * params[layerNumber]['w'])))
    param_winc_[layerNumber]['b'] = (mu * param_winc_[layerNumber]['b']) + (b_rate * param_grad[layerNumber]['b'])

    params_[layerNumber]['w'] = params_[layerNumber]['w'] - param_winc_[layerNumber]['w']
    params_[layerNumber]['b'] = params_[layerNumber]['b'] - param_winc_[layerNumber]['b']

  assert len(params_) == len(param_grad), 'params_ does not have the right length (error at sgd momentum)'
  assert len(param_winc_) == len(param_grad), 'param_winc_ does not have the right length (error at sgd momentum)'
  return params_, param_winc_
Update the parameters with sgd with momentum

  Args:
    w_rate: sgd rate for updating w
    b_rate: sgd rate for updating b
    mu: momentum
    decay: weight decay of w
    params: original weight parameters
    param_winc: buffer to store history gradient accumulation
    param_grad: gradient of parameter

  Returns:
    params_: updated parameters
    param_winc_: gradient buffer of previous step
  
Update the parameters with sgd with momentum
Update the parameters with sgd with momentum
def sgd_momentum(w_rate, b_rate, mu, decay, params, param_winc, param_grad):
  params_ = copy.deepcopy(params)
  param_winc_ = copy.deepcopy(param_winc)
  for layerNumber in param_winc_:
    param_winc_[layerNumber]['w'] = (mu * param_winc_[layerNumber]['w']) + (w_rate * (param_grad[layerNumber]['w']  + (decay * params[layerNumber]['w'])))
    param_winc_[layerNumber]['b'] = (mu * param_winc_[layerNumber]['b']) + (b_rate * param_grad[layerNumber]['b'])
    params_[layerNumber]['w'] = params_[layerNumber]['w'] - param_winc_[layerNumber]['w']
    params_[layerNumber]['b'] = params_[layerNumber]['b'] - param_winc_[layerNumber]['b']
  assert len(params_) == len(param_grad), 'params_ does not have the right length (error at sgd momentum)'
  assert len(param_winc_) == len(param_grad), 'param_winc_ does not have the right length (error at sgd momentum)'
  return params_, param_winc_
def sgd_momentum ( w_rate , b_rate , mu , decay , params , param_winc , param_grad ) : params_ = copy . deepcopy ( params ) param_winc_ = copy . deepcopy ( param_winc ) for layerNumber in param_winc_ : param_winc_ [ layerNumber ] [ 'w' ] = ( mu * param_winc_ [ layerNumber ] [ 'w' ] ) + ( w_rate * ( param_grad [ layerNumber ] [ 'w' ] + ( decay * params [ layerNumber ] [ 'w' ] ) ) ) param_winc_ [ layerNumber ] [ 'b' ] = ( mu * param_winc_ [ layerNumber ] [ 'b' ] ) + ( b_rate * param_grad [ layerNumber ] [ 'b' ] ) params_ [ layerNumber ] [ 'w' ] = params_ [ layerNumber ] [ 'w' ] - param_winc_ [ layerNumber ] [ 'w' ] params_ [ layerNumber ] [ 'b' ] = params_ [ layerNumber ] [ 'b' ] - param_winc_ [ layerNumber ] [ 'b' ] assert len ( params_ ) == len ( param_grad ) , 'params_ does not have the right length (error at sgd momentum)' assert len ( param_winc_ ) == len ( param_grad ) , 'param_winc_ does not have the right length (error at sgd momentum)' return params_ , param_winc_
Update the parameters with sgd with momentum
Update the parameters with sgd with momentum
"""Update the parameters with sgd with momentum

  Args:
    w_rate: sgd rate for updating w
    b_rate: sgd rate for updating b
    mu: momentum
    decay: weight decay of w
    params: original weight parameters
    param_winc: buffer to store history gradient accumulation
    param_grad: gradient of parameter

  Returns:
    params_: updated parameters
    param_winc_: gradient buffer of previous step
  """
updated parameters
param_winc_: gradient buffer of previous step
updated parameters param_winc_ : gradient buffer of previous step
params_
w_rate
sgd rate for updating w
sgd rate for updating w
b_rate
sgd rate for updating b
sgd rate for updating b
mu
None
decay
weight decay of w
weight decay of w
params
original weight parameters
original weight parameters
param_winc
buffer to store history gradient accumulation
buffer to store history gradient accumulation
param_grad
gradient of parameter
gradient of parameter