deepy/trainers/cores/nag.py
def nag_core(params, J, momentum=0.9, learning_rate=0.01):
    """
    Nesterov's Accelerated Gradient (NAG).
    See http://www.cs.toronto.edu/~fritz/absps/momentum.pdf .
    Still unfinished
    """
    # TODO: this requires some refractorings.
    for param in params:
        step = theano.shared(np.zeros_like(param.get_value()), name=param.name + '_step')
        velocity = theano.shared(np.zeros_like(param.get_value()), name=param.name + '_vel')
        yield step, momentum * velocity
        yield param, param + step
        yield velocity, step - learning_rate * T.grad(J, param)
        yield param, param + velocity - step
def nag_core(params, J, momentum=0.9, learning_rate=0.01):
    """
    Nesterov's Accelerated Gradient (NAG).
    See http://www.cs.toronto.edu/~fritz/absps/momentum.pdf .
    Still unfinished
    """
    # TODO: this requires some refractorings.
    for param in params:
        step = theano.shared(np.zeros_like(param.get_value()), name=param.name + '_step')
        velocity = theano.shared(np.zeros_like(param.get_value()), name=param.name + '_vel')
        yield step, momentum * velocity
        yield param, param + step
        yield velocity, step - learning_rate * T.grad(J, param)
        yield param, param + velocity - step
def nag_core ( params , J , momentum = 0.9 , learning_rate = 0.01 ) : # TODO: this requires some refractorings. for param in params : step = theano . shared ( np . zeros_like ( param . get_value ( ) ) , name = param . name + '_step' ) velocity = theano . shared ( np . zeros_like ( param . get_value ( ) ) , name = param . name + '_vel' ) yield step , momentum * velocity yield param , param + step yield velocity , step - learning_rate * T . grad ( J , param ) yield param , param + velocity - step
Nesterov's Accelerated Gradient (NAG).
    See http://www.cs.toronto.edu/~fritz/absps/momentum.pdf .
    Still unfinished
Nesterov s Accelerated Gradient ( NAG ) . See http : // www . cs . toronto . edu / ~fritz / absps / momentum . pdf . Still unfinished