src/actrec.py
def adam(lr, tparams, grads, inp, cost):
    """
    Adam: A Method for Stochastic Optimization (Diederik Kingma, Jimmy Ba)
    """
    gshared = [theano.shared(p.get_value() * numpy.float32(0.), name='%s_grad'%k) for k, p in tparams.iteritems()]
    gsup = [(gs, g) for gs, g in zip(gshared, grads)]
    #print '\n\ncheck\n\n'
    f_grad_shared = theano.function(inp, cost, updates=gsup, allow_input_downcast=True)

    # Magic numbers
    lr0 = 0.0002
    b1 = 0.1
    b2 = 0.001
    e = 1e-8

    updates = []

    i = theano.shared(numpy.float32(0.))
    i_t = i + 1.
    fix1 = 1. - b1**(i_t)
    fix2 = 1. - b2**(i_t)
    lr_t = lr0 * (tensor.sqrt(fix2) / fix1)

    for p, g in zip(tparams.values(), gshared):
        m = theano.shared(p.get_value() * numpy.float32(0.))
        v = theano.shared(p.get_value() * numpy.float32(0.))
        m_t = (b1 * g) + ((1. - b1) * m)
        v_t = (b2 * tensor.sqr(g)) + ((1. - b2) * v)
        g_t = m_t / (tensor.sqrt(v_t) + e)
        p_t = p - (lr_t * g_t)
        updates.append((m, m_t))
        updates.append((v, v_t))
        updates.append((p, p_t))
    updates.append((i, i_t))

    f_update = theano.function([lr], [], updates=updates, on_unused_input='ignore')

    return f_grad_shared, f_update

    Adam: A Method for Stochastic Optimization (Diederik Kingma, Jimmy Ba)
    
A Method for Stochastic Optimization (Diederik Kingma, Jimmy Ba)
A Method for Stochastic Optimization ( Diederik Kingma Jimmy Ba )
def adam(lr, tparams, grads, inp, cost):
    gshared = [theano.shared(p.get_value() * numpy.float32(0.), name='%s_grad'%k) for k, p in tparams.iteritems()]
    gsup = [(gs, g) for gs, g in zip(gshared, grads)]
    f_grad_shared = theano.function(inp, cost, updates=gsup, allow_input_downcast=True)
    lr0 = 0.0002
    b1 = 0.1
    b2 = 0.001
    e = 1e-8
    updates = []
    i = theano.shared(numpy.float32(0.))
    i_t = i + 1.
    fix1 = 1. - b1**(i_t)
    fix2 = 1. - b2**(i_t)
    lr_t = lr0 * (tensor.sqrt(fix2) / fix1)
    for p, g in zip(tparams.values(), gshared):
        m = theano.shared(p.get_value() * numpy.float32(0.))
        v = theano.shared(p.get_value() * numpy.float32(0.))
        m_t = (b1 * g) + ((1. - b1) * m)
        v_t = (b2 * tensor.sqr(g)) + ((1. - b2) * v)
        g_t = m_t / (tensor.sqrt(v_t) + e)
        p_t = p - (lr_t * g_t)
        updates.append((m, m_t))
        updates.append((v, v_t))
        updates.append((p, p_t))
    updates.append((i, i_t))
    f_update = theano.function([lr], [], updates=updates, on_unused_input='ignore')
    return f_grad_shared, f_update
def adam ( lr , tparams , grads , inp , cost ) : gshared = [ theano . shared ( p . get_value ( ) * numpy . float32 ( 0. ) , name = '%s_grad' % k ) for k , p in tparams . iteritems ( ) ] gsup = [ ( gs , g ) for gs , g in zip ( gshared , grads ) ] f_grad_shared = theano . function ( inp , cost , updates = gsup , allow_input_downcast = True ) lr0 = 0.0002 b1 = 0.1 b2 = 0.001 e = 1e-8 updates = [ ] i = theano . shared ( numpy . float32 ( 0. ) ) i_t = i + 1. fix1 = 1. - b1 ** ( i_t ) fix2 = 1. - b2 ** ( i_t ) lr_t = lr0 * ( tensor . sqrt ( fix2 ) / fix1 ) for p , g in zip ( tparams . values ( ) , gshared ) : m = theano . shared ( p . get_value ( ) * numpy . float32 ( 0. ) ) v = theano . shared ( p . get_value ( ) * numpy . float32 ( 0. ) ) m_t = ( b1 * g ) + ( ( 1. - b1 ) * m ) v_t = ( b2 * tensor . sqr ( g ) ) + ( ( 1. - b2 ) * v ) g_t = m_t / ( tensor . sqrt ( v_t ) + e ) p_t = p - ( lr_t * g_t ) updates . append ( ( m , m_t ) ) updates . append ( ( v , v_t ) ) updates . append ( ( p , p_t ) ) updates . append ( ( i , i_t ) ) f_update = theano . function ( [ lr ] , [ ] , updates = updates , on_unused_input = 'ignore' ) return f_grad_shared , f_update
Adam: A Method for Stochastic Optimization (Diederik Kingma, Jimmy Ba)
Adam : A Method for Stochastic Optimization ( Diederik Kingma Jimmy Ba )
"""
    Adam: A Method for Stochastic Optimization (Diederik Kingma, Jimmy Ba)
    """ #print '\n\ncheck\n\n' # Magic numbers
lr
tparams
grads
inp
cost