src/actrec.py
def build_model(tparams, options):
    """ Builds the entire computational graph used for training
  
    Parameters
    ----------
    tparams : OrderedDict
        maps names of variables to theano shared variables
    options : dict
        big dictionary with all the settings and hyperparameters
    
    Returns
    -------
    trng: theano random number generator
        Used for dropout, stochastic attention, etc
    use_noise: theano shared variable
        flag that toggles noise on and off
    [x, mask, ctx]: theano variables
        Represent the captions, binary mask, and annotations
        for a single batch (see dimensions below)
    alphas: theano variables
        Attention weights
    cost: theano variable
        negative log likelihood
    opt_outs: OrderedDict
        extra outputs required depending on configuration in options
    preds:
	computed labels
    """
    trng = RandomStreams(1234)
    use_noise = theano.shared(numpy.float32(0.))
    last_n = options['last_n']

    # video blocks. (n_timesteps, n_samples, n_annotations, ctxdim)
    x = tensor.tensor4('x', dtype='float32')
    mask = tensor.matrix('mask', dtype='float32')
    n_timesteps = x.shape[0]
    n_samples = x.shape[1]
    n_annotations = x.shape[2]  # n_annotation not used in the following
    ctxdim = x.shape[3]         # ctxdim not used in the following
  
    # action labels
    y = tensor.matrix('y', dtype='int64')

    #ctx = tensor.reshape(ctx, (n_timesteps, n_samples, n_annotations, ctxdim))
    ctx = x

    # initial state/cell
    ctx_mean = ctx.mean(0) ### ctx_mean is now (n_samples, n_annotations, ctxdim)
    ctx_mean = ctx_mean.mean(1) ### you want ctx_mean to be n_samples x ctxdim
    
    # get_layer('ff')[1](...) should be equivalent to the call fflayer(...)
    # why this (unreadable) design pattern?
    for lidx in xrange(1, options['n_layers_init']):
        ctx_mean = get_layer('ff')[1](tparams, ctx_mean, options,
                                      prefix='ff_init_%d'%lidx, activ='rectifier')
        if options['use_dropout']:
            ctx_mean = dropout_layer(ctx_mean, use_noise, trng)
    # initialization as from eq (6) of the paper
    init_state = get_layer('ff')[1](tparams, ctx_mean, options, prefix='ff_state', activ='tanh')
    init_memory = get_layer('ff')[1](tparams, ctx_mean, options, prefix='ff_memory', activ='tanh')

    # decoder (1 lstm layer only)
    # here again it should be equivalent to proj = lstm_cond_layer(tparams,....)
    proj = get_layer('lstm_cond')[1](tparams, ctx, options,
                                     prefix='decoder',
                                     mask=mask,
                                     init_state=init_state,
                                     init_memory=init_memory,
                                     trng=trng,
                                     use_noise=use_noise)
    # collection
    proj_h = proj[0]
    alphas = proj[2]
    ctxs = proj[3]
    if options['selector']:
        sels = proj[4]
    if options['use_dropout']:
        proj_h = dropout_layer(proj_h, use_noise, trng)

    # outputs
    logit = get_layer('ff')[1](tparams, proj_h, options, prefix='ff_logit_lstm', activ='linear')
    if options['ctx2out']:
        logit += get_layer('ff')[1](tparams, ctxs, options, prefix='ff_logit_ctx', activ='linear')
    logit = tanh(logit)
    if options['use_dropout']:
        logit = dropout_layer(logit, use_noise, trng)
    if options['n_layers_out'] > 1:
        for lidx in xrange(1, options['n_layers_out']):
            logit = get_layer('ff')[1](tparams, logit, options, prefix='ff_logit_h%d'%lidx, activ='rectifier')
            if options['use_dropout']:
                logit = dropout_layer(logit, use_noise, trng)

    logit = get_layer('ff')[1](tparams, logit, options, prefix='ff_logit', activ='linear')
    logit_shp = logit.shape #(TS, BS, #actions) (TimeSize, BatchSize?)

    logit = logit.reshape([logit_shp[0]*logit_shp[1], logit_shp[2]])  #(TSxBS,#actions)
    probs = tensor.nnet.softmax(logit)                                #(TSxBS,#actions)
    probs = probs.reshape([logit_shp[0], logit_shp[1], logit_shp[2]]) #(TS,BS,#actions)

    # Predictions
    preds = tensor.sum(probs[-last_n:, :, :],axis=0) #(BS,#actions)
    preds = tensor.argmax(preds,axis=1) # computed y; true y is in 'y' #(BS,1)

    # Cost function
    probs = probs.reshape([logit_shp[0]*logit_shp[1], logit_shp[2]])
    tmp = tensor.reshape(y, [x.shape[0]*x.shape[1],])
    cost = -tensor.log(probs[tensor.arange(n_timesteps*n_samples), tmp] + 1e-8)
    cost = cost.reshape([x.shape[0], x.shape[1]])
    cost = (cost * mask).sum(0).sum(0)

    opt_outs = dict()
    if options['selector']:
        opt_outs['selector'] = sels

    return trng, use_noise, [x, mask, y], alphas, cost, opt_outs, preds
 Builds the entire computational graph used for training
  
    Parameters
    ----------
    tparams : OrderedDict
        maps names of variables to theano shared variables
    options : dict
        big dictionary with all the settings and hyperparameters
    
    Returns
    -------
    trng: theano random number generator
        Used for dropout, stochastic attention, etc
    use_noise: theano shared variable
        flag that toggles noise on and off
    [x, mask, ctx]: theano variables
        Represent the captions, binary mask, and annotations
        for a single batch (see dimensions below)
    alphas: theano variables
        Attention weights
    cost: theano variable
        negative log likelihood
    opt_outs: OrderedDict
        extra outputs required depending on configuration in options
    preds:
	computed labels
    
Builds the entire computational graph used for training
Parameters

tparams : OrderedDict
maps names of variables to theano shared variables
options : dict
big dictionary with all the settings and hyperparameters

Returns


Builds the entire computational graph used for training Parameters tparams : OrderedDict maps names of variables to theano shared variables options : dict big dictionary with all the settings and hyperparameters Returns
def build_model(tparams, options):
    trng = RandomStreams(1234)
    use_noise = theano.shared(numpy.float32(0.))
    last_n = options['last_n']
    x = tensor.tensor4('x', dtype='float32')
    mask = tensor.matrix('mask', dtype='float32')
    n_timesteps = x.shape[0]
    n_samples = x.shape[1]
    n_annotations = x.shape[2]  
    ctxdim = x.shape[3]         
    y = tensor.matrix('y', dtype='int64')
    ctx = x
    ctx_mean = ctx.mean(0) 
    ctx_mean = ctx_mean.mean(1) 
    for lidx in xrange(1, options['n_layers_init']):
        ctx_mean = get_layer('ff')[1](tparams, ctx_mean, options,
                                      prefix='ff_init_%d'%lidx, activ='rectifier')
        if options['use_dropout']:
            ctx_mean = dropout_layer(ctx_mean, use_noise, trng)
    init_state = get_layer('ff')[1](tparams, ctx_mean, options, prefix='ff_state', activ='tanh')
    init_memory = get_layer('ff')[1](tparams, ctx_mean, options, prefix='ff_memory', activ='tanh')
    proj = get_layer('lstm_cond')[1](tparams, ctx, options,
                                     prefix='decoder',
                                     mask=mask,
                                     init_state=init_state,
                                     init_memory=init_memory,
                                     trng=trng,
                                     use_noise=use_noise)
    proj_h = proj[0]
    alphas = proj[2]
    ctxs = proj[3]
    if options['selector']:
        sels = proj[4]
    if options['use_dropout']:
        proj_h = dropout_layer(proj_h, use_noise, trng)
    logit = get_layer('ff')[1](tparams, proj_h, options, prefix='ff_logit_lstm', activ='linear')
    if options['ctx2out']:
        logit += get_layer('ff')[1](tparams, ctxs, options, prefix='ff_logit_ctx', activ='linear')
    logit = tanh(logit)
    if options['use_dropout']:
        logit = dropout_layer(logit, use_noise, trng)
    if options['n_layers_out'] > 1:
        for lidx in xrange(1, options['n_layers_out']):
            logit = get_layer('ff')[1](tparams, logit, options, prefix='ff_logit_h%d'%lidx, activ='rectifier')
            if options['use_dropout']:
                logit = dropout_layer(logit, use_noise, trng)
    logit = get_layer('ff')[1](tparams, logit, options, prefix='ff_logit', activ='linear')
    logit_shp = logit.shape 
    logit = logit.reshape([logit_shp[0]*logit_shp[1], logit_shp[2]])  
    probs = tensor.nnet.softmax(logit)                                
    probs = probs.reshape([logit_shp[0], logit_shp[1], logit_shp[2]]) 
    preds = tensor.sum(probs[-last_n:, :, :],axis=0) 
    preds = tensor.argmax(preds,axis=1) 
    probs = probs.reshape([logit_shp[0]*logit_shp[1], logit_shp[2]])
    tmp = tensor.reshape(y, [x.shape[0]*x.shape[1],])
    cost = -tensor.log(probs[tensor.arange(n_timesteps*n_samples), tmp] + 1e-8)
    cost = cost.reshape([x.shape[0], x.shape[1]])
    cost = (cost * mask).sum(0).sum(0)
    opt_outs = dict()
    if options['selector']:
        opt_outs['selector'] = sels
    return trng, use_noise, [x, mask, y], alphas, cost, opt_outs, preds
def build_model ( tparams , options ) : trng = RandomStreams ( 1234 ) use_noise = theano . shared ( numpy . float32 ( 0. ) ) last_n = options [ 'last_n' ] x = tensor . tensor4 ( 'x' , dtype = 'float32' ) mask = tensor . matrix ( 'mask' , dtype = 'float32' ) n_timesteps = x . shape [ 0 ] n_samples = x . shape [ 1 ] n_annotations = x . shape [ 2 ] ctxdim = x . shape [ 3 ] y = tensor . matrix ( 'y' , dtype = 'int64' ) ctx = x ctx_mean = ctx . mean ( 0 ) ctx_mean = ctx_mean . mean ( 1 ) for lidx in xrange ( 1 , options [ 'n_layers_init' ] ) : ctx_mean = get_layer ( 'ff' ) [ 1 ] ( tparams , ctx_mean , options , prefix = 'ff_init_%d' % lidx , activ = 'rectifier' ) if options [ 'use_dropout' ] : ctx_mean = dropout_layer ( ctx_mean , use_noise , trng ) init_state = get_layer ( 'ff' ) [ 1 ] ( tparams , ctx_mean , options , prefix = 'ff_state' , activ = 'tanh' ) init_memory = get_layer ( 'ff' ) [ 1 ] ( tparams , ctx_mean , options , prefix = 'ff_memory' , activ = 'tanh' ) proj = get_layer ( 'lstm_cond' ) [ 1 ] ( tparams , ctx , options , prefix = 'decoder' , mask = mask , init_state = init_state , init_memory = init_memory , trng = trng , use_noise = use_noise ) proj_h = proj [ 0 ] alphas = proj [ 2 ] ctxs = proj [ 3 ] if options [ 'selector' ] : sels = proj [ 4 ] if options [ 'use_dropout' ] : proj_h = dropout_layer ( proj_h , use_noise , trng ) logit = get_layer ( 'ff' ) [ 1 ] ( tparams , proj_h , options , prefix = 'ff_logit_lstm' , activ = 'linear' ) if options [ 'ctx2out' ] : logit += get_layer ( 'ff' ) [ 1 ] ( tparams , ctxs , options , prefix = 'ff_logit_ctx' , activ = 'linear' ) logit = tanh ( logit ) if options [ 'use_dropout' ] : logit = dropout_layer ( logit , use_noise , trng ) if options [ 'n_layers_out' ] > 1 : for lidx in xrange ( 1 , options [ 'n_layers_out' ] ) : logit = get_layer ( 'ff' ) [ 1 ] ( tparams , logit , options , prefix = 'ff_logit_h%d' % lidx , activ = 'rectifier' ) if options [ 'use_dropout' ] : logit = dropout_layer ( logit , use_noise , trng ) logit = get_layer ( 'ff' ) [ 1 ] ( tparams , logit , options , prefix = 'ff_logit' , activ = 'linear' ) logit_shp = logit . shape logit = logit . reshape ( [ logit_shp [ 0 ] * logit_shp [ 1 ] , logit_shp [ 2 ] ] ) probs = tensor . nnet . softmax ( logit ) probs = probs . reshape ( [ logit_shp [ 0 ] , logit_shp [ 1 ] , logit_shp [ 2 ] ] ) preds = tensor . sum ( probs [ - last_n : , : , : ] , axis = 0 ) preds = tensor . argmax ( preds , axis = 1 ) probs = probs . reshape ( [ logit_shp [ 0 ] * logit_shp [ 1 ] , logit_shp [ 2 ] ] ) tmp = tensor . reshape ( y , [ x . shape [ 0 ] * x . shape [ 1 ] , ] ) cost = - tensor . log ( probs [ tensor . arange ( n_timesteps * n_samples ) , tmp ] + 1e-8 ) cost = cost . reshape ( [ x . shape [ 0 ] , x . shape [ 1 ] ] ) cost = ( cost * mask ) . sum ( 0 ) . sum ( 0 ) opt_outs = dict ( ) if options [ 'selector' ] : opt_outs [ 'selector' ] = sels return trng , use_noise , [ x , mask , y ] , alphas , cost , opt_outs , preds
Builds the entire computational graph used for training
Parameters
Builds the entire computational graph used for training Parameters
""" Builds the entire computational graph used for training
  
    Parameters
    ----------
    tparams : OrderedDict
        maps names of variables to theano shared variables
    options : dict
        big dictionary with all the settings and hyperparameters
    
    Returns
    -------
    trng: theano random number generator
        Used for dropout, stochastic attention, etc
    use_noise: theano shared variable
        flag that toggles noise on and off
    [x, mask, ctx]: theano variables
        Represent the captions, binary mask, and annotations
        for a single batch (see dimensions below)
    alphas: theano variables
        Attention weights
    cost: theano variable
        negative log likelihood
    opt_outs: OrderedDict
        extra outputs required depending on configuration in options
    preds:
	computed labels
    """ # video blocks. (n_timesteps, n_samples, n_annotations, ctxdim) # n_annotation not used in the following # ctxdim not used in the following # action labels #ctx = tensor.reshape(ctx, (n_timesteps, n_samples, n_annotations, ctxdim)) # initial state/cell ### ctx_mean is now (n_samples, n_annotations, ctxdim) ### you want ctx_mean to be n_samples x ctxdim # get_layer('ff')[1](...) should be equivalent to the call fflayer(...) # why this (unreadable) design pattern? # initialization as from eq (6) of the paper # decoder (1 lstm layer only) # here again it should be equivalent to proj = lstm_cond_layer(tparams,....) # collection # outputs #(TS, BS, #actions) (TimeSize, BatchSize?) #(TSxBS,#actions) #(TSxBS,#actions) #(TS,BS,#actions) # Predictions #(BS,#actions) # computed y; true y is in 'y' #(BS,1) # Cost function
tparams
options