rfho/examples/mnist_deep_ho.py
def experiment(mnist, optimizer=rf.AdamOptimizer, optimizer_kwargs=None,
               hyper_batch_size=100, T=200, hyper_learning_rate=1.e-4, use_mse=False):
    """
    Modified MNIST for expert (CNN part) tensorflow tutorial experiment to include real time
    hyperparameter optimization. Hyperparameters being optimized are learning rate for
    ADAM optimizer and coefficient of L2 norm of fully connected part of the network.
    Note that this codes requires ~ 3x (gpu) memory and ~ 4x time compared to the original one
    but should yield a final test error of around 99.4 %

    :return:
    """
    # Create the model
    x = tf.placeholder(tf.float32, [None, 784], name='x')

    # Define loss and optimizer
    y_ = tf.placeholder(tf.float32, [None, 10], name='y')

    # Build the graph for the deep net
    y_conv, W_fc1, W_fc2 = deepnn(x)

    # RFHO: collect model variables and "vectorize the model"
    model_vairables = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES)
    # use adam optimizer:
    w, y_conv, W_fc1, W_fc2 = rf.vectorize_model(model_vairables, y_conv, W_fc1, W_fc2,
                                                 augment=optimizer.get_augmentation_multiplier())
    # w is now a vector that contains all the weights, y_conv and W_fc2 are the same tensor as earlier,
    # but in the new graph

    # RFHO use cross entropy defined in the package since tensorflow one does not have Hessian,
    # eps is the clipping threshold for cross entropy.
    if use_mse:
        error = tf.reduce_mean(tf.squared_difference(y_, y_conv), name='error')
    else:
        error = tf.reduce_mean(
            rf.cross_entropy_loss(labels=y_, logits=y_conv, eps=1.e-4), name='error')
    # RFHO add an L2 regularizer on the last weight matrix, whose weight will be optimized
    rho = tf.Variable(0., name='rho')
    constraints = [rf.positivity(rho)]  # rho >= 0
    iterations_per_epoch = 1100  # with mini batch size of 50
    training_error = error + 1/iterations_per_epoch*tf.multiply(
        rho, tf.nn.l2_loss(W_fc1) + tf.nn.l2_loss(W_fc2))

    # train_step = tf.train.AdamOptimizer(1e-4).minimize(cross_entropy)
    # RFHO define learning rate as an hyperparameter and create the parameter optimization dynamics
    if optimizer_kwargs is None:
        optimizer_kwargs = {'lr': tf.Variable(1.e-4, name='alpha')}
    dynamics = optimizer.create(w, loss=training_error, **optimizer_kwargs)
    constraints += dynamics.get_natural_hyperparameter_constraints()  # add 'usual' constraints for
    # if optimizer is rf.AdamOptimizer:
    #     constraints.append(dynamics.learning_rate.assign(tf.minimum(1.e-3, dynamics.learning_rate)))
    # algorithmic hyperparameters

    # RFHO we want to optimize learning rate and L2 coefficient w.r.t. cross entropy loss on validation set
    hyper_dict = {error: [rho] + dynamics.get_optimization_hyperparameters(only_variables=True)}
    # RFHO define the hyperparameter optimizer, we use Forward-HG method to compute hyper-gradients and RTHO algorithm
    hyper_opt = rf.HyperOptimizer(dynamics, hyper_dict, rf.ForwardHG, lr=hyper_learning_rate)

    correct_prediction = tf.equal(tf.argmax(y_conv, 1), tf.argmax(y_, 1))
    accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32), name='accuracy')

    # RFHO last thing before running: define the example supplier:
    _train_fd = rf.ExampleVisiting(mnist.train, batch_size=50).create_feed_dict_supplier(x, y_)
    _validation_fd = mnist.validation.create_supplier(x, y_)

    with tf.Session(config=rf.CONFIG_GPU_GROWTH).as_default():  # RFHO use default session.
        hyper_opt.initialize()  # RFHO this will initialize all the variables, including hyperparameters
        for i in range(T):  # RFHO we run for 200 hyper-iterations
            hyper_opt.run(hyper_batch_size, train_feed_dict_supplier=_train_fd,
                          val_feed_dict_suppliers={error: _validation_fd},
                          hyper_constraints_ops=constraints)

        test_accuracy = accuracy.eval(feed_dict=mnist.test.create_supplier(x, y_)())
        print('test accuracy %g' % test_accuracy)
        return test_accuracy

    Modified MNIST for expert (CNN part) tensorflow tutorial experiment to include real time
    hyperparameter optimization. Hyperparameters being optimized are learning rate for
    ADAM optimizer and coefficient of L2 norm of fully connected part of the network.
    Note that this codes requires ~ 3x (gpu) memory and ~ 4x time compared to the original one
    but should yield a final test error of around 99.4 %

    :return:
    
Modified MNIST for expert (CNN part) tensorflow tutorial experiment to include real time
hyperparameter optimization. Hyperparameters being optimized are learning rate for
ADAM optimizer and coefficient of L2 norm of fully connected part of the network.
Note that this codes requires ~ 3x (gpu) memory and ~ 4x time compared to the original one
but should yield a final test error of around 99.4 %
Modified MNIST for expert ( CNN part ) tensorflow tutorial experiment to include real time hyperparameter optimization . Hyperparameters being optimized are learning rate for ADAM optimizer and coefficient of L2 norm of fully connected part of the network . Note that this codes requires ~ 3x ( gpu ) memory and ~ 4x time compared to the original one but should yield a final test error of around 99 . 4 %
def experiment(mnist, optimizer=rf.AdamOptimizer, optimizer_kwargs=None,
               hyper_batch_size=100, T=200, hyper_learning_rate=1.e-4, use_mse=False):
    x = tf.placeholder(tf.float32, [None, 784], name='x')
    y_ = tf.placeholder(tf.float32, [None, 10], name='y')
    y_conv, W_fc1, W_fc2 = deepnn(x)
    model_vairables = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES)
    w, y_conv, W_fc1, W_fc2 = rf.vectorize_model(model_vairables, y_conv, W_fc1, W_fc2,
                                                 augment=optimizer.get_augmentation_multiplier())
    if use_mse:
        error = tf.reduce_mean(tf.squared_difference(y_, y_conv), name='error')
    else:
        error = tf.reduce_mean(
            rf.cross_entropy_loss(labels=y_, logits=y_conv, eps=1.e-4), name='error')
    rho = tf.Variable(0., name='rho')
    constraints = [rf.positivity(rho)]  
    iterations_per_epoch = 1100  
    training_error = error + 1/iterations_per_epoch*tf.multiply(
        rho, tf.nn.l2_loss(W_fc1) + tf.nn.l2_loss(W_fc2))
    if optimizer_kwargs is None:
        optimizer_kwargs = {'lr': tf.Variable(1.e-4, name='alpha')}
    dynamics = optimizer.create(w, loss=training_error, **optimizer_kwargs)
    constraints += dynamics.get_natural_hyperparameter_constraints()  
    hyper_dict = {error: [rho] + dynamics.get_optimization_hyperparameters(only_variables=True)}
    hyper_opt = rf.HyperOptimizer(dynamics, hyper_dict, rf.ForwardHG, lr=hyper_learning_rate)
    correct_prediction = tf.equal(tf.argmax(y_conv, 1), tf.argmax(y_, 1))
    accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32), name='accuracy')
    _train_fd = rf.ExampleVisiting(mnist.train, batch_size=50).create_feed_dict_supplier(x, y_)
    _validation_fd = mnist.validation.create_supplier(x, y_)
    with tf.Session(config=rf.CONFIG_GPU_GROWTH).as_default():  
        hyper_opt.initialize()  
        for i in range(T):  
            hyper_opt.run(hyper_batch_size, train_feed_dict_supplier=_train_fd,
                          val_feed_dict_suppliers={error: _validation_fd},
                          hyper_constraints_ops=constraints)
        test_accuracy = accuracy.eval(feed_dict=mnist.test.create_supplier(x, y_)())
        print('test accuracy %g' % test_accuracy)
        return test_accuracy
def experiment ( mnist , optimizer = rf . AdamOptimizer , optimizer_kwargs = None , hyper_batch_size = 100 , T = 200 , hyper_learning_rate = 1.e-4 , use_mse = False ) : x = tf . placeholder ( tf . float32 , [ None , 784 ] , name = 'x' ) y_ = tf . placeholder ( tf . float32 , [ None , 10 ] , name = 'y' ) y_conv , W_fc1 , W_fc2 = deepnn ( x ) model_vairables = tf . get_collection ( tf . GraphKeys . GLOBAL_VARIABLES ) w , y_conv , W_fc1 , W_fc2 = rf . vectorize_model ( model_vairables , y_conv , W_fc1 , W_fc2 , augment = optimizer . get_augmentation_multiplier ( ) ) if use_mse : error = tf . reduce_mean ( tf . squared_difference ( y_ , y_conv ) , name = 'error' ) else : error = tf . reduce_mean ( rf . cross_entropy_loss ( labels = y_ , logits = y_conv , eps = 1.e-4 ) , name = 'error' ) rho = tf . Variable ( 0. , name = 'rho' ) constraints = [ rf . positivity ( rho ) ] iterations_per_epoch = 1100 training_error = error + 1 / iterations_per_epoch * tf . multiply ( rho , tf . nn . l2_loss ( W_fc1 ) + tf . nn . l2_loss ( W_fc2 ) ) if optimizer_kwargs is None : optimizer_kwargs = { 'lr' : tf . Variable ( 1.e-4 , name = 'alpha' ) } dynamics = optimizer . create ( w , loss = training_error , ** optimizer_kwargs ) constraints += dynamics . get_natural_hyperparameter_constraints ( ) hyper_dict = { error : [ rho ] + dynamics . get_optimization_hyperparameters ( only_variables = True ) } hyper_opt = rf . HyperOptimizer ( dynamics , hyper_dict , rf . ForwardHG , lr = hyper_learning_rate ) correct_prediction = tf . equal ( tf . argmax ( y_conv , 1 ) , tf . argmax ( y_ , 1 ) ) accuracy = tf . reduce_mean ( tf . cast ( correct_prediction , tf . float32 ) , name = 'accuracy' ) _train_fd = rf . ExampleVisiting ( mnist . train , batch_size = 50 ) . create_feed_dict_supplier ( x , y_ ) _validation_fd = mnist . validation . create_supplier ( x , y_ ) with tf . Session ( config = rf . CONFIG_GPU_GROWTH ) . as_default ( ) : hyper_opt . initialize ( ) for i in range ( T ) : hyper_opt . run ( hyper_batch_size , train_feed_dict_supplier = _train_fd , val_feed_dict_suppliers = { error : _validation_fd } , hyper_constraints_ops = constraints ) test_accuracy = accuracy . eval ( feed_dict = mnist . test . create_supplier ( x , y_ ) ( ) ) print ( 'test accuracy %g' % test_accuracy ) return test_accuracy
Modified MNIST for expert (CNN part) tensorflow tutorial experiment to include real time
hyperparameter optimization.
Modified MNIST for expert ( CNN part ) tensorflow tutorial experiment to include real time hyperparameter optimization .
"""
    Modified MNIST for expert (CNN part) tensorflow tutorial experiment to include real time
    hyperparameter optimization. Hyperparameters being optimized are learning rate for
    ADAM optimizer and coefficient of L2 norm of fully connected part of the network.
    Note that this codes requires ~ 3x (gpu) memory and ~ 4x time compared to the original one
    but should yield a final test error of around 99.4 %

    :return:
    """ # Create the model # Define loss and optimizer # Build the graph for the deep net # RFHO: collect model variables and "vectorize the model" # use adam optimizer: # w is now a vector that contains all the weights, y_conv and W_fc2 are the same tensor as earlier, # but in the new graph # RFHO use cross entropy defined in the package since tensorflow one does not have Hessian, # eps is the clipping threshold for cross entropy. # RFHO add an L2 regularizer on the last weight matrix, whose weight will be optimized # rho >= 0 # with mini batch size of 50 # train_step = tf.train.AdamOptimizer(1e-4).minimize(cross_entropy) # RFHO define learning rate as an hyperparameter and create the parameter optimization dynamics # add 'usual' constraints for # if optimizer is rf.AdamOptimizer: #     constraints.append(dynamics.learning_rate.assign(tf.minimum(1.e-3, dynamics.learning_rate))) # algorithmic hyperparameters # RFHO we want to optimize learning rate and L2 coefficient w.r.t. cross entropy loss on validation set # RFHO define the hyperparameter optimizer, we use Forward-HG method to compute hyper-gradients and RTHO algorithm # RFHO last thing before running: define the example supplier: # RFHO use default session. # RFHO this will initialize all the variables, including hyperparameters # RFHO we run for 200 hyper-iterations
None
mnist
optimizer
optimizer_kwargs
hyper_batch_size
T
hyper_learning_rate
use_mse