rfho/examples/mnist_deep_ho.py
def main(_):
    """
    Modified MNIST for expert (CNN part) tensorflow tutorial experiment to include real time
    hyperparameter optimization. Hyperparameters being optimized are learning rate for
    ADAM optimizer and coefficient of L2 norm of fully connected part of the network.
    Note that this codes requires ~ 3x (gpu) memory and ~ 4x time compared to the original one
    but should yield a final test error of around 99.4 %

    :param _:
    :return:
    """
    # Import data
    mnist = input_data.read_data_sets(FLAGS.data_dir, one_hot=True)

    # Create the model
    x = tf.placeholder(tf.float32, [None, 784])

    # Define loss and optimizer
    y_ = tf.placeholder(tf.float32, [None, 10])

    # Build the graph for the deep net
    y_conv, W_fc1, W_fc2 = deepnn(x)

    # RFHO: collect model variables and "vectorize the model"
    model_vairables = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES)
    # use adam optimizer:
    optimizer = rf.AdamOptimizer
    w, y_conv, W_fc1, W_fc2 = rf.vectorize_model(model_vairables, y_conv, W_fc1, W_fc2,
                                                 augment=optimizer.get_augmentation_multiplier(),
                                                 suppress_err_out=False)
    # w is now a vector that contains all the weights, y_conv and W_fc2 are the same tensor as earlier,
    # but in the new graph

    # RFHO use cross entropy defined in the package since tensorflow one does not have Hessian,
    # eps is the clipping threshold for cross entropy.
    cross_entropy = tf.reduce_mean(
        rf.cross_entropy_loss(labels=y_, logits=y_conv, eps=1.e-4))
    # RFHO add an L2 regularizer on the last weight matrix, whose weight will be optimized
    rho = tf.Variable(0., name='rho')
    constraints = [rf.positivity(rho)]  # rho >= 0
    iterations_per_epoch = 1100  # with mini batch size of 50
    training_error = cross_entropy + 1/iterations_per_epoch*tf.multiply(
        rho, tf.nn.l2_loss(W_fc1) + tf.nn.l2_loss(W_fc2))

    # train_step = tf.train.AdamOptimizer(1e-4).minimize(cross_entropy)
    # RFHO define learning rate as an hyperparameter and create the parameter optimization dynamics
    alpha = tf.Variable(1.e-4, name='alpha')
    constraints.append(rf.positivity(alpha))
    dynamics = optimizer.create(w, lr=alpha, loss=training_error)

    # RFHO we want to optimize learning rate and L2 coefficient w.r.t. cross entropy loss on validation set
    hyper_dict = {cross_entropy: [alpha, rho]}
    # RFHO define the hyperparameter optimizer, we use Forward-HG method to compute hyper-gradients and RTHO algorithm
    hyper_opt = rf.HyperOptimizer(dynamics, hyper_dict, rf.ForwardHG, lr=1.e-5)

    correct_prediction = tf.equal(tf.argmax(y_conv, 1), tf.argmax(y_, 1))
    accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32), name='accuracy')

    # RFHO last thing before running: define the example supplier:
    def _train_fd():
        batch = mnist.train.next_batch(50)  # batch size of 50
        return {x: batch[0], y_: batch[1]}

    def _validation_fd():
        return {x: mnist.validation.images, y_: mnist.validation.labels}

    with tf.Session(config=rf.CONFIG_GPU_GROWTH).as_default() as ss:  # RFHO use default session.
        hyper_opt.initialize()  # RFHO this will initialize all the variables, including hyperparameters
        for i in range(200):  # RFHO we run for 200 hyper-iterations
            hyper_opt.run(100, train_feed_dict_supplier=_train_fd,
                          val_feed_dict_suppliers={cross_entropy: _validation_fd},
                          hyper_constraints_ops=constraints)

            # if i % 100 == 0:
            train_accuracy = accuracy.eval(feed_dict=_train_fd())
            val_accuracy, val_error = ss.run([accuracy, cross_entropy], feed_dict=_validation_fd())

            print('step %d, training accuracy %.2f; validation accuracy: %.4f, validation error: %.5f; '
                  'alpha: %.6f, %.5f, rho: %.6f, %.5f'
                  % (i*100, train_accuracy, val_accuracy, val_error, alpha.eval(),
                     hyper_opt.hyper_gradients.hyper_gradients_dict[alpha].eval(),
                     rho.eval(), hyper_opt.hyper_gradients.hyper_gradients_dict[rho].eval()))
            # train_step.run(feed_dict={x: batch[0], y_: batch[1], keep_prob: 0.5})

        test_accuracy = accuracy.eval(feed_dict={
            x: mnist.test.images, y_: mnist.test.labels})
        print('test accuracy %g' % test_accuracy)
        return test_accuracy

    Modified MNIST for expert (CNN part) tensorflow tutorial experiment to include real time
    hyperparameter optimization. Hyperparameters being optimized are learning rate for
    ADAM optimizer and coefficient of L2 norm of fully connected part of the network.
    Note that this codes requires ~ 3x (gpu) memory and ~ 4x time compared to the original one
    but should yield a final test error of around 99.4 %

    :param _:
    :return:
    
Modified MNIST for expert (CNN part) tensorflow tutorial experiment to include real time
hyperparameter optimization. Hyperparameters being optimized are learning rate for
ADAM optimizer and coefficient of L2 norm of fully connected part of the network.
Note that this codes requires ~ 3x (gpu) memory and ~ 4x time compared to the original one
but should yield a final test error of around 99.4 %
Modified MNIST for expert ( CNN part ) tensorflow tutorial experiment to include real time hyperparameter optimization . Hyperparameters being optimized are learning rate for ADAM optimizer and coefficient of L2 norm of fully connected part of the network . Note that this codes requires ~ 3x ( gpu ) memory and ~ 4x time compared to the original one but should yield a final test error of around 99 . 4 %
def main(_):
    mnist = input_data.read_data_sets(FLAGS.data_dir, one_hot=True)
    x = tf.placeholder(tf.float32, [None, 784])
    y_ = tf.placeholder(tf.float32, [None, 10])
    y_conv, W_fc1, W_fc2 = deepnn(x)
    model_vairables = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES)
    optimizer = rf.AdamOptimizer
    w, y_conv, W_fc1, W_fc2 = rf.vectorize_model(model_vairables, y_conv, W_fc1, W_fc2,
                                                 augment=optimizer.get_augmentation_multiplier(),
                                                 suppress_err_out=False)
    cross_entropy = tf.reduce_mean(
        rf.cross_entropy_loss(labels=y_, logits=y_conv, eps=1.e-4))
    rho = tf.Variable(0., name='rho')
    constraints = [rf.positivity(rho)]  
    iterations_per_epoch = 1100  
    training_error = cross_entropy + 1/iterations_per_epoch*tf.multiply(
        rho, tf.nn.l2_loss(W_fc1) + tf.nn.l2_loss(W_fc2))
    alpha = tf.Variable(1.e-4, name='alpha')
    constraints.append(rf.positivity(alpha))
    dynamics = optimizer.create(w, lr=alpha, loss=training_error)
    hyper_dict = {cross_entropy: [alpha, rho]}
    hyper_opt = rf.HyperOptimizer(dynamics, hyper_dict, rf.ForwardHG, lr=1.e-5)
    correct_prediction = tf.equal(tf.argmax(y_conv, 1), tf.argmax(y_, 1))
    accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32), name='accuracy')
    def _train_fd():
        batch = mnist.train.next_batch(50)  
        return {x: batch[0], y_: batch[1]}
    def _validation_fd():
        return {x: mnist.validation.images, y_: mnist.validation.labels}
    with tf.Session(config=rf.CONFIG_GPU_GROWTH).as_default() as ss:  
        hyper_opt.initialize()  
        for i in range(200):  
            hyper_opt.run(100, train_feed_dict_supplier=_train_fd,
                          val_feed_dict_suppliers={cross_entropy: _validation_fd},
                          hyper_constraints_ops=constraints)
            train_accuracy = accuracy.eval(feed_dict=_train_fd())
            val_accuracy, val_error = ss.run([accuracy, cross_entropy], feed_dict=_validation_fd())
            print('step %d, training accuracy %.2f; validation accuracy: %.4f, validation error: %.5f; '
                  'alpha: %.6f, %.5f, rho: %.6f, %.5f'
                  % (i*100, train_accuracy, val_accuracy, val_error, alpha.eval(),
                     hyper_opt.hyper_gradients.hyper_gradients_dict[alpha].eval(),
                     rho.eval(), hyper_opt.hyper_gradients.hyper_gradients_dict[rho].eval()))
        test_accuracy = accuracy.eval(feed_dict={
            x: mnist.test.images, y_: mnist.test.labels})
        print('test accuracy %g' % test_accuracy)
        return test_accuracy
def main ( _ ) : mnist = input_data . read_data_sets ( FLAGS . data_dir , one_hot = True ) x = tf . placeholder ( tf . float32 , [ None , 784 ] ) y_ = tf . placeholder ( tf . float32 , [ None , 10 ] ) y_conv , W_fc1 , W_fc2 = deepnn ( x ) model_vairables = tf . get_collection ( tf . GraphKeys . GLOBAL_VARIABLES ) optimizer = rf . AdamOptimizer w , y_conv , W_fc1 , W_fc2 = rf . vectorize_model ( model_vairables , y_conv , W_fc1 , W_fc2 , augment = optimizer . get_augmentation_multiplier ( ) , suppress_err_out = False ) cross_entropy = tf . reduce_mean ( rf . cross_entropy_loss ( labels = y_ , logits = y_conv , eps = 1.e-4 ) ) rho = tf . Variable ( 0. , name = 'rho' ) constraints = [ rf . positivity ( rho ) ] iterations_per_epoch = 1100 training_error = cross_entropy + 1 / iterations_per_epoch * tf . multiply ( rho , tf . nn . l2_loss ( W_fc1 ) + tf . nn . l2_loss ( W_fc2 ) ) alpha = tf . Variable ( 1.e-4 , name = 'alpha' ) constraints . append ( rf . positivity ( alpha ) ) dynamics = optimizer . create ( w , lr = alpha , loss = training_error ) hyper_dict = { cross_entropy : [ alpha , rho ] } hyper_opt = rf . HyperOptimizer ( dynamics , hyper_dict , rf . ForwardHG , lr = 1.e-5 ) correct_prediction = tf . equal ( tf . argmax ( y_conv , 1 ) , tf . argmax ( y_ , 1 ) ) accuracy = tf . reduce_mean ( tf . cast ( correct_prediction , tf . float32 ) , name = 'accuracy' ) def _train_fd ( ) : batch = mnist . train . next_batch ( 50 ) return { x : batch [ 0 ] , y_ : batch [ 1 ] } def _validation_fd ( ) : return { x : mnist . validation . images , y_ : mnist . validation . labels } with tf . Session ( config = rf . CONFIG_GPU_GROWTH ) . as_default ( ) as ss : hyper_opt . initialize ( ) for i in range ( 200 ) : hyper_opt . run ( 100 , train_feed_dict_supplier = _train_fd , val_feed_dict_suppliers = { cross_entropy : _validation_fd } , hyper_constraints_ops = constraints ) train_accuracy = accuracy . eval ( feed_dict = _train_fd ( ) ) val_accuracy , val_error = ss . run ( [ accuracy , cross_entropy ] , feed_dict = _validation_fd ( ) ) print ( 'step %d, training accuracy %.2f; validation accuracy: %.4f, validation error: %.5f; ' 'alpha: %.6f, %.5f, rho: %.6f, %.5f' % ( i * 100 , train_accuracy , val_accuracy , val_error , alpha . eval ( ) , hyper_opt . hyper_gradients . hyper_gradients_dict [ alpha ] . eval ( ) , rho . eval ( ) , hyper_opt . hyper_gradients . hyper_gradients_dict [ rho ] . eval ( ) ) ) test_accuracy = accuracy . eval ( feed_dict = { x : mnist . test . images , y_ : mnist . test . labels } ) print ( 'test accuracy %g' % test_accuracy ) return test_accuracy
Modified MNIST for expert (CNN part) tensorflow tutorial experiment to include real time
hyperparameter optimization.
Modified MNIST for expert ( CNN part ) tensorflow tutorial experiment to include real time hyperparameter optimization .
"""
    Modified MNIST for expert (CNN part) tensorflow tutorial experiment to include real time
    hyperparameter optimization. Hyperparameters being optimized are learning rate for
    ADAM optimizer and coefficient of L2 norm of fully connected part of the network.
    Note that this codes requires ~ 3x (gpu) memory and ~ 4x time compared to the original one
    but should yield a final test error of around 99.4 %

    :param _:
    :return:
    """ # Import data # Create the model # Define loss and optimizer # Build the graph for the deep net # RFHO: collect model variables and "vectorize the model" # use adam optimizer: # w is now a vector that contains all the weights, y_conv and W_fc2 are the same tensor as earlier, # but in the new graph # RFHO use cross entropy defined in the package since tensorflow one does not have Hessian, # eps is the clipping threshold for cross entropy. # RFHO add an L2 regularizer on the last weight matrix, whose weight will be optimized # rho >= 0 # with mini batch size of 50 # train_step = tf.train.AdamOptimizer(1e-4).minimize(cross_entropy) # RFHO define learning rate as an hyperparameter and create the parameter optimization dynamics # RFHO we want to optimize learning rate and L2 coefficient w.r.t. cross entropy loss on validation set # RFHO define the hyperparameter optimizer, we use Forward-HG method to compute hyper-gradients and RTHO algorithm # RFHO last thing before running: define the example supplier: # batch size of 50 # RFHO use default session. # RFHO this will initialize all the variables, including hyperparameters # RFHO we run for 200 hyper-iterations # if i % 100 == 0: # train_step.run(feed_dict={x: batch[0], y_: batch[1], keep_prob: 0.5})
None
_
None