rfho/examples/all_methods_on_mnist.py
def experiment_no_saver(datasets=None, model='log_reg', model_kwargs=None, l1=0., l2=0.,
                        synthetic_hypers=None, set_T=None,
                        optimizer=rf.MomentumOptimizer, optimizer_kwargs=None, batch_size=200,
                        algo_hyper_wrt_tr_error=False,
                        mode='reverse', hyper_optimizer=rf.AdamOptimizer, hyper_optimizer_kwargs=None,
                        hyper_iterations=100, hyper_batch_size=100, epochs=None, do_print=True):
    """
    General method for conducting various simple experiments (on MNIST dataset) with RFHO package.

    :param datasets: (some dataset, usually MNIST....)
    :param model: (default logarithmic regression) model type
    :param model_kwargs:
    :param l1: Initial value for l1 regularizer weight (if None does not uses it)
    :param l2: Initial value for l2 regularizer weight (if None does not uses it)
    :param synthetic_hypers: (default None, for benchmarking purposes) if integer adds some synthetic hyperparameters
                                to the task.
    :param set_T:
    :param optimizer: (default `MomentumOptimizer`) optimizer for parameters
    :param optimizer_kwargs:
    :param batch_size:
    :param epochs: number of ephocs
    :param algo_hyper_wrt_tr_error: (default False) if True optimizes the algorithmic hyperparameters (learning rate, ..
                                    w.r.t. training error instead of validation error
    :param mode: forward reverse or rtho
    :param hyper_optimizer: optimizer for the hyperparameters
    :param hyper_optimizer_kwargs:
    :param hyper_iterations: number of hyper-iterations
    :param hyper_batch_size: hyper-batch size when RTHO is used
    :param do_print: if True (default) prints intermediate results...
    :return:
    """
    assert mode in HO_MODES

    if synthetic_hypers:
        batch_size = synthetic_hypers  # altrimenti divento matto!

    if datasets is None: datasets = load_dataset()

    x, model = create_model(datasets, model, **model_kwargs or {})
    s, out, ws, y, error, training_error, rho_l1s, reg_l1s, rho_l2s, reg_l2s, accuracy, base_tr_error, gamma = \
        define_errors_default_models(model, l1, l2, synthetic_hypers=synthetic_hypers,
                                     augment=optimizer.get_augmentation_multiplier())
    tf.add_to_collection('errors', error)

    if optimizer_kwargs is None: optimizer_kwargs = {'lr': tf.Variable(.01, name='eta'),
                                                     'mu': tf.Variable(.5, name='mu')}

    tr_dynamics = optimizer.create(s, loss=training_error, w_is_state=True, **optimizer_kwargs)

    regularization_hyperparameters = []
    if rho_l1s is not None:
        regularization_hyperparameters += rho_l1s
    if rho_l2s is not None:
        regularization_hyperparameters += rho_l2s

    synthetic_hyperparameters = []
    if synthetic_hypers:
        synthetic_hyperparameters.append(gamma)

    hyper_dict = {error: regularization_hyperparameters + synthetic_hyperparameters}  # create hyper_dict
    # end of hyperparameters

    if algo_hyper_wrt_tr_error:  # it is possible to optimize different hyperparameters wrt different validation errors
        hyper_dict[training_error] = tr_dynamics.get_optimization_hyperparameters(only_variables=True)
    else:
        hyper_dict[error] += tr_dynamics.get_optimization_hyperparameters(only_variables=True)

    hyper_opt = rf.HyperOptimizer(tr_dynamics, hyper_dict,
                                  method=rf.ReverseHG if mode == 'reverse' else rf.ForwardHG,
                                  hyper_optimizer_class=hyper_optimizer, **hyper_optimizer_kwargs or {})

    positivity = rf.positivity(hyper_opt.hyper_list)

    # stochastic descent
    ev_data = ExampleVisiting(datasets.train, batch_size=batch_size, epochs=epochs)
    if epochs: ev_data.generate_visiting_scheme()
    tr_supplier = ev_data.create_feed_dict_supplier(x, y)
    val_supplier = datasets.validation.create_supplier(x, y)

    def _all_training_supplier():
        return {x: datasets.train.data, y: datasets.train.target}

    # feed_dict supplier for validation errors
    val_feed_dict_suppliers = {error: val_supplier}
    if algo_hyper_wrt_tr_error: val_feed_dict_suppliers[training_error] = _all_training_supplier

    # number of iterations
    T = set_T or ev_data.T if mode != 'rtho' else hyper_batch_size

    with tf.Session(config=rf.CONFIG_GPU_GROWTH).as_default():
        hyper_opt.initialize()
        for k in range(hyper_iterations):
            hyper_opt.run(T, train_feed_dict_supplier=tr_supplier, val_feed_dict_suppliers=val_feed_dict_suppliers,
                          hyper_constraints_ops=positivity)

            if mode != 'rtho':
                hyper_opt.initialize()

    General method for conducting various simple experiments (on MNIST dataset) with RFHO package.

    :param datasets: (some dataset, usually MNIST....)
    :param model: (default logarithmic regression) model type
    :param model_kwargs:
    :param l1: Initial value for l1 regularizer weight (if None does not uses it)
    :param l2: Initial value for l2 regularizer weight (if None does not uses it)
    :param synthetic_hypers: (default None, for benchmarking purposes) if integer adds some synthetic hyperparameters
                                to the task.
    :param set_T:
    :param optimizer: (default `MomentumOptimizer`) optimizer for parameters
    :param optimizer_kwargs:
    :param batch_size:
    :param epochs: number of ephocs
    :param algo_hyper_wrt_tr_error: (default False) if True optimizes the algorithmic hyperparameters (learning rate, ..
                                    w.r.t. training error instead of validation error
    :param mode: forward reverse or rtho
    :param hyper_optimizer: optimizer for the hyperparameters
    :param hyper_optimizer_kwargs:
    :param hyper_iterations: number of hyper-iterations
    :param hyper_batch_size: hyper-batch size when RTHO is used
    :param do_print: if True (default) prints intermediate results...
    :return:
    
General method for conducting various simple experiments (on MNIST dataset) with RFHO package.
General method for conducting various simple experiments ( on MNIST dataset ) with RFHO package .
def experiment_no_saver(datasets=None, model='log_reg', model_kwargs=None, l1=0., l2=0.,
                        synthetic_hypers=None, set_T=None,
                        optimizer=rf.MomentumOptimizer, optimizer_kwargs=None, batch_size=200,
                        algo_hyper_wrt_tr_error=False,
                        mode='reverse', hyper_optimizer=rf.AdamOptimizer, hyper_optimizer_kwargs=None,
                        hyper_iterations=100, hyper_batch_size=100, epochs=None, do_print=True):
    assert mode in HO_MODES
    if synthetic_hypers:
        batch_size = synthetic_hypers  
    if datasets is None: datasets = load_dataset()
    x, model = create_model(datasets, model, **model_kwargs or {})
    s, out, ws, y, error, training_error, rho_l1s, reg_l1s, rho_l2s, reg_l2s, accuracy, base_tr_error, gamma = \
        define_errors_default_models(model, l1, l2, synthetic_hypers=synthetic_hypers,
                                     augment=optimizer.get_augmentation_multiplier())
    tf.add_to_collection('errors', error)
    if optimizer_kwargs is None: optimizer_kwargs = {'lr': tf.Variable(.01, name='eta'),
                                                     'mu': tf.Variable(.5, name='mu')}
    tr_dynamics = optimizer.create(s, loss=training_error, w_is_state=True, **optimizer_kwargs)
    regularization_hyperparameters = []
    if rho_l1s is not None:
        regularization_hyperparameters += rho_l1s
    if rho_l2s is not None:
        regularization_hyperparameters += rho_l2s
    synthetic_hyperparameters = []
    if synthetic_hypers:
        synthetic_hyperparameters.append(gamma)
    hyper_dict = {error: regularization_hyperparameters + synthetic_hyperparameters}  
    if algo_hyper_wrt_tr_error:  
        hyper_dict[training_error] = tr_dynamics.get_optimization_hyperparameters(only_variables=True)
    else:
        hyper_dict[error] += tr_dynamics.get_optimization_hyperparameters(only_variables=True)
    hyper_opt = rf.HyperOptimizer(tr_dynamics, hyper_dict,
                                  method=rf.ReverseHG if mode == 'reverse' else rf.ForwardHG,
                                  hyper_optimizer_class=hyper_optimizer, **hyper_optimizer_kwargs or {})
    positivity = rf.positivity(hyper_opt.hyper_list)
    ev_data = ExampleVisiting(datasets.train, batch_size=batch_size, epochs=epochs)
    if epochs: ev_data.generate_visiting_scheme()
    tr_supplier = ev_data.create_feed_dict_supplier(x, y)
    val_supplier = datasets.validation.create_supplier(x, y)
    def _all_training_supplier():
        return {x: datasets.train.data, y: datasets.train.target}
    val_feed_dict_suppliers = {error: val_supplier}
    if algo_hyper_wrt_tr_error: val_feed_dict_suppliers[training_error] = _all_training_supplier
    T = set_T or ev_data.T if mode != 'rtho' else hyper_batch_size
    with tf.Session(config=rf.CONFIG_GPU_GROWTH).as_default():
        hyper_opt.initialize()
        for k in range(hyper_iterations):
            hyper_opt.run(T, train_feed_dict_supplier=tr_supplier, val_feed_dict_suppliers=val_feed_dict_suppliers,
                          hyper_constraints_ops=positivity)
            if mode != 'rtho':
                hyper_opt.initialize()
def experiment_no_saver ( datasets = None , model = 'log_reg' , model_kwargs = None , l1 = 0. , l2 = 0. , synthetic_hypers = None , set_T = None , optimizer = rf . MomentumOptimizer , optimizer_kwargs = None , batch_size = 200 , algo_hyper_wrt_tr_error = False , mode = 'reverse' , hyper_optimizer = rf . AdamOptimizer , hyper_optimizer_kwargs = None , hyper_iterations = 100 , hyper_batch_size = 100 , epochs = None , do_print = True ) : assert mode in HO_MODES if synthetic_hypers : batch_size = synthetic_hypers if datasets is None : datasets = load_dataset ( ) x , model = create_model ( datasets , model , ** model_kwargs or { } ) s , out , ws , y , error , training_error , rho_l1s , reg_l1s , rho_l2s , reg_l2s , accuracy , base_tr_error , gamma = define_errors_default_models ( model , l1 , l2 , synthetic_hypers = synthetic_hypers , augment = optimizer . get_augmentation_multiplier ( ) ) tf . add_to_collection ( 'errors' , error ) if optimizer_kwargs is None : optimizer_kwargs = { 'lr' : tf . Variable ( .01 , name = 'eta' ) , 'mu' : tf . Variable ( .5 , name = 'mu' ) } tr_dynamics = optimizer . create ( s , loss = training_error , w_is_state = True , ** optimizer_kwargs ) regularization_hyperparameters = [ ] if rho_l1s is not None : regularization_hyperparameters += rho_l1s if rho_l2s is not None : regularization_hyperparameters += rho_l2s synthetic_hyperparameters = [ ] if synthetic_hypers : synthetic_hyperparameters . append ( gamma ) hyper_dict = { error : regularization_hyperparameters + synthetic_hyperparameters } if algo_hyper_wrt_tr_error : hyper_dict [ training_error ] = tr_dynamics . get_optimization_hyperparameters ( only_variables = True ) else : hyper_dict [ error ] += tr_dynamics . get_optimization_hyperparameters ( only_variables = True ) hyper_opt = rf . HyperOptimizer ( tr_dynamics , hyper_dict , method = rf . ReverseHG if mode == 'reverse' else rf . ForwardHG , hyper_optimizer_class = hyper_optimizer , ** hyper_optimizer_kwargs or { } ) positivity = rf . positivity ( hyper_opt . hyper_list ) ev_data = ExampleVisiting ( datasets . train , batch_size = batch_size , epochs = epochs ) if epochs : ev_data . generate_visiting_scheme ( ) tr_supplier = ev_data . create_feed_dict_supplier ( x , y ) val_supplier = datasets . validation . create_supplier ( x , y ) def _all_training_supplier ( ) : return { x : datasets . train . data , y : datasets . train . target } val_feed_dict_suppliers = { error : val_supplier } if algo_hyper_wrt_tr_error : val_feed_dict_suppliers [ training_error ] = _all_training_supplier T = set_T or ev_data . T if mode != 'rtho' else hyper_batch_size with tf . Session ( config = rf . CONFIG_GPU_GROWTH ) . as_default ( ) : hyper_opt . initialize ( ) for k in range ( hyper_iterations ) : hyper_opt . run ( T , train_feed_dict_supplier = tr_supplier , val_feed_dict_suppliers = val_feed_dict_suppliers , hyper_constraints_ops = positivity ) if mode != 'rtho' : hyper_opt . initialize ( )
General method for conducting various simple experiments (on MNIST dataset) with RFHO package.
General method for conducting various simple experiments ( on MNIST dataset ) with RFHO package .
"""
    General method for conducting various simple experiments (on MNIST dataset) with RFHO package.

    :param datasets: (some dataset, usually MNIST....)
    :param model: (default logarithmic regression) model type
    :param model_kwargs:
    :param l1: Initial value for l1 regularizer weight (if None does not uses it)
    :param l2: Initial value for l2 regularizer weight (if None does not uses it)
    :param synthetic_hypers: (default None, for benchmarking purposes) if integer adds some synthetic hyperparameters
                                to the task.
    :param set_T:
    :param optimizer: (default `MomentumOptimizer`) optimizer for parameters
    :param optimizer_kwargs:
    :param batch_size:
    :param epochs: number of ephocs
    :param algo_hyper_wrt_tr_error: (default False) if True optimizes the algorithmic hyperparameters (learning rate, ..
                                    w.r.t. training error instead of validation error
    :param mode: forward reverse or rtho
    :param hyper_optimizer: optimizer for the hyperparameters
    :param hyper_optimizer_kwargs:
    :param hyper_iterations: number of hyper-iterations
    :param hyper_batch_size: hyper-batch size when RTHO is used
    :param do_print: if True (default) prints intermediate results...
    :return:
    """ # altrimenti divento matto! # create hyper_dict # end of hyperparameters # it is possible to optimize different hyperparameters wrt different validation errors # stochastic descent # feed_dict supplier for validation errors # number of iterations
None
datasets
(some dataset, usually MNIST....)
( some dataset usually MNIST .... )
model
(default logarithmic regression) model type
( default logarithmic regression ) model type
model_kwargs
None
l1
Initial value for l1 regularizer weight (if None does not uses it)
Initial value for l1 regularizer weight ( if None does not uses it )
l2
Initial value for l2 regularizer weight (if None does not uses it)
Initial value for l2 regularizer weight ( if None does not uses it )
synthetic_hypers
(default None, for benchmarking purposes) if integer adds some synthetic hyperparameters
to the task.
( default None for benchmarking purposes ) if integer adds some synthetic hyperparameters to the task .
set_T
None
optimizer
(default `MomentumOptimizer`) optimizer for parameters
( default ` MomentumOptimizer ` ) optimizer for parameters
optimizer_kwargs
None
batch_size
None
algo_hyper_wrt_tr_error
(default False) if True optimizes the algorithmic hyperparameters (learning rate,
w.r.t. training error instead of validation error
( default False ) if True optimizes the algorithmic hyperparameters ( learning rate w . r . t . training error instead of validation error
mode
forward reverse or rtho
forward reverse or rtho
hyper_optimizer
optimizer for the hyperparameters
optimizer for the hyperparameters
hyper_optimizer_kwargs
None
hyper_iterations
number of hyper-iterations
number of hyper - iterations
hyper_batch_size
hyper-batch size when RTHO is used
hyper - batch size when RTHO is used
epochs
number of ephocs
number of ephocs
do_print
if True (default) prints intermediate results
if True ( default ) prints intermediate results