class_docstr: Quasi-Newton algorithm. Every iteration quasi-Network method approximates
  inverse Hessian matrix with iterative updates. It doesn't have ``step``
  parameter. Instead, algorithm applies line search for the step parameter
  that satisfies strong Wolfe condition. Parameters that control wolfe
  search start with the ``wolfe_`` prefix.
  
  Parameters
  ----------
  update_function : ``bfgs``, ``dfp``, ``sr1``
      Update function for the iterative inverse hessian matrix
      approximation. Defaults to ``bfgs``.
  
      - ``bfgs`` -  It's rank 2 formula update. It can suffer from
        round-off error and inaccurate line searches.
  
      - ``dfp`` - DFP is a method very similar to BFGS. It's rank 2 formula
        update. It can suffer from round-off error and inaccurate line
        searches.
  
      - ``sr1`` - Symmetric rank 1 (SR1). Generates update for the
        inverse hessian matrix adding symmetric rank-1 matrix. It's
        possible that there is no rank 1 updates for the matrix and in
        this case update won't be applied and original inverse hessian
        will be returned.
  
  h0_scale : float
      Default Hessian matrix is an identity matrix. The
      ``h0_scale`` parameter scales identity matrix.
      Defaults to ``1``.
  
  epsilon : float
      Controls numerical stability for the ``update_function`` parameter.
      Defaults to ``1e-7``.
  
  {WolfeLineSearchForStep.Parameters}
  
  {BaseOptimizer.network}
  
  {BaseOptimizer.loss}
  
  {BaseOptimizer.show_epoch}
  
  {BaseOptimizer.shuffle_data}
  
  {BaseOptimizer.signals}
  
  {BaseOptimizer.verbose}
  
  {BaseOptimizer.regularizer}
  
  Notes
  -----
  - Method requires all training data during propagation, which means
    it cannot be trained with mini-batches.
  
  Attributes
  ----------
  {BaseOptimizer.Attributes}
  
  Methods
  -------
  {BaseOptimizer.Methods}
  
  Examples
  --------
  >>> import numpy as np
  >>> from neupy import algorithms
  >>> from neupy.layers import *
  >>>
  >>> x_train = np.array([[1, 2], [3, 4]])
  >>> y_train = np.array([[1], [0]])
  >>>
  >>> optimizer = algorithms.QuasiNewton(
  ...     Input(2) >> Sigmoid(3) >> Sigmoid(1),
  ...     update_function='bfgs'
  ... )
  >>> optimizer.train(x_train, y_train, epochs=10)
  
  References
  ----------
  [1] Yang Ding, Enkeleida Lushi, Qingguo Li,
      Investigation of quasi-Newton methods for unconstrained optimization.
      http://people.math.sfu.ca/~elushi/project_833.pdf
  
  [2] Jorge Nocedal, Stephen J. Wright, Numerical Optimization.
      Chapter 6, Quasi-Newton Methods, p. 135-163
class_docstr_tok: ["Quasi-Newton","algorithm",".","Every","iteration","quasi-Network","method","approximates","inverse","Hessian","matrix","with","iterative","updates",".","It","does","n't","have","``","step","``","parameter",".","Instead",",","algorithm","applies","line","search","for","the","step","parameter","that","satisfies","strong","Wolfe","condition",".","Parameters","that","control","wolfe","search","start","with","the","``","wolfe_","``","prefix",".","Parameters","--","--","--","--","--","update_function",":","``","bfgs","``",",","``","dfp","``",",","``","sr1","``","Update","function","for","the","iterative","inverse","hessian","matrix","approximation",".","Defaults","to","``","bfgs","``",".","-","``","bfgs","``","-","It","'s","rank","2","formula","update",".","It","can","suffer","from","round-off","error","and","inaccurate","line","searches",".","-","``","dfp","``","-","DFP","is","a","method","very","similar","to","BFGS",".","It","'s","rank","2","formula","update",".","It","can","suffer","from","round-off","error","and","inaccurate","line","searches",".","-","``","sr1","``","-","Symmetric","rank","1","(","SR1",")",".","Generates","update","for","the","inverse","hessian","matrix","adding","symmetric","rank-1","matrix",".","It's","possible","that","there","is","no","rank","1","updates","for","the","matrix","and","in","this","case","update","wo","n't","be","applied","and","original","inverse","hessian","will","be","returned",".","h0_scale",":","float","Default","Hessian","matrix","is","an","identity","matrix",".","The","``","h0_scale","``","parameter","scales","identity","matrix",".","Defaults","to","``","1","``",".","epsilon",":","float","Controls","numerical","stability","for","the","``","update_function","``","parameter",".","Defaults","to","``","1e-7","``",".","{","WolfeLineSearchForStep.Parameters","}","{","BaseOptimizer.network","}","{","BaseOptimizer.loss","}","{","BaseOptimizer.show_epoch","}","{","BaseOptimizer.shuffle_data","}","{","BaseOptimizer.signals","}","{","BaseOptimizer.verbose","}","{","BaseOptimizer.regularizer","}","Notes","--","--","-","-","Method","requires","all","training","data","during","propagation",",","which","means","it","can","not","be","trained","with","mini-batches",".","Attributes","--","--","--","--","--","{","BaseOptimizer.Attributes","}","Methods","--","--","--","-","{","BaseOptimizer.Methods","}","Examples","--","--","--","--",">",">",">","import","numpy","as","np",">",">",">","from","neupy","import","algorithms",">",">",">","from","neupy.layers","import","*",">",">",">",">",">",">","x_train","=","np.array","(","[","[","1",",","2","]",",","[","3",",","4","]","]",")",">",">",">","y_train","=","np.array","(","[","[","1","]",",","[","0","]","]",")",">",">",">",">",">",">","optimizer","=","algorithms.QuasiNewton","(","...","Input","(","2",")",">",">","Sigmoid","(","3",")",">",">","Sigmoid","(","1",")",",","...","update_function='bfgs'","...",")",">",">",">","optimizer.train","(","x_train",",","y_train",",","epochs=10",")","References","--","--","--","--","--","[","1","]","Yang","Ding",",","Enkeleida","Lushi",",","Qingguo","Li",",","Investigation","of","quasi-Newton","methods","for","unconstrained","optimization",".","http",":","\/\/people.math.sfu.ca\/~elushi\/project_833.pdf","[","2","]","Jorge","Nocedal",",","Stephen","J.","Wright",",","Numerical","Optimization",".","Chapter","6",",","Quasi-Newton","Methods",",","p.","135-163"]
code: def init_train_updates(self):
      self.init_variables()
  
      iteration = self.variables.iteration
      inv_hessian = self.variables.inv_hessian
      prev_params = self.variables.prev_params
      prev_full_gradient = self.variables.prev_full_gradient
  
      variables = self.network.variables
      params = [var for var in variables.values() if var.trainable]
      param_vector = make_single_vector(params)
  
      gradients = tf.gradients(self.variables.loss, params)
      full_gradient = make_single_vector(gradients)
  
      new_inv_hessian = tf.where(
          tf.equal(iteration, 0),
          inv_hessian,
          self.update_function(
              inv_H=inv_hessian,
              delta_w=param_vector - prev_params,
              delta_grad=full_gradient - prev_full_gradient,
              epsilon=self.epsilon,
          ),
      )
      param_delta = -dot(new_inv_hessian, full_gradient)
      step = self.find_optimal_step(param_vector, param_delta)
      updated_params = param_vector + step * param_delta
      updates = setup_parameter_updates(params, updated_params)
  
      # We have to compute these values first, otherwise
      # parallelization, in tensorflow, can mix update order
      # and, for example, previous gradient can be equal to
      # current gradient value. It happens because tensorflow
      # try to execute operations in parallel.
      required_variables = [
          new_inv_hessian,
          param_vector,
          full_gradient,
      ]
      with tf.control_dependencies(required_variables):
          updates.extend(
              [
                  inv_hessian.assign(new_inv_hessian),
                  prev_params.assign(param_vector),
                  prev_full_gradient.assign(full_gradient),
                  iteration.assign(iteration + 1),
              ]
          )
  
      return updates
  
file_path: neupy/neupy/algorithms/gd/quasi_newton.py
imports: import tensorflow
  from neupy.core.config import Configurable
  from neupy.core.properties import ChoiceProperty, NumberProperty, WithdrawProperty, IntProperty
  from neupy.utils.tf_utils import setup_parameter_updates
  from neupy.algorithms.minsearch.wolfe import line_search
  from neupy.utils import asfloat, dot, outer, as_tuple, function_name_scope, make_single_vector
  from .base import BaseOptimizer
label_desc: Use image node_id 2 for calling the QuasiNewton obj's underlying member method code with example usage: obj.init_train_updates() and returns: updates