anchor.diff:          (C (lr+sgd-C learner))
           (eta (lr+sgd-eta learner))
           (tmp-vec (lr+sgd-tmp-vec learner))
  -        (g (lr+sgd-g learner)))
  +        (g (lr+sgd-g learner))
  +        (tmp-float (lr+sgd-tmp-float learner)))
       (declare (type double-float bias C eta)
  -             (type (simple-array double-float) weight tmp-vec g))
  +             (type (simple-array double-float) weight tmp-vec g)
  +             (type (simple-array double-float 1) tmp-float))
       ;; calc g (gradient)
  -    (logistic-regression-gradient! training-label input weight bias C tmp-vec g)
  +    (logistic-regression-gradient! training-label input weight bias C tmp-vec g tmp-float)
       (v*n g eta g)
       (v- weight g weight)
  -    (setf (lr+sgd-bias learner) (- bias (* eta (refr))))))
  +    (setf (lr+sgd-bias learner) (- bias (* eta (aref tmp-float 0))))))
   
   ;; Adam: A Method for Stochastic Optimization (https://arxiv.org/abs/1412.6980)
   (defstruct (lr+adam (:constructor %make-lr+adam)
  
positive.code:    :v0 0d0
     :beta1^t beta1
     :beta2^t beta2
     :tmp-vec (make-dvec input-dimension 0d0)))
  
  (define-learner lr+adam (learner input training-label)
    (let ((weight (lr+adam-weight learner)) (bias (lr+adam-bias learner))
          (C (lr+adam-C learner)) (tmp-vec (lr+adam-tmp-vec learner))
          (g (lr+adam-g learner)) (g0 0d0)
          (m (lr+adam-m learner)) (m0 (lr+adam-m0 learner))
          (v (lr+adam-v learner)) (v0 (lr+adam-v0 learner))
  
negative.code:      :sigma    (make-dvec input-dimension 1d0)
       :sigma0 1d0
       :tmp-vec1 (make-dvec input-dimension 0d0)
       :tmp-vec2 (make-dvec input-dimension 0d0)
       :tmp-float (make-dvec 1 0d0))))
  
  (define-learner scw (learner input training-label)
    (let ((tmp-float (scw-tmp-float learner)))
      (declare (type (simple-array double-float 1) tmp-float))
      (f! input (scw-weight learner) (scw-bias learner) tmp-float)
      (let ((m (* training-label (aref tmp-float 0)))
            (bias (scw-bias learner))
            (sigma0 (scw-sigma0 learner))
            (phi (scw-phi learner))
            (psi (scw-psi learner))
            (zeta (scw-zeta learner))
            (C (scw-C learner)))
        (declare (type double-float m bias sigma0 phi psi zeta C))
        (dot! (v* (scw-sigma learner) input (scw-tmp-vec1 learner)) input tmp-float)
        (let ((v (+ sigma0 (aref tmp-float 0))))
          (declare (type (double-float 0d0) v))
          (let ((loss (- (* phi (sqrt v)) m)))
            (declare (type double-float loss))
            (when (> loss 0d0)
              (let ((alpha-sqrt-inner (+ (/ (* m m phi phi phi phi) 4d0) (* v phi phi zeta))))
                (declare (type (double-float 0d0) alpha-sqrt-inner))
                (let ((alpha (min C (max 0d0 (- (sqrt alpha-sqrt-inner) (* m psi))))))
                  (declare (type double-float alpha))
                  (let ((u-sqrt-inner (+ (* alpha alpha v v phi phi) (* 4d0 v))))
                    (declare (type (double-float 0d0) u-sqrt-inner))
                    (let ((u (let ((base (- (sqrt u-sqrt-inner) (* alpha v phi))))
                               (declare (type double-float base))
                               (/ (* base base) 4d0))))
                      (declare (type (double-float 0d0) u))
                      (let ((beta (/ (* alpha phi) (+ (sqrt u) (* v alpha phi)))))
                        (declare (type double-float beta))
                        ;; Update weight
                        (v*n (scw-tmp-vec1 learner) (* alpha training-label) (scw-tmp-vec2 learner))
                        (v+ (scw-weight learner) (scw-tmp-vec2 learner) (scw-weight learner))
                        ;; Update bias
                        (setf (scw-bias learner) (+ bias (* alpha sigma0 training-label)))
                        ;; Update sigma
                        (v* (scw-tmp-vec1 learner) (scw-tmp-vec1 learner) (scw-tmp-vec1 learner))
                        (v*n (scw-tmp-vec1 learner) beta (scw-tmp-vec1 learner))
                        (v- (scw-sigma learner) (scw-tmp-vec1 learner) (scw-sigma learner))
                        ;; Update sigma0
                        (setf (scw-sigma0 learner)
                              (- sigma0 (* beta sigma0 sigma0))))))))))))))
  
  ;;; Logistic regression (L2 regularization)
  
  (defmacro sigmoid (x)
    `(/ 1d0 (+ 1d0 (exp (* -1d0 ,x)))))
  
  (defun logistic-regression-gradient! (training-label input-vector weight-vector bias C tmp-vec g-result g0-result)
    (declare (type double-float training-label bias C)
             (type (simple-array double-float) input-vector weight-vector tmp-vec g-result)
             (type (simple-array double-float 1) g0-result)
             (optimize (speed 3) (safety 0)))
    (f! input-vector weight-vector bias g0-result)
    (let ((sigmoid-val (sigmoid (* training-label (aref g0-result 0)))))
      (declare (type (double-float 0d0) sigmoid-val))
      ;; set gradient-vector to g-result
      (v*n input-vector
           (* (- 1d0 sigmoid-val) (* -1d0 training-label))
           tmp-vec)
      (v*n weight-vector (* 2d0 C) g-result)
      (v+ tmp-vec g-result g-result)
      ;; return g0
      (setf (aref g0-result 0)
            (+ (* (- 1d0 sigmoid-val)
                  (* -1d0 training-label))
               (* 2d0 C bias)))
      (values)))
  