### Kernel functions
abstract type SmoothingKernel{T} end

struct GaussianKernel{T} <: SmoothingKernel{T}
  h::T
end

struct EpanechnikovKernel{T} <: SmoothingKernel{T}
  h::T
end

createKernel(::Type{GaussianKernel{T}}, h::T) where {T <: AbstractFloat} = GaussianKernel{T}(h)
createKernel(::Type{EpanechnikovKernel{T}}, h::T) where {T <: AbstractFloat} = EpanechnikovKernel{T}(h)

evaluate(k::GaussianKernel{T}, x::T, y::T) where {T <: AbstractFloat} = exp(-(x-y)^2. / k.h) / k.h
function evaluate(k::EpanechnikovKernel{T}, x::T, y::T) where {T <: AbstractFloat}
    u = (x - y) / k.h
    abs(u) >= 1. ? zero(T) : 0.75 * (1. - u^2.) / k.h
end

############################################################
#
# local polynomial regression with lasso
#
############################################################

function locpolyl1(
  X::Matrix{T}, z::Vector{T}, y::Vector{T},
  zgrid::Vector{T},
  degree::Int64,
  kernel::SmoothingKernel{T},
  λ0::T,
  refit::Bool,
  options::CDOptions=CDOptions()) where {T <: AbstractFloat}

  # construct inner options because we do not want warmStart = false
  # we want to start from the previous iteration since the points
  # on the grid should be close to each other
  opt = CDOptions(options.maxIter, options.optTol, options.randomize, true, options.numSteps)

  n, p = size(X)
  ep = p * (degree + 1)
  out = spzeros(T, ep, length(zgrid))
  outR = spzeros(T, ep, length(zgrid))

  # temporary storage
  w = Array{T}(undef, n)
  expandX = Array{T}(undef, n, ep)
  S = BitArray(undef, ep)
  stdX = Array{T}(undef, ep)
  f = CDWeightedLSLoss(y, expandX, w)        # inner parts of f will be modified in a loop
  g = ProxL1(λ0, stdX)
  β = SparseIterate(ep)

  ind = 0
  for z0 in zgrid
    ind += 1

    # the following two should update f
    w .= evaluate.(Ref(kernel), z, Ref(z0))
    _expand_X!(expandX, X, z, z0, degree)
    _stdX!(stdX, w, expandX)

    # solve for β
    coordinateDescent!(β, f, g, opt)
    out[:, ind] = β

    if refit
        get_nonzero_coordinates!(S, β, p, degree, true)
        Xs = view(expandX, :, S)
        tmp = Xs' * Diagonal(w)
        outR[S, ind] = (tmp * Xs) \ (tmp * y)
    end
  end
  out, outR
end

# leave one out for h selection
function lvocv_locpolyl1(
    X::Matrix{T}, z::Vector{T}, y::Vector{T},
    degree::Int64,                                     # degree of the polynomial
    hArr::Vector{T},
    kernelType::Type{<:SmoothingKernel},
    λ0::T,
    options::CDOptions=CDOptions()) where {T <: AbstractFloat}

    n, p = size(X)
    numH = length(hArr)
    MSE = zeros(numH)

    opt = CDOptions(options.maxIter, options.optTol, options.randomize, true, options.numSteps)

    ep = p * (degree + 1)
    w = Array{T}(undef, n)
    wX = Array{T}(undef, n, ep)
    stdX = Array{T}(undef, ep)
    S = BitArray(undef, ep)

    f = CDWeightedLSLoss(y, wX, w)
    β = SparseIterate(ep)

    for indH = 1:numH
        kernel = createKernel(kernelType, hArr[indH])
        for i = 1:n
            # update variables
            z0 = z[i]
            w .= evaluate.(Ref(kernel), z, Ref(z0))
            w[i] = zero(T)
            _expand_X!(wX, X, z, z0, degree)
            _stdX!(stdX, w, wX)
            _findInitResiduals!(w, wX, y, min(10, ep), f.r)

            # compute sigma
            σ = _getSigma(w, f.r)
            g = ProxL1(λ0 * σ, stdX)
            for iter=1:10
              coordinateDescent!(β, f, g, opt)
              σnew = _getSigma(w, f.r)
              if abs(σnew - σ) / σ < 1e-2
                break
              end
              σ = σnew
              g = ProxL1(λ0 * σ, stdX)
            end

            # refit and make prediction
            get_nonzero_coordinates!(S, β, p, degree, true)
            Xs = view(wX, :, S)
            Yh = dot(wX[i, S], (Xs' * Diagonal(w) * Xs) \ (Xs' * Diagonal(w) * y))
            MSE[indH] += (Yh - y[i])^2.
        end
    end
    MSE
end

function refit_locpolyl1(
    X::Matrix{T}, z::Vector{T}, y::Vector{T},
    z0::T,
    degree::Int64,
    kernel::SmoothingKernel{T},
    β::Union{SparseIterate{T}, SparseVector{T}}
    ) where {T <: AbstractFloat}

    n, p = size(X)
    ep = p * (degree + 1)

    S = BitArray(undef, p)
    get_nonzero_coordinates!(S, β, p, degree, false)
    βr = locpoly(view(X, :, S), z, y, z0, degree, kernel)
    (βr, S)
end

# function refit_locpolyl1(
#     X::Matrix{T}, z::Vector{T}, y::Vector{T},
#     zgrid::Vector{T},
#     degree::Int64,
#     kernel::SmoothingKernel{T},
#     β::SparseMatrixCSC{T}
#     ) where {T <: AbstractFloat}
#
#     n, p = size(X)
#     ep = p * (degree + 1)
#     S = BitArray(undef, p, length(zgrid))
#     fill!(S, false)
#     w = Array{T}(undef, n)
#     expandedX = Array{T}(undef, n, ep)
#     βr = Array{Array{T}}(undef, length(zgrid))
#     tS = BitArray(undef, p)
#
#     for indZ = 1:length(zgrid)
#         o_coordinates!(tS, β[:, indZ], p, degree, false)
#         @show β[:, indZ]
#         @show S[:, indZ] = tS
#         z0 = zgrid[indZ]
#         num_col_refit = sum(S[:, indZ])*(degree+1)
#         Xs = view(X, :, S[:, indZ])
#         βr[indZ] = _locpoly!(view(expandedX, :, 1:num_col_refit), w, Xs, z, y, z0, degree, kernel)
#     end
#     (βr, S)
# end

############################################################
#
# local polynomial regression low dimensions
#
############################################################

function _locpoly!(
  wX::Union{SubArray{T, 2}, Matrix{T}}, w::Vector{T},
  X::Union{SubArray{T, 2}, Matrix{T}},
  z::Union{SubArray{T, 1}, Vector{T}},
  y::Union{SubArray{T, 1}, Vector{T}},
  z0::T,
  degree::Int64,
  kernel::SmoothingKernel{T}) where {T <: AbstractFloat}

  w .= sqrt.(evaluate.(Ref(kernel), z, Ref(z0)))    # square root of kernel weights
  _expand_wX!(wX, w, X, z, z0, degree)              # √w ⋅ x ⊗ [1 (zi - z0) ... (zi-z0)^q]
  @. w *= y                                         # √w ⋅ y
  qr!(wX) \ w
end

locpoly(
  X::Union{SubArray{T, 2}, Matrix{T}}, z::Vector{T}, y::Vector{T},
  z0::T, degree::Int64, kernel::SmoothingKernel=GaussianKernel(one(T))) where {T <: AbstractFloat} =
    _locpoly!(Array{T}(undef, length(y), size(X, 2) * (degree+1)), similar(y), X, z, y, z0, degree, kernel)

function locpoly(
  X::Union{SubArray{T, 2}, Matrix{T}}, z::Vector{T}, y::Vector{T},
  zgrid::Vector{T},
  degree::Int64,                                     # degree of the polynomial
  kernel::SmoothingKernel{T}=GaussianKernel(one(T))) where {T <: AbstractFloat}

  n, p = size(X)
  ep = p * (degree + 1)
  out = Array{T}(undef, ep, length(zgrid))
  w = Array{T}(undef, n)
  wX = Array{T}(undef, n, ep)

  ind = 0
  for z0 in zgrid
    ind += 1
    out[:, ind] = _locpoly!(wX, w, X, z, y, z0, degree, kernel)
  end
  out
end

function getResiduals!(
     ϵhat::Vector{T},
     X::Union{SubArray{T, 2}, Matrix{T}}, z::Vector{T}, y::Vector{T},
     zgrid::Vector{T},
     βhat::Union{Matrix{T}, SparseMatrixCSC{T}},
     degree::Int64,
     kernel::SmoothingKernel{T}=GaussianKernel(one(T))
    ) where {T <: AbstractFloat}

    n, p = size(X)
    ep = p * (degree + 1)
    βi = spzeros(ep)
    for i=1:n
        get_beta!(βi, zgrid, βhat, z[i])
        ϵhat[i] = y[i] - dot(X[i, :], βi[1:(degree+1):ep])
    end

    ϵhat
end

function getStandardError(
    X::Union{SubArray{T, 2}, Matrix{T}}, z::Vector{T},
    σ2::T,
    z0::T,
    degree::Int64,                                     # degree of the polynomial
    kernel::SmoothingKernel{T}) where {T <: AbstractFloat}

    n, p = size(X)
    ep = p * (degree + 1)
    out = Array{T}(undef, p)
    w = Array{T}(undef, n)
    w1 = Array{T}(undef, n)
    XtwX = Array{T}(undef, ep, ep)
    XtwwX = Array{T}(undef, ep, ep)

    w .= evaluate.(Ref(kernel), z, Ref(z0))
    _expand_Xt_w_X!(XtwX, w, X, z, z0, degree)

    w .= w .* w
    _expand_Xt_w_X!(XtwwX, w, X, z, z0, degree)

    A = inv(XtwX)
    varMat = A * XtwwX * A

    for j=1:p
        out[j] = varMat[(j-1)*(degree+1)+1, (j-1)*(degree+1)+1]
    end

    out
end

function getStandardErrorHEW(
    X::Union{SubArray{T, 2}, Matrix{T}}, z::Vector{T},
    ϵ_sqr::Vector{T},
    z0::T,
    degree::Int64,                                     # degree of the polynomial
    kernel::SmoothingKernel{T}) where {T <: AbstractFloat}

    n, p = size(X)
    ep = p * (degree + 1)
    out = Array{T}(undef, p)
    w = Array{T}(undef, n)
    w1 = Array{T}(undef, n)
    XtwX = Array{T}(undef, ep, ep)
    XtwΨwX = Array{T}(undef, ep, ep)

    w .= evaluate.(Ref(kernel), z, Ref(z0))
    _expand_Xt_w_X!(XtwX, w, X, z, z0, degree)

    @. w = w * w * ϵ_sqr
    _expand_Xt_w_X!(XtwΨwX, w, X, z, z0, degree)

    A = inv(XtwX)
    varMat = A * XtwΨwX * A

    for j=1:p
        out[j] = varMat[(j-1)*(degree+1)+1, (j-1)*(degree+1)+1]
    end

    out
end

# function locpoly_alt(
#   X::Matrix{T}, z::Vector{T}, y::Vector{T},
#   zgrid::Vector{T},
#   degree::Int64,                                     # degree of the polynomial
#   kernel::SmoothingKernel{T}=GaussianKernel(one(T))) where {T <: AbstractFloat}
#
#   n, p = size(X)
#   ep = p * (degree + 1)
#   out = Array{T}(undef, ep, length(zgrid))
#   w = Array{T}(undef, n)
#   Xt_w_Y = Array{T}(undef, ep)
#   Xt_w_X = Array{T}(undef, ep, ep)
#
#   ind = 0
#   for z0 in zgrid
#       w .= evaluate.(Ref(kernel), z, Ref(z0))
#       _expand_Xt_w_Y!(Xt_w_Y, w, X, z, y, z0, degree)
#       _expand_Xt_w_X!(Xt_w_X, w, X, z, z0, degree)
#       ind += 1
#       out[:, ind] = Xt_w_X \ Xt_w_Y
#   end
#   out
# end

# leave one out for h selection
function lvocv_locpoly(
    X::Matrix{T}, z::Vector{T}, y::Vector{T},
    degree::Int64,                                     # degree of the polynomial
    hArr::Vector{T},
    kernelType::Type{<:SmoothingKernel}) where {T <: AbstractFloat}

    n, p = size(X)
    numH = length(hArr)
    MSE = zeros(numH)

    ep = p * (degree + 1)
    w = Array{T}(undef, n-1)
    wX = Array{T}(undef, n-1, ep)
    indOut = BitArray(undef, n)

    for indH = 1:numH
        kernel = createKernel(kernelType, hArr[indH])
        for i = 1:n
            fill!(indOut, true)
            indOut[i] = false
            Xview = view(X, indOut, :)
            Yview = view(y, indOut)
            Zview = view(z, indOut)

            hbeta = _locpoly!(wX, w, Xview, Zview, Yview, z[i], degree, kernel)

            # make prediction
            Yh = dot(X[i, :], hbeta[1:(degree+1):ep])
            MSE[indH] += (Yh - y[i])^2.
        end
    end
    MSE
end

# data split for h selection
function split_locpoly(
    X::Matrix{T}, z::Vector{T}, y::Vector{T},
    Xtest::Matrix{T}, ztest::Vector{T}, ytest::Vector{T},
    zgrid::Vector{T},
    degree::Int64,                                     # degree of the polynomial
    hArr::Vector{T},
    kernelType::Type{<:SmoothingKernel}) where {T <: AbstractFloat}

    n, p = size(X)
    numH = length(hArr)
    MSE = zeros(numH)
    ep = p * (degree + 1)
    βhati = zeros(ep)

    for indH = 1:numH
        kernel = createKernel(kernelType, hArr[indH])
        βhat = locpoly(X, z, y, zgrid, degree, kernel)
        for i = 1:n
            get_beta!(βhati, zgrid, βhat, ztest[i])

            # make prediction
            Yh = dot(Xtest[i, :], βhati[1:(degree+1):ep])
            MSE[indH] += (ytest[i] - Yh)^2.
        end
    end
    MSE
end

# # leave one out for h selection
# function lvocv_locpoly(
#     X::Matrix{T}, z::Vector{T}, y::Vector{T},
#     degree::Int64,                                     # degree of the polynomial
#     hArr::Vector{T},
#     kernelType::Type{<:SmoothingKernel}) where {T <: AbstractFloat}
#
#     n, p = size(X)
#     numH = length(hArr)
#     MSE = zeros(numH)
#
#     ep = p * (degree + 1)
#     w = Array{T}(undef, n)
#     Xt_w_Y = Array{T}(undef, ep)
#     Xt_w_X = Array{T}(undef, ep, ep)
#
#     for indH = 1:numH
#         kernel = createKernel(kernelType, hArr[indH])
#         for i = 1:n
#             z0 = z[i]
#             w .= evaluate.(Ref(kernel), z, Ref(z0))
#             w[i] = zero(T)
#             _expand_Xt_w_Y!(Xt_w_Y, w, X, z, y, z0, degree)
#             _expand_Xt_w_X!(Xt_w_X, w, X, z, z0, degree)
#             hbeta = Xt_w_X \ Xt_w_Y
#             # make prediction
#             Yh = dot(X[i, :], hbeta[1:(degree+1):ep])
#             MSE[indH] += (Yh - y[i])^2.
#         end
#     end
#     MSE
# end

############################################################
#
# utils
#
############################################################

"""
For a given z0 finds two closest points in zgrid
and corresponding values of beta. The output is obtained
by interpolating the beta values.
"""
function get_beta!(
       out::Union{SparseVector{T}, Vector{T}},
       zgrid::Vector{T},
       beta_grid::Union{Matrix{T}, SparseMatrixCSC{T}},
       z0::T) where {T <: AbstractFloat}
    id1 = searchsortedlast(zgrid, z0)
    id2 = searchsortedfirst(zgrid, z0)

    if id1 == id2
        out .= beta_grid[:, id1]
    else
        z1 = zgrid[id1]
        z2 = zgrid[id2]
        α = (z0 - z1) / (z2 - z1)
        out .= α * beta_grid[:, id1] .+ (1-α) * beta_grid[:, id2]
    end
    out
end

get_nonzero_coordinates(
    β::Union{SparseVector{T}, SparseIterate{T}},
    p::Int,
    degree::Int,
    expanded::Bool
    ) where {T <: AbstractFloat}  =
       expanded ? get_nonzero_coordinates!(BitArray(undef, p*(degree+1)), β, p, degree, expanded) : get_nonzero_coordinates!(BitArray(undef, p), β, p, degree, expanded)

function get_nonzero_coordinates!(
    S::BitArray,
    β::Union{SparseVector{T}, SparseIterate{T}},
    p::Int,
    degree::Int,
    expanded::Bool) where {T <: AbstractFloat}

    fill!(S, false)
    for j = 1:p
        nonzero = false
        for k=((j-1)*(degree+1)+1):(j*(degree+1))
            nonzero = nonzero | !iszero(β[k])
        end
        if nonzero
            if expanded
                for k=((j-1)*(degree+1)+1):(j*(degree+1))
                    S[k] = true
                end
            else
                S[j] = true
            end
        end
    end
    S
end

"""
Computes matrix whose each row is equal to
w[i] ⋅ (X[i, :] ⊗ [1, (z - z0), ..., (z-z0)^q])
where q is the degree of the polynomial.

The output matrix is preallocated.
"""
function _expand_wX!(
  wX::Union{SubArray{T, 2}, Matrix{T}},
  w::Vector{T},
  X::Union{SubArray{T, 2}, Matrix{T}},
  z::Union{SubArray{T, 1}, Vector{T}}, z0::T, degree::Int64) where {T <: AbstractFloat}

  n, p = size(X)
  # wX = zeros(n, p*(degree+1))
  for j=1:p
    @inbounds for i=1:n
      v = X[i, j] * w[i]
      df = z[i] - z0
      col = (j-1)*(degree+1) + 1
      @inbounds wX[i, col] = v
      for l=1:degree
        v *= df
        @inbounds wX[i, col + l] = v
      end
    end
  end
  # return qrfact!(tX)
  wX
end

# expands the data matrix X
# each row becomes
# (X[i, :] ⊗ [1, (z - z0), ..., (z-z0)^q])
function _expand_X!(
  tX::Matrix{T},
  X::Union{SubArray{T, 2}, Matrix{T}},
  z::Union{SubArray{T, 1}, Vector{T}}, z0::T, degree::Int64) where {T <: AbstractFloat}

  n, p = size(X)
  for j=1:p
    for i=1:n
      v = X[i, j]
      df = z[i] - z0
      col = (j-1)*(degree+1) + 1
      @inbounds tX[i, col] = v
      for l=1:degree
        v *= df
        @inbounds tX[i, col + l] = v
      end
    end
  end
  tX
end

function _expand_Xt_w_X!(
  Xt_w_X::Matrix{T},
  w::Vector{T}, X::Matrix{T},
  z::Vector{T}, z0::T, degree::Int64) where {T <: AbstractFloat}

  n, p = size(X)
  fill!(Xt_w_X, zero(T))
  @inbounds for j=1:p
    @inbounds for k=j:p
      @inbounds for i=1:n
        # if iszero(w[i])
        #   continue
        # end
        v1 = X[i, j] * w[i]
        df1 = z[i] - z0

        col=(j-1)*(degree+1)+1
        for jj=0:degree
          v2 = X[i, k]
          df2 = z[i] - z0
          if k != j
            krange = 0:degree
          else
            krange = jj:degree
            v2 = v2 * df2^jj
          end

          row = (k-1)*(degree+1)+1
          for kk=krange
            # need X[i, j] * (z[i] - z0)^(jj+kk) * X[i, k] * w[i]
            # v2 = (z[i] - z0)^kk * X[i, k]
            # v1 = (z[i] - z0)^jj * X[i, k] * w[i]
            Xt_w_X[row+kk, col+jj] += v2 * v1
            v2 *= df2
          end
          v1 *= df1
        end
      end
    end
  end

  @inbounds for c=1:size(Xt_w_X, 2)
    for r=c+1:size(Xt_w_X, 1)
      Xt_w_X[c, r] = Xt_w_X[r, c]
    end
  end

  Xt_w_X
end

function _expand_Xt_w_Y!(
  Xt_w_Y::Vector{T},
  w::Vector{T}, X::Matrix{T}, z::Vector{T}, y::Vector{T},
  z0::T, degree::Int64) where {T <: AbstractFloat}

  n, p = size(X)
  fill!(Xt_w_Y, zero(T))
  @inbounds for j=1:p
    @inbounds for i=1:n
      # if iszero(w[i])
      #   continue
      # end
      v = X[i, j] * w[i] * y[i]
      df = z[i] - z0

      col=(j-1)*(degree+1)+1
      @inbounds for jj=0:degree
        # need X[i, j] * (z[i] - z0)^jj * Y[i, k] * w[i]
        Xt_w_Y[col+jj] += v
        v *= df
      end
    end
  end

  Xt_w_Y
end


using Images, ImageView

function draw_seam(img, seam)
    img_w_seam = copy(img)
    for i = 1:size(img)[1]
        img_w_seam[i,seam[i]] = RGB(1,1,1)
    end

    return img_w_seam
end

function write_image(img, i; filebase = "out")
    save(filebase*lpad(string(i),5,string(0))*".png", img)
end

# function to return magnitude of image elements
function brightness(img_element)
    return img_element.r + img_element.g + img_element.b
end

# function to find energy of image
function find_energy(img)
    energy_x = imfilter(brightness.(img), Kernel.sobel()[2])
    energy_y = imfilter(brightness.(img), Kernel.sobel()[1])
    return sqrt.(energy_x.^2 + energy_y.^2)
    return abs2.(imfilter(brightness.(img), Kernel.sobel()))
end

# function to find direction to move in
function find_energy_map(energy)
    energy_map = zeros(size(energy))
    energy_map[end,:] .= energy[end,:]
    next_elements = zeros(Int, size(energy))

    for i = size(energy)[1]-1:-1:1
        for j = 1:size(energy)[2]
            left = max(1, j-1)
            right = min(size(energy)[2], j+1)
            if left < 1
                println("less than 1")
            end

            local_energy, next_element = findmin(energy_map[i+1, left:right])
            energy_map[i,j] += local_energy + energy[i,j]
            next_elements[i,j] = next_element -2

            # correct for only having 2 options on left edge.
            if left == 1 && size(energy)[2] > 3
                next_elements[i,j] += 1
            end
        end
    end

    return energy_map, next_elements
end

function generate_seam(energy, next_elements, element)
    seam = zeros(Int, size(next_elements)[1])
    seam[1] = element

    seam_energy = energy[element]

    for i = 2:length(seam)
        seam[i] = seam[i-1] + next_elements[i, seam[i-1]]
        seam_energy += energy[i, seam[i]]
    end

    return seam, seam_energy
end

# function to create seams and return seam of minimum energy
# n is number of seams
function find_seam(energy)

    energy_map, next_elements = find_energy_map(energy)

    kept_energy = Inf
    kept_seam = zeros(size(energy)[1])

    for i = 1:size(energy)[2]
        seam, seam_energy = generate_seam(energy, next_elements, i)
        if seam_energy < kept_energy
            kept_seam = seam
            kept_energy = seam_energy
        end
    end

    return kept_seam, kept_energy
end

# function to remove seams
function remove_seam(img, seam)
    img_res = (size(img)[1], size(img)[2]-1)

    # preallocate image
    new_img = Array{RGB}(undef, img_res)

    for i = 1:length(seam)
        if i > 1 && i < size(img)[2]
            new_img[i, :] .= vcat(img[i, 1:seam[i]-1],
                                  img[i, seam[i]+1:end])
        elseif i == 1
            new_img[i,:] .= img[i,2:end]
        elseif i == size(img)[2]
            new_img[i,:] .= img[i,1:end-1]
        end
    end

    return new_img
end

# Putting it all together
function seam_carving(img, res)

    if res < 0 || res > size(img)[2]
        error("resolution not acceptable")
    end

    for i = (1:size(img)[2] - res)
        energy = find_energy(img)
        seam, energy = find_seam(energy)
        img = remove_seam(img, seam)
        write_image(img, res, i)
    end
end


function gen_symmetric_matrix(n)
  a = randn(n,n)
  (a + a')/2
end

for n in [10, 100, 500, 1000, 2000]
  a = gen_symmetric_matrix(n)

  tic()
    de, ve = eig(a)
  teig = toq()

  k = 5
  while k <= n/2
    tic()
      des, ves = eigs(a; nev = k)
    teigs = toq()

    #dnorm = norm( de-des[1:k] )
    #vnorm = norm( ve-ves )

    println("$(n): k = $(k), eigd norm = , eigv norm = , eigs time factor = $(teig/teigs), eig time = $(teig), eigs time = $(teigs)")

    k *= 10
  end
end


####
#### MulticlassLDA
####
"""
    LDA(; kwargs...)

$LDA_DESCR

# Keyword Parameters

- `method::Symbol=:gevd`:  choice of solver, one of `:gevd` or `:whiten` methods
- `cov_w::CovarianceEstimator`=SimpleCovariance: an estimator for the within-class
    covariance (used in computing within-class scatter matrix, Sw), by default set
    to the standard `MultivariateStats.SimpleCovariance()` but
    could be set to any robust estimator from `CovarianceEstimation.jl`.
- `cov_b::CovarianceEstimator`=SimpleCovariance: same as `cov_w` but for the between-class
    covariance (used in computing between-class scatter matrix, Sb)
- `out_dim::Int=0`: the output dimension, i.e dimension of the transformed space,
    automatically set if 0 is given (default).
- `regcoef::Float64=1e-6`: regularization coefficient (default value 1e-6). A positive
    value `regcoef * eigmax(Sw)` where `Sw` is the within-class scatter matrix, is added
    to the diagonal of Sw to improve numerical stability. This can be useful if using
    the standard covariance estimator.
- `dist::SemiMetric=SqEuclidean`: the distance metric to use when performing classification
    (to compare the distance between a new point and centroids in the transformed space),
    an alternative choice can be the `CosineDist`.Defaults to `SqEuclidean`

See also the
[package documentation](https://multivariatestatsjl.readthedocs.io/en/latest/lda.html).
For more information about the algorithm, see the paper by Li, Zhu and Ogihara,
[Using Discriminant Analysis for Multi-class Classification:
An Experimental Investigation](http://citeseerx.ist.psu.edu/viewdoc/
download?doi=10.1.1.89.7068&rep=rep1&type=pdf).
"""
@mlj_model mutable struct LDA <: MMI.Probabilistic
    method::Symbol = :gevd::(_ in (:gevd, :whiten))
    cov_w::CovarianceEstimator = MS.SimpleCovariance()
    cov_b::CovarianceEstimator = MS.SimpleCovariance()
    out_dim::Int = 0::(_ ≥ 0)
    regcoef::Float64 = 1e-6::(_ ≥ 0)
    dist::SemiMetric = SqEuclidean()
end

function MMI.fit(model::LDA, ::Int, X, y)
    Xm_t, yplain, classes_seen, p, n, nc, nclasses, integers_seen, out_dim =
        _check_lda_data(model, X, y)
    core_res = MS.fit(
        MS.MulticlassLDA, nc, Xm_t, Int.(yplain);
        method=model.method,
        outdim=out_dim,
        regcoef=model.regcoef,
        covestimator_within=model.cov_w,
        covestimator_between=model.cov_b
    )
    cache = nothing
    report = (
        classes=classes_seen,
        out_dim=MS.outdim(core_res),
        class_means=MS.classmeans(core_res),
        mean=MS.mean(core_res),
        class_weights=MS.classweights(core_res),
        Sw=MS.withclass_scatter(core_res),
        Sb=MS.betweenclass_scatter(core_res),
        nc=nc
    )
    fitresult = (core_res, classes_seen)
    return fitresult, cache, report
end

function _check_lda_data(model, X, y)
    class_list = MMI.classes(y[1]) # Class list containing entries in pool of y.
    nclasses = length(class_list)
    # Class list containing unique entries in y.
    classes_seen = filter(in(unique(y)), class_list)
    nc = length(classes_seen) # Number of classes in pool of y.
    integers_seen = MMI.int(classes_seen)
    # NOTE: copy/transpose.
    Xm_t = _matrix_transpose(model, X) # Now p x n matrix
    yplain = MMI.int(y) # Vector of n ints in {1,..., nclasses}.
    p, n = size(Xm_t)
    # Recode yplain to be in {1,..., nc}
    nc == nclasses || replace!(yplain, (integers_seen .=> 1:nc)...)
    # Check to make sure we have more than one class in training sample.
    # This is to prevent Sb from being a zero matrix.
    if nc <= 1
        throw(
            ArgumentError(
                "The number of unique classes in "*
                "traning sample has to be greater than one"
            )
        )
    end

    # Check to make sure we have more samples than classes.
    # This is to prevent Sw from being the zero matrix.
    if n <= nc
        throw(
            ArgumentError(
                "The number of training samples `n` has"*
                " to be greater than number of unique classes `nc`"
            )
        )
    end
    # Check output dimension default is min(p, nc-1)
    def_outdim = min(p, nc - 1)
    # If unset (0) use the default; otherwise try to use the provided one
    out_dim = ifelse(model.out_dim == 0, def_outdim, model.out_dim)
    # Check if the given one is sensible
    if out_dim > p
        throw(
            ArgumentError(
                "`out_dim` must not be larger than `p`"*
                "where `p` is the number of features in `X`"
            )
        )
    end
    return Xm_t, yplain, classes_seen, p, n, nc, nclasses, integers_seen, out_dim
end

function MMI.fitted_params(::LDA, (core_res, classes_seen))
    return (
        projected_class_means=MS.classmeans(core_res),
        projection_matrix=MS.projection(core_res)
    )
end

function MMI.predict(m::LDA, (core_res, classes_seen), Xnew)
    # projection of Xnew, XWt is nt x o  where o = number of out dims
    # nt = number ot test samples
    XWt = MMI.matrix(Xnew) * core_res.proj
    # centroids in the transformed space, nc x o
    centroids = transpose(core_res.pmeans)

    # compute the distances in the transformed space between pairs of rows
    # the probability matrix Pr is `n x nc` and normalised accross rows
    Pr = pairwise(m.dist, XWt, centroids, dims=1)
    Pr .= Pr .* -1
    # apply a softmax transformation
    softmax!(Pr)
    return MMI.UnivariateFinite(classes_seen, Pr)
end

metadata_model(LDA,
    input=Table(Continuous),
    target=AbstractVector{<:Finite},
    weights=false,
    output=Table(Continuous),
    descr=LDA_DESCR,
    path="$(PKG).LDA"
)

####
#### BayesianLDA
####

"""
    BayesianLDA(; kwargs...)

$BayesianLDA_DESCR

# Keyword Parameters

- `method::Symbol=:gevd`: choice of solver, one of `:gevd` or `:whiten` methods
- `cov_w::CovarianceEstimator=SimpleCovariance()`: an estimator for the within-class
    covariance (used in computing within-class scatter matrix, Sw), by default set to the
    standard `MultivariateStats.CovarianceEstimator` but could be set to any robust
    estimator from `CovarianceEstimation.jl`.
- `cov_b::CovarianceEstimator=SimpleCovariance()`: same as `cov_w` but for the
    between-class covariance(used in computing between-class scatter matrix, Sb).
- `out_dim::Int=0`: the output dimension, i.e dimension of the transformed space,
    automatically set if 0 is given (default).
- `regcoef::Float64=1e-6`: regularization coefficient (default value 1e-6). A positive
value `regcoef * eigmax(Sw)` where `Sw` is the within-class covariance estimator, is added
    to the diagonal of Sw to improve numerical stability. This can be useful if using the
    standard covariance estimator.
- `priors::Union{Nothing, Vector{Float64}}=nothing`: For use in prediction with Baye's rule. If `priors = nothing` then
    `priors` are estimated from the class proportions in the training data. Otherwise it
    requires a `Vector` containing class probabilities with probabilities specified using
    the order given by `levels(y)` where y is the target vector.

See also the [package documentation](
https://multivariatestatsjl.readthedocs.io/en/latest/lda.html).
For more information about the algorithm, see the paper by Li, Zhu and Ogihara,
[Using Discriminant Analysis for Multi-class Classification: An Experimental Investigation](
http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.89.7068&rep=rep1&type=pdf).
"""
@mlj_model mutable struct BayesianLDA <: MMI.Probabilistic
    method::Symbol = :gevd::(_ in (:gevd, :whiten))
    cov_w::CovarianceEstimator=MS.SimpleCovariance()
    cov_b::CovarianceEstimator=MS.SimpleCovariance()
    out_dim::Int=0::(_ ≥ 0)
    regcoef::Float64=1e-6::(_ ≥ 0)
    priors::Union{Nothing, Vector{Float64}}=nothing
end

function MMI.fit(model::BayesianLDA, ::Int, X, y)
    Xm_t, yplain, classes_seen, p, n, nc, nclasses, integers_seen, out_dim =
        _check_lda_data(model, X, y)
    ## If piors are specified check if they makes sense.
    ## This was put here to through errors much earlier
    if isa(model.priors, Vector)
        priors = _check_lda_priors(model.priors, nc, nclasses, integers_seen)
    end

    core_res = MS.fit(
        MS.MulticlassLDA, nc, Xm_t, Int.(yplain);
        method=model.method,
        outdim=out_dim,
        regcoef=model.regcoef,
        covestimator_within=model.cov_w,
        covestimator_between=model.cov_b
    )

    ## Estimates prior probabilities if specified by user.
    ## Put it here to avoid recomputing as the fitting process does this already
    if isa(model.priors, Nothing)
        weights = MS.classweights(core_res)
        total = core_res.stats.tweight
        priors = weights ./ total
    end
    cache     = nothing
    report    = (
        classes=classes_seen,
        out_dim=MS.outdim(core_res),
        class_means=MS.classmeans(core_res),
        mean=MS.mean(core_res),
        class_weights=MS.classweights(core_res),
        Sw=MS.withclass_scatter(core_res),
        Sb=MS.betweenclass_scatter(core_res),
        nc=nc
    )

    fitresult = (core_res, classes_seen, priors, n)
    return fitresult, cache, report
end

function _matrix_transpose(model::Union{LDA, BayesianLDA}, X)
    return MMI.matrix(X; transpose=true)
end

function _check_lda_priors(priors, nc, nclasses, integers_seen)
    if length(priors) != nclasses
        throw(ArgumentError("Invalid size of `priors`."))
    end
    if !isapprox(sum(priors), 1)
        throw(ArgumentError("probabilities specified in `priors` must sum to 1"))
    end
    if any(model.priors .< 0)
        throw(ArgumentError("probabilities specified in `priors` must non-negative"))
    end
    # Select priors for unique classes in `y` (For resampling purporses).
    priors_ = nc == nclasses ? model.priors : @view model.priors[integers_seen]
    return priors_
end

function MMI.fitted_params(::BayesianLDA, (core_res, classes_seen, priors, n))
   return (
       projected_class_means=MS.classmeans(core_res),
       projection_matrix=MS.projection(core_res),
       priors=priors
    )
end

function MMI.predict(m::BayesianLDA, (core_res, classes_seen, priors, n), Xnew)
     # projection of Xnew, XWt is nt x o  where o = number of out dims
    # nt = number ot test samples
    XWt = MMI.matrix(Xnew) * core_res.proj
    # centroids in the transformed space, nc x o
    centroids = transpose(core_res.pmeans)

    # The discriminant matrix `Pr` is of dimension `nt x nc`
    # Pr[i,k] = -0.5*(xᵢ −  µₖ)ᵀ(Σw⁻¹)(xᵢ −  µₖ) + log(priorsₖ) where (Σw = Sw/n)
    # In the transformed space this becomes
    # Pr[i,k] = -0.5*(Pᵀxᵢ −  Pᵀµₖ)ᵀ(PᵀΣw⁻¹P)(Pᵀxᵢ −  Pᵀµₖ) + log(priorsₖ)
    # But PᵀSw⁻¹P = I and PᵀΣw⁻¹P = n*I due to the nature of the projection_matrix, P
    # Giving Pr[i,k] = -0.5*n*(Pᵀxᵢ −  Pᵀµₖ)ᵀ(Pᵀxᵢ −  Pᵀµₖ) + log(priorsₖ)
    # with (Pᵀxᵢ −  Pᵀµₖ)ᵀ(Pᵀxᵢ −  Pᵀµₖ) being the SquaredEquclidean distance between
    # pairs of rows in the transformed space
    Pr = pairwise(SqEuclidean(), XWt, centroids, dims=1)
    Pr .*= (-0.5*n)
    Pr .+= log.(priors)'

    # apply a softmax transformation to convert Pr to a probability matrix
    softmax!(Pr)
    return MMI.UnivariateFinite(classes_seen, Pr)
end

function MMI.transform(m::T, (core_res,), X) where T<:Union{LDA, BayesianLDA}
    # projection of X, XWt is nt x o  where o = out dims
    proj = core_res.projw * core_res.projLDA #proj is the projection_matrix
    XWt = MMI.matrix(X) * proj
    return MMI.table(XWt, prototype = X)
end

metadata_model(
    BayesianLDA,
    input=Table(Continuous),
    target= AbstractVector{<:Finite},
    weights=false,
    output=Table(Continuous),
    descr=BayesianLDA_DESCR,
    path="$(PKG).BayesianLDA"
)

####
#### SubspaceLDA
####

"""
    SubspaceLDA(; kwargs...)

$SubspaceLDA_DESCR

# Keyword Parameters

- `normalize=true`: Option to normalize the between class variance for the number of
    observations in each class, one of `true` or `false`.
- `out_dim`: the dimension of the transformed space to be used by `predict` and
    `transform` methods, automatically set if 0 is given (default).
- `dist=SqEuclidean`: the distance metric to use when performing classification
    (to compare the distance between a new point and centroids in the transformed space),
    an alternative choice can be the `CosineDist`.

See also the [package documentation](
https://multivariatestatsjl.readthedocs.io/en/latest/lda.html).
For more information about the algorithm, see the paper by Howland & Park (2006),
"Generalizing discriminant analysis using the generalized singular value decomposition",
IEEE Trans. Patt. Anal. & Mach. Int., 26: 995-1006.
"""
@mlj_model mutable struct SubspaceLDA <: MMI.Probabilistic
    normalize::Bool = true
    out_dim::Int = 0::(_ ≥ 0)
    dist::SemiMetric = SqEuclidean()
end

function MMI.fit(model::SubspaceLDA, ::Int, X, y)
    Xm_t, yplain, classes_seen, p, n, nc, nclasses, integers_seen, out_dim =
        _check_lda_data(model, X, y)

    core_res = MS.fit(
        MS.SubspaceLDA, Xm_t, Int.(yplain), nc;
        normalize = model.normalize
    )
    # λ is a (nc -1) x 1 vector containing the eigen values sorted in descending order.
    λ = core_res.λ
    explained_variance_ratio = λ ./ sum(λ) #proportions of variance

    cache = nothing
    report = (
        explained_variance_ratio=explained_variance_ratio,
        classes=classes_seen,
        class_means=MS.classmeans(core_res),
        mean=MS.mean(core_res),
        class_weights=MS.classweights(core_res),
        nc=nc
    )
    fitresult = (core_res, out_dim, classes_seen)
    return fitresult, cache, report
end

function MMI.fitted_params(::SubspaceLDA, (core_res, _))
    return (class_means=MS.classmeans(core_res), projection_matrix=MS.projection(core_res))
end

function MMI.predict(m::SubspaceLDA, (core_res, out_dim, classes_seen), Xnew)
     # projection of Xnew, XWt is nt x o  where o = number of out dims
    # nt = number ot test samples
    proj = core_res.projw * view(core_res.projLDA, :, 1:out_dim) #proj is the projection_matrix
    XWt = MMI.matrix(Xnew) * proj
    # centroids in the transformed space, nc x o
    centroids = transpose(core_res.cmeans) * proj

    # compute the distances in the transformed space between pairs of rows
    # the probability matrix is `nt x nc` and normalised accross rows
    Pr = pairwise(m.dist, XWt, centroids, dims=1)
    Pr .= Pr .* -1
    # apply a softmax transformation
    softmax!(Pr)
    return MMI.UnivariateFinite(classes_seen, Pr)
end

metadata_model(
    SubspaceLDA,
    input=Table(Continuous),
    target=AbstractVector{<:Finite},
    weights=false,
    output=Table(Continuous),
    descr=SubspaceLDA_DESCR,
    path="$(PKG).SubspaceLDA"
)

####
#### BayesianSubspaceLDA
####

"""
    BayesianSubspaceLDA(; kwargs...)

$BayesianSubspaceLDA_DESCR

# Keyword Parameters

- `normalize::Bool=true`: Option to normalize the between class variance for the number of
    observations in each class, one of `true` or `false`.
- `out_dim::Int=0`: the dimension of the transformed space to be used by `predict` and
    `transform` methods, automatically set if 0 is given (default).
- `priors::Union{Nothing, Vector{Float64}}=nothing`: For use in prediction with Baye's
    rule. If `priors = nothing` then `priors` are estimated from the class proportions
    in the training data. Otherwise it requires a `Vector` containing class
    probabilities with probabilities specified using the order given by `levels(y)`
    where y is the target vector.

For more information about the algorithm, see the paper by Howland & Park (2006),
"Generalizing discriminant analysis using the generalized singular value decomposition"
,IEEE Trans. Patt. Anal. & Mach. Int., 26: 995-1006.
"""
@mlj_model mutable struct BayesianSubspaceLDA <: MMI.Probabilistic
    normalize::Bool=false
    out_dim::Int= 0::(_ ≥ 0)
    priors::Union{Nothing, Vector{Float64}}=nothing
end

function MMI.fit(model::BayesianSubspaceLDA, ::Int, X, y)
    Xm_t, yplain, classes_seen, p, n, nc, nclasses, integers_seen, out_dim =
        _check_lda_data(model, X, y)
    ## If piors are specified check if they makes sense.
    ## This was put here to through errors much earlier
    if isa(model.priors, Vector)
        priors = _check_lda_priors(model.priors, nc, nclasses, integers_seen)
    end

    core_res = MS.fit(
        MS.SubspaceLDA, Xm_t, Int.(yplain), nc;
        normalize = model.normalize
    )
    # λ is a (nc -1) x 1 vector containing the eigen values sorted in descending order.
    λ = core_res.λ
    explained_variance_ratio = λ ./ sum(λ) #proportions of variance
    mult = model.normalize ? n : 1 #used in prediction

    ## Estimates prior probabilities if specified by user.
    ## Put it here to avoid recomputing as the fitting process does this already
    if isa(model.priors, Nothing)
        weights = MS.classweights(core_res)
        priors = weights ./ n
    end

    cache = nothing
    report = (
        explained_variance_ratio=explained_variance_ratio,
        classes=classes_seen,
        class_means=MS.classmeans(core_res),
        mean=MS.mean(core_res),
        class_weights=MS.classweights(core_res),
        nc=nc
    )
    fitresult = (core_res, out_dim, classes_seen, priors, n, mult)
    return fitresult, cache, report
end

function _matrix_transpose(model::Union{SubspaceLDA, BayesianSubspaceLDA}, X)
    return transpose(MMI.matrix(X))
end

function MMI.fitted_params(::BayesianSubspaceLDA, (core_res, _, _, priors,_))
    return (
        projected_class_means=MS.classmeans(core_res),
        projection_matrix=MS.projection(core_res),
        priors=priors
    )
end

function MMI.predict(
    m::BayesianSubspaceLDA,
    (core_res, out_dim, classes_seen, priors, n, mult),
    Xnew
)
    # projection of Xnew, XWt is nt x o  where o = number of out dims
    # nt = number ot test samples
    #proj is the projection_matrix
    proj = core_res.projw * view(core_res.projLDA, :, 1:out_dim)
    XWt = MMI.matrix(Xnew) * proj

    # centroids in the transformed space, nc x o
    centroids = transpose(core_res.cmeans) * proj
    nc = length(classes_seen)

    # compute the distances in the transformed space between pairs of rows
    # The discriminant matrix `Pr` is of dimension `nt x nc`
    # Pr[i,k] = -0.5*(xᵢ −  µₖ)ᵀ(Σw⁻¹)(xᵢ −  µₖ) + log(priorsₖ) where Σw = Sw/(n-nc)
    # In the transformed space this becomes
    # Pr[i,k] = -0.5*(Pᵀxᵢ −  Pᵀµₖ)ᵀ(PᵀΣw⁻¹P)(Pᵀxᵢ −  Pᵀµₖ) + log(priorsₖ)
    # But PᵀSw⁻¹P = (1/mult)*I and PᵀΣw⁻¹P = (n-nc)/mult*I
    # Giving Pr[i,k] = -0.5*n*(Pᵀxᵢ −  Pᵀµₖ)ᵀ(Pᵀxᵢ −  Pᵀµₖ) + log(priorsₖ)
    # (Pᵀxᵢ −  Pᵀµₖ)ᵀ(Pᵀxᵢ −  Pᵀµₖ) is the SquaredEquclidean distance in the
    # transformed space
    Pr = pairwise(SqEuclidean(), XWt, centroids, dims=1)
    Pr .*= (-0.5 * (n-nc)/mult)
    Pr .+= log.(priors)'

    # apply a softmax transformation to convert Pr to a probability matrix
    softmax!(Pr)
    return MMI.UnivariateFinite(classes_seen, Pr)
end

function MMI.transform(m::T, (core_res, out_dim, _), X) where T<:Union{SubspaceLDA, BayesianSubspaceLDA}
    # projection of X, XWt is nt x o  where o = out dims
    proj = core_res.projw * view(core_res.projLDA, :, 1:out_dim) #proj is the projection_matrix
    XWt = MMI.matrix(X) * proj
    return MMI.table(XWt, prototype = X)
end

metadata_model(
    BayesianSubspaceLDA,
    input=Table(Continuous),
    target=AbstractVector{<:Finite},
    weights=false,
    output=Table(Continuous),
    descr=BayesianSubspaceLDA_DESCR,
    path="$(PKG).BayesianSubspaceLDA"
)


const iv = initvar

cf = Dict()

simpleloop = [
    :(x = 1/2*x)
    :(y = 2y)
]
cf[:simpleloop] = [
    "x(n) = (1/2)^n * $(iv(:x))"
    "y(n) = 2^n * $(iv(:y))"
]

cohencu = [
    :(m = m+1)
    :(x = x+y)
    :(y = y+z)
    :(z = z+6)
]
cf[:cohencu] = []

freire1 = [
    :(x = x-r)
    :(r = r+1)
]
cf[:freire1] = []

freire2 = [
    :(x = x-s)
    :(s = s+6*r+3)
    :(r = r+1)
]
cf[:freire2] = []

euclidex = [[
    :(a = a - b)
    :(p = p - q)
    :(r = r - s)
], [
    :(q = q - p)
    :(b = b - a)
    :(s = s - r)
]]
cf[:(euclidex[1])] = []
cf[:(euclidex[2])] = []

fermat = [[
    :(r = r - v)
    :(v = v + 2)
], [
    :(r = r + u)
    :(u = u + 2)
]]
cf[:(fermat[1])] = []
cf[:(fermat[2])] = []

wensley = [[
    :(b = b/2)
    :(d = d/2)
], [
    :(a = a+b)
    :(y = y+d/2)
    :(b = b/2)
    :(d = d/2)
]]
cf[:(wensley[1])] = []
cf[:(wensley[2])] = []

lcm = [[
    :(x = x - y)
    :(v = v + u)
], [
    :(y = y - x)
    :(u = u + v)
]]
cf[:(lcm[1])] = []
cf[:(lcm[2])] = []

knuth = [[
    :(t  = r)
    :(r  = 2*r-rp+q+d+2)
    :(rp = t)
    :(q  = q+4)
    :(d  = d+2)
], [
    :(t  = r)
    :(r  = 2*r-rp+q)
    :(rp = t)
    :(d  = d+2)
], [
    :(t  = r)
    :(r  = 2*r-rp+q-d-2)
    :(rp = t)
    :(q  = q-4)
    :(d  = d+2)
], [
    :(t  = r)
    :(r  = 2*r-rp+q-2*d-4)
    :(rp = t)
    :(q  = q-8)
    :(d  = d+2)
]]
cf[:(knuth[1])] = []
cf[:(knuth[2])] = []
cf[:(knuth[3])] = []
cf[:(knuth[4])] = []

mannadiv = [[
    :(y1 = y1 + 1)
    :(y2 = 0)
    :(y3 = y3 - 1)
], [
    :(y2 = y2 + 1)
    :(y3 = y3 - 1)
]]
cf[:(mannadiv[1])] = []
cf[:(mannadiv[2])] = []

divbin = [[
    :(x = 2*x)
    :(b = b/2)
], [
    :(b = b/2)
    :(x = 2x+1)
    :(r = r-b)
]]
cf[:(divbin[1])] = []
cf[:(divbin[2])] = []


"""
```
t = find_threshold(Moments(),  histogram, edges)
```

The following rule determines the threshold:  if one assigns all observations
below the threshold to a value z₀ and all observations above the threshold to a
value z₁, then the first three moments of the original histogram must match the
moments of this specially constructed bilevel histogram.

# Output

Returns a real number `t` that specifies the threshold.

# Arguments

The function arguments are described in more detail below.

##  `histogram`

An `AbstractArray` storing the frequency distribution.

##  `edges`

An `AbstractRange` specifying how the intervals for the frequency distribution
are divided.

# Example

Compute the threshold for the "camerman" image in the `TestImages` package.

```julia
using TestImages, ImageContrastAdjustment, HistogramThresholding

img = testimage("cameraman")
edges, counts = build_histogram(img,256)
#=
  The `counts` array stores at index 0 the frequencies that were below the
  first bin edge. Since we are seeking a threshold over the interval
  partitioned by `edges` we need to discard the first bin in `counts`
  so that the dimensions of `edges` and `counts` match.
=#
t = find_threshold(Moments(),counts[1:end], edges)
```

# Reference

[1] W.-H. Tsai, “Moment-preserving thresolding: A new approach,” Computer Vision, Graphics, and Image Processing, vol. 29, no. 3, pp. 377–393, Mar. 1985. [doi:10.1016/0734-189x(85)90133-1](https://doi.org/10.1016/0734-189x%2885%2990133-1)
"""
function find_threshold(algorithm::Moments, histogram::AbstractArray, edges::AbstractRange)
    n = sum(histogram)
    m₀ = 1.0
    m₁, m₂, m₃ =  compute_moments(histogram, edges, n)
    c₀, c₁ = compute_auxiliary_values(m₀, m₁, m₂, m₃)
    z₀, z₁ = compute_gray_levels(c₀, c₁)
    p₀, p₁ = compute_pixel_fractions(z₀, z₁, m₁)
    threshold_percentile = p₀
    target_count = ceil(Int,threshold_percentile * n)
    final_bin = determine_threshold(histogram, target_count)
    return edges[final_bin]
end

function determine_threshold(histogram, target_count)
    #= Finding the bin for which the bins below and including it contain p₀
     pixels. =#
    current_count = 0
    temp_bin = 0
    final_bin = 0
    holding_pixel_value = 0
    while current_count <= target_count
        temp_bin += 1
        current_count += histogram[temp_bin]
        if abs(target_count - current_count) <= abs(target_count - holding_pixel_value)
            holding_pixel_value = current_count
            final_bin = temp_bin
        end
    end
    final_bin
end

function compute_moments(histogram, edges, n)
    m₁ = 0.0
    m₂ = 0.0
    m₃ = 0.0
    # (2) Compute the moments from a histogram.
    for j in eachindex(histogram)
        m₁ += (histogram[j] / n) * edges[j]
        m₂ += (histogram[j] / n) * (edges[j])^2
        m₃ += (histogram[j] / n) * (edges[j])^3
    end
    m₁, m₂, m₃
end

function compute_auxiliary_values(m₀, m₁, m₂, m₃)
    # Appendix A.1.i calculates the auxiliary values c.
    cₐ = det([m₀ m₁;
              m₁ m₂])
    c₀ = (1/cₐ) * det([-m₂ m₁;
                       -m₃ m₂])
    c₁ = (1/cₐ) * det([m₀ -m₂;
                       m₁ -m₃])
    c₀, c₁
end

function compute_gray_levels(c₀, c₁)
    #= Appendix A.1.ii calculates grey values z which are the bi-level intensities
     associated with the hypothetical "unblurred" image. =#
    z₀ = (1/2) * ( -c₁ - sqrt(c₁^2 - 4*c₀))
    z₁ = (1/2) * ( -c₁ + sqrt(c₁^2 - 4*c₀))
    z₀, z₁
end

function compute_pixel_fractions(z₀, z₁, m₁)
    #= Appendix A.1.iii calculates the fraction of pixels below and above the
     threshold. =#
    pₐ = det([1 1;
              z₀ z₁])
    p₀ = (1/pₐ) * det([1 1;
                       m₁ z₁])
    p₁ = 1 - p₀
    p₀, p₁
end

function preserved_moments(p₀, z₀, p₁, z₁)
    # (3) Preserving the moments of the resulting image.
    m₁₀ = p₀ * (z₀)^0 + p₁ * (z₁)^0
    m₁₁ = p₀ * (z₀)^1 + p₁ * (z₁)^1
    m₁₂ = p₀ * (z₀)^2 + p₁ * (z₁)^2
    m₁₃ = p₀ * (z₀)^3 + p₁ * (z₁)^3
    m₁₀, m₁₁, m₁₂, m₁₃
end


"""
    mJy_to_W( c::Cosmology.AbstractCosmology, S::Unitful.AbstractQuantity, z::Real )

Converts synchrotron flux density `S` in `[mJy]` to `[W/Hz]` for a given redshift `z` and cosmology `c`.
Conserves unit information.
"""
function mJy_to_W( c::Cosmology.AbstractCosmology, S::Unitful.AbstractQuantity, z::Real )

    # Luminsity Distance in Mpc
    dL = luminosity_dist(c, z)

    # P(ν)= S(ν) * 4π * dL^2
    # dL is the luminosity distance
    return S * 4π * dL^2 |> u"W/Hz"
end

"""
    mJy_to_W( c::Cosmology.AbstractCosmology, S::Real, z::Real )

Converts synchrotron flux density `S` in `[mJy]` to `[W/Hz]` for a given redshift `z` and cosmology `c`.
Deletes unit information.
"""
mJy_to_W( c::Cosmology.AbstractCosmology, S::Real, z::Real ) = mJy_to_W(c, S * 1.0u"mJy", z) |> ustrip

"""
    mJy_to_W(S::Union{Real, Unitful.AbstractQuantity}, h::AbstractGadgetHeader)

Converts synchrotron flux density `S` in `[mJy]` to `[W/Hz]` for a given redshift and cosmology taken from `AbstractGadgetHeader`.
"""
mJy_to_W(S::Union{Real, Unitful.AbstractQuantity}, h::AbstractGadgetHeader) = mJy_to_W( cosmology(h), S, h.z )


### QuantumOscillator.jl
# Convenience functionality for handling finite-dimensional systems,
# specifically related to the (truncated) harmonic oscillator
###

"""
    osc(n::Int[, name="Osc(n)"::QName])

Create harmonic oscillator in Fock basis with `n` levels.

### Default ops for QFactor:
  - 'i' : identity operator
  - 'n' : number operator
  - 'd' : lowering operator
  - 'u' : raising operator
  - 'x' : in-phase quadrature
  - 'y' : out-of-phase quadrature

"""
function osc(levels::Int, name=""::QName)
    if name == ""
        name = "Osc($(levels))"
    end
    s = QFactor(levels, name)
    s.ops['d'] = sparse([x == y - 1 ? sqrt(QComp(x)) : QComp(0) for x=1:levels, y=1:levels])
    s.ops['u'] = s('d')'
    s.ops['n'] = let l=1:levels; sparse(l,l,map(QComp, 0:(levels - 1))) end
    s.ops['x'] = s('d') + s('u')
    s.ops['y'] = (s('d') - s('u')) .* im
    s
end

"""
    coherentvec(o::QObj, α::Number)::QKet

Create a coherent state ket vector with amplitude `α`.
"""
function coherentvec(o::QObj, α::Number)::QKet
  m = length(o)
  nbar = abs2(α)
  @assert (nbar + 3*sqrt(nbar) <= m) "Mean n of $nbar too large for max n of $m."
  e = exp(-nbar/2)
  cv = map(k -> QComp(e*α^k/sqrt(gamma(k+1))), 0:(m - 1))
  sparsevec(cv / norm(cv))
end

"""
    coherent(o::QObj, α::QComp)::QKet

Create a coherent state projection operator with amplitude `α`.
"""
coherent(o::QObj, α) = projector(coherentvec(o, α))


using CellListMap
using StaticArrays
import Random

#
# In this example we compute the histogram of distances, expected to follow the
# function f(f) = ρ(4/3)π(r[i+1]^3 - r[i]^3) with ρ being the density of the system.
#
function distance_histogram(;N=100_000,parallel=true,x=nothing)

    # Number of particles, sides and cutoff
    sides = SVector{3,Float64}(250, 250, 250)
    cutoff = 10.
    box = Box(sides, cutoff)

    # Particle positions
    Random.seed!(321)
    if x === nothing
        x = [ sides .* rand(SVector{3,Float64}) for i in 1:N ]
    end

    # Initialize auxiliary linked lists
    cl = CellList(x, box, parallel=parallel)

    # Function to be evalulated for each pair: build distance histogram
    function build_histogram!(d2, hist)
        d = sqrt(d2)
        ibin = floor(Int, d) + 1
        hist[ibin] += 1
        return hist
    end

    # Preallocate and initialize histogram
    hist = zeros(Int, 10)

    # Run pairwise computation
    map_pairwise!(
        (x, y, i, j, d2, hist) -> build_histogram!(d2, hist),
        hist, box, cl,
        parallel=parallel
    )

    # Normalization (convert to float)
    hist = (N / (N * (N - 1) / 2)) * hist

    return hist
end


"""
    BilinearUpsample(factor::Tuple{Integer,Integer})

Create an upsampling layer that uses bilinear interpolation.

The width and height dimensions grow by the `factor` tuple.

# Examples
```jldoctest; setup = :(using Flux: BilinearUpsample; using Random; Random.seed!(0))
julia> b = Flux.BilinearUpsample((2, 2))
BilinearUpsample(2, 2)

julia> b(rand(2, 2, 1, 1))
4×4×1×1 Array{Float64,4}:
[:, :, 1, 1] =
 0.823648  0.658877  0.329336  0.164566
 0.845325  0.675933  0.337149  0.167757
 0.888679  0.710044  0.352773  0.174138
 0.910357  0.7271    0.360586  0.177329```
"""
struct BilinearUpsample{T<:Integer}
    factor::Tuple{T,T}
end

function (b::BilinearUpsample)(x::AbstractArray)
    W, H, C, N = size(x)

    newW = W * b.factor[1]
    newH = H * b.factor[2]

    out = similar(x, (newW, newH, C, N))

    @inbounds for w = 1:newW, h = 1:newH
        w₀ = (w - 0.5) / b.factor[1] + 0.5
        h₀ = (h - 0.5) / b.factor[2] + 0.5

        w1 = floor(Int, w₀)
        w2 = w1 + 1
        h1 = floor(Int, h₀)
        h2 = h1 + 1

        i1 = clamp(w1, 1, W)
        i2 = clamp(w2, 1, W)
        j1 = clamp(h1, 1, H)
        j2 = clamp(h2, 1, H)

        @views out[w, h, :, :] =
            (
                x[i1, j1, :, :] * (w2 - w₀) * (h2 - h₀) +
                x[i1, j2, :, :] * (w2 - w₀) * (h₀ - h1) +
                x[i2, j1, :, :] * (w₀ - w1) * (h2 - h₀) +
                x[i2, j2, :, :] * (w₀ - w1) * (h₀ - h1)
            ) / ((w2 - w1) * (h2 - h1))
    end

    out
end

function Base.show(io::IO, b::BilinearUpsample)
    print(io, "BilinearUpsample(", b.factor[1], ", ", b.factor[2], ")")
end


using Kalman
using Base.Test
using StaticArrays
using LaTeXStrings
include("testsystem1.jl")
srand(11)
using Plots

Y, X = sample(20, 1, M)
Xhat = kalmanrts(Y, M)[2]
scatter(Y[:], label=L"Y (= X_1 + \epsilon = obs)")
plot!(reshape(X,2,20)', color = [:red :blue], label=[L"X_1" L"X_2"] )# label=[L"X_1", L"X_2" ])
plot!(reshape(Xhat,2,20)', line= :dot, color = [:red :blue], label=[L"X_1 (est.)" L"X_2 (est.)"] )# label=[L"X_1", L"X_2" ])

savefig("test/testplot.png")


include("../../HamiltonODE.jl")

using Plots
pyplot()

d = 2
m = (1)
q0 = [1.,0.]
p0 = [0.,1.]

F(q)= -q
U(q)= q⋅q
sol(q)=[cos(q),sin(q)]

HS = HamiltonSystem(d,m,q0,p0,F,U,sol)

T=2*pi
dt = 1e-1

t,Q,P=integrate(HS,Integrator(verlet_step,dt,T));

p1 = plot(t,Q')
p2 = plot(Q[1,:],Q[2,:])
plot(p1,p2)


push!(LOAD_PATH, "/Users/saydjari/Dropbox/GradSchool_AKS/Doug/Projects/DHC/main")
using DHC_2DUtils
using DHC_tests
using Profile
using BenchmarkTools
using FFTW
using HDF5
using Test
using SparseArrays
using Plots
theme(:dark)

function fink_filter_bank_csafe(c, L; nx=256, wd=1, pc=1, shift=false, Omega=false, safety_on=true)
    #c     - sets the scale sampling rate (1 is dyadic, 2 is half dyadic)
    #L     - number of angular bins (usually 8*pc or 16*pc)
    #wd    - width of the wavelets (default 1, wd=2 for a double covering)
    #pc    - plane coverage (default 1, full 2pi 2)
    #shift - shift in θ by 1/2 of the θ spacing
    #Omega - true= append Omega filter (all power beyond Nyquist) so the sum of filters is 1.0

    # -------- assertion errors to make sure arguments are reasonable
    @test wd <= L/2

    # -------- set parameters
    dθ   = pc*π/L
    θ_sh = shift ? dθ/2 : 0.0
    dx   = nx/2-1

    im_scale = convert(Int8,log2(nx))
    # -------- number of bins in radial direction (size scales)
    J = (im_scale-3)*c + 1

    # -------- allocate output array of zeros
    filt      = zeros(nx, nx, J*L+(Omega ? 2 : 1))
    psi_index = zeros(Int32, J, L)
    psi_ind_in= zeros(Int32, J*L+(Omega ? 2 : 1), 2)
    theta     = zeros(Float64, L)
    j_value   = zeros(Float64, J)

    # -------- compute the required wd
    j_rad_exp = zeros(J)
    for j_ind = 1:J
        j = (j_ind-1)/c
        jrad  = im_scale-j-2
        j_rad_exp[j_ind] = 2^(jrad)
    end

    wd_j = max.(ceil.(L./(pc.*π.*j_rad_exp)),wd)

    if !safety_on
        wd_j.=wd
    end

    # loop over wd from small to large
    ## there is some uneeded redundancy added by doing this esp in l loop
    for wd in sort(unique(wd_j))
        # -------- allocate theta and logr arrays
        θ    = zeros(nx, nx)
        logr = zeros(nx, nx)

        wdθ  = wd*dθ
        norm = 1.0/sqrt(wd)
        # -------- loop over l
        for l = 0:L-1
            θ_l        = dθ*l+θ_sh
            theta[l+1] = θ_l

        # -------- allocate anggood BitArray
            anggood = falses(nx, nx)

        # -------- loop over pixels
            for x = 1:nx
                sx = mod(x+dx,nx)-dx -1    # define sx,sy so that no fftshift() needed
                for y = 1:nx
                    sy = mod(y+dx,nx)-dx -1
                    θ_pix  = mod(atan(sy, sx)+π -θ_l, 2*π)
                    θ_good = abs(θ_pix-π) <= wdθ

                    # If this is a pixel we might use, calculate log2(r)
                    if θ_good
                        anggood[y, x] = θ_good
                        θ[y, x]       = θ_pix
                        r2            = sx^2 + sy^2
                        logr[y, x]    = 0.5*log2(max(1,r2))
                    end
                end
            end
            angmask = findall(anggood)
        # -------- compute the wavelet in the Fourier domain
        # -------- the angular factor is the same for all j
            F_angular = norm .* cos.((θ[angmask].-π).*(L/(2*wd*pc)))

        # -------- loop over j for the radial part
        #    for (j_ind, j) in enumerate(1/c:1/c:im_scale-2)
            j_ind_w_wd = findall(wd_j.==wd)
            for j_ind in j_ind_w_wd
                j = (j_ind-1)/c
                j_value[j_ind] = 1+j  # store for later
                jrad  = im_scale-j-2
                Δj    = abs.(logr[angmask].-jrad)
                rmask = (Δj .<= 1) #deprecating the 1/c to 1, constant width

        # -------- radial part
                F_radial = cos.(Δj[rmask] .* (π/2)) #deprecating c*π/2 to π/2
                ind      = angmask[rmask]
        #      Let's have these be (J,L) if you reshape...
        #        f_ind    = (j_ind-1)*L+l+1
                f_ind    = j_ind + l*J
                filt[ind, f_ind] = F_radial .* F_angular[rmask]
                psi_index[j_ind,l+1] = f_ind
                psi_ind_in[f_ind,:] = [j_ind-1,l]
            end
        end
    end

    # -------- phi contains power near k=0 not yet accounted for
    filter_power = (sum(filt.*filt, dims=3))[:,:,1]

    # -------- for plane half-covered (pc=1), add other half-plane
    if pc == 1
        filter_power .+= circshift(filter_power[end:-1:1,end:-1:1],(1,1))
    end

    # -------- compute power required to sum to 1.0
    i0 = round(Int16,nx/2-2)
    i1 = round(Int16,nx/2+4)
    center_power = 1.0 .- fftshift(filter_power)[i0:i1,i0:i1]
    zind = findall(center_power .< 1E-15)
    center_power[zind] .= 0.0  # set small numbers to zero
    phi_cen = zeros(nx, nx)
    phi_cen[i0:i1,i0:i1] = sqrt.(center_power)

    # -------- before adding ϕ to filter bank, renormalize ψ if pc=1
    if pc==1 filt .*= sqrt(2.0) end  # double power for half coverage

    # -------- add result to filter array
    phi_index  = J*L+1
    filt[:,:,phi_index] .= fftshift(phi_cen)
    psi_ind_in[phi_index,:] = [J,0]

    # -------- metadata dictionary
    info=Dict{String,Any}()
    info["npix"]         = nx
    info["j_value"]      = j_value
    info["theta_value"]  = theta
    info["psi_index"]    = psi_index
    info["phi_index"]    = phi_index
    info["J_L"]          = psi_ind_in
    info["pc"]           = pc
    info["wd"]           = wd_j
    info["fs_center_r"]  = j_rad_exp

    if Omega     # append a filter containing the rest (outside Nyquist)
        filter_power += filt[:,:,phi_index].^2
        edge_power    = 1.0 .- filter_power
        zind          = findall(edge_power .< 1E-15)
        edge_power[zind]     .= 0.0  # set small numbers to zero
        Omega_index           = J*L+2
        info["Omega_index"]   = Omega_index
        filt[:,:,Omega_index] = sqrt.(edge_power)
    end

    return filt, info
end

function fink_filter_hash_csafe(c, L; nx=256, wd=1, pc=1, shift=false, Omega=false, safety_on=true)
    # -------- compute the filter bank
    filt, hash = fink_filter_bank_csafe(c, L; nx=nx, wd=wd, pc=pc, shift=shift, Omega=Omega, safety_on=safety_on)

    # -------- list of non-zero pixels
    flist = fink_filter_list(filt)

    # -------- pack everything you need into the info structure
    hash["filt_index"] = flist[1]
    hash["filt_value"] = flist[2]

    # -------- compute matrix that projects iso coeffs, add to hash
    S1_iso_mat = S1_iso_matrix(hash)
    hash["S1_iso_mat"] = S1_iso_mat
    S2_iso_mat = S2_iso_matrix(hash)
    hash["S2_iso_mat"] = S2_iso_mat
    hash["num_iso_coeff"] = size(S1_iso_mat)[1] + size(S2_iso_mat)[1] + 2
    hash["num_coeff"] = size(S1_iso_mat)[2] + size(S2_iso_mat)[2] + 2

    return hash
end

fhash = fink_filter_hash_csafe(1,8,wd=2,pc=1)
filt_test = fink_filter_bank_csafe(1,8,wd=2,pc=1)[1]

plot_filter_bank_QA(filt_test, fhash; fname="/Users/saydjari/Dropbox/GradSchool_AKS/Doug/Projects/DHC/scratch_AKS/images/filter_bank_QA_csafe.png")

fhash = fink_filter_hash_csafe(2,8,wd=2,pc=1)
filt_test = fink_filter_bank_csafe(2,8,wd=2,pc=1)[1]

plot_filter_bank_QA(filt_test, fhash; fname="/Users/saydjari/Dropbox/GradSchool_AKS/Doug/Projects/DHC/scratch_AKS/images/filter_bank_QA_csafe_C2.png")

fhash = fink_filter_hash_csafe(2,8,wd=1,pc=1)

fhash = fink_filter_hash(2,8,wd=1,pc=1)

#In both formulations, only the smallest FS wavelet gets widened by our original criteria, even with c=2

fhash = fink_filter_hash_csafe(2,8,wd=1,pc=1)

function fink_filter_bank_csafe(c, L; nx=256, wd=1, pc=1, shift=false, Omega=false, safety_on=true, wd_cutoff=2)
    #c     - sets the scale sampling rate (1 is dyadic, 2 is half dyadic)
    #L     - number of angular bins (usually 8*pc or 16*pc)
    #wd    - width of the wavelets (default 1, wd=2 for a double covering)
    #pc    - plane coverage (default 1, full 2pi 2)
    #shift - shift in θ by 1/2 of the θ spacing
    #Omega - true= append Omega filter (all power beyond Nyquist) so the sum of filters is 1.0

    # -------- assertion errors to make sure arguments are reasonable
    @test wd <= L/2

    # -------- set parameters
    dθ   = pc*π/L
    θ_sh = shift ? dθ/2 : 0.0
    dx   = nx/2-1

    im_scale = convert(Int8,log2(nx))
    # -------- number of bins in radial direction (size scales)
    J = (im_scale-3)*c + 1
    normj = 1/sqrt(c)

    # -------- allocate output array of zeros
    filt      = zeros(nx, nx, J*L+(Omega ? 2 : 1))
    psi_index = zeros(Int32, J, L)
    psi_ind_in= zeros(Int32, J*L+(Omega ? 2 : 1), 2)
    theta     = zeros(Float64, L)
    j_value   = zeros(Float64, J)

    # -------- compute the required wd
    j_rad_exp = zeros(J)
    for j_ind = 1:J
        j = (j_ind-1)/c
        jrad  = im_scale-j-2
        j_rad_exp[j_ind] = 2^(jrad)
    end

    wd_j = max.(ceil.(wd_cutoff.*L./(pc.*π.*j_rad_exp)),wd)

    if !safety_on
        wd_j.=wd
    end

    # loop over wd from small to large
    ## there is some uneeded redundancy added by doing this esp in l loop
    for wd in sort(unique(wd_j))
        # -------- allocate theta and logr arrays
        θ    = zeros(nx, nx)
        logr = zeros(nx, nx)

        wdθ  = wd*dθ
        norm = 1.0/(sqrt(wd))
        # -------- loop over l
        for l = 0:L-1
            θ_l        = dθ*l+θ_sh
            theta[l+1] = θ_l

        # -------- allocate anggood BitArray
            anggood = falses(nx, nx)

        # -------- loop over pixels
            for x = 1:nx
                sx = mod(x+dx,nx)-dx -1    # define sx,sy so that no fftshift() needed
                for y = 1:nx
                    sy = mod(y+dx,nx)-dx -1
                    θ_pix  = mod(atan(sy, sx)+π -θ_l, 2*π)
                    θ_good = abs(θ_pix-π) <= wdθ

                    # If this is a pixel we might use, calculate log2(r)
                    if θ_good
                        anggood[y, x] = θ_good
                        θ[y, x]       = θ_pix
                        r2            = sx^2 + sy^2
                        logr[y, x]    = 0.5*log2(max(1,r2))
                    end
                end
            end
            angmask = findall(anggood)
        # -------- compute the wavelet in the Fourier domain
        # -------- the angular factor is the same for all j
            F_angular = norm .* cos.((θ[angmask].-π).*(L/(2*wd*pc)))

        # -------- loop over j for the radial part
        #    for (j_ind, j) in enumerate(1/c:1/c:im_scale-2)
            j_ind_w_wd = findall(wd_j.==wd)
            for j_ind in j_ind_w_wd
                j = (j_ind-1)/c
                j_value[j_ind] = 1+j  # store for later
                jrad  = im_scale-j-2
                Δj    = abs.(logr[angmask].-jrad)
                rmask = (Δj .<= 1) #deprecating the 1/c to 1, constant width

        # -------- radial part
                F_radial = normj .* cos.(Δj[rmask] .* (π/2)) #deprecating c*π/2 to π/2
                ind      = angmask[rmask]
        #      Let's have these be (J,L) if you reshape...
        #        f_ind    = (j_ind-1)*L+l+1
                f_ind    = j_ind + l*J
                filt[ind, f_ind] = F_radial .* F_angular[rmask]
                psi_index[j_ind,l+1] = f_ind
                psi_ind_in[f_ind,:] = [j_ind-1,l]
            end
        end
    end

    # -------- phi contains power near k=0 not yet accounted for
    filter_power = (sum(filt.*filt, dims=3))[:,:,1]

    # -------- for plane half-covered (pc=1), add other half-plane
    if pc == 1
        filter_power .+= circshift(filter_power[end:-1:1,end:-1:1],(1,1))
    end

    # -------- compute power required to sum to 1.0
    i0 = round(Int16,nx/2-2)
    i1 = round(Int16,nx/2+4)
    center_power = 1.0 .- fftshift(filter_power)[i0:i1,i0:i1]
    zind = findall(center_power .< 1E-15)
    center_power[zind] .= 0.0  # set small numbers to zero
    phi_cen = zeros(nx, nx)
    phi_cen[i0:i1,i0:i1] = sqrt.(center_power)

    # -------- before adding ϕ to filter bank, renormalize ψ if pc=1
    if pc==1 filt .*= sqrt(2.0) end  # double power for half coverage

    # -------- add result to filter array
    phi_index  = J*L+1
    filt[:,:,phi_index] .= fftshift(phi_cen)
    psi_ind_in[phi_index,:] = [J,0]

    # -------- metadata dictionary
    info=Dict{String,Any}()
    info["npix"]         = nx
    info["j_value"]      = j_value
    info["theta_value"]  = theta
    info["psi_index"]    = psi_index
    info["phi_index"]    = phi_index
    info["J_L"]          = psi_ind_in
    info["pc"]           = pc
    info["wd"]           = wd_j
    info["wd_cutoff"]    = wd_cutoff
    info["fs_center_r"]  = j_rad_exp

    if Omega     # append a filter containing the rest (outside Nyquist)
        filter_power += filt[:,:,phi_index].^2
        edge_power    = 1.0 .- filter_power
        zind          = findall(edge_power .< 1E-15)
        edge_power[zind]     .= 0.0  # set small numbers to zero
        Omega_index           = J*L+2
        info["Omega_index"]   = Omega_index
        filt[:,:,Omega_index] = sqrt.(edge_power)
    end

    return filt, info
end

fhash = fink_filter_hash_csafe(2,8,wd=2,pc=1)
filt_test = fink_filter_bank_csafe(2,8,wd=2,pc=1)[1]

plot_filter_bank_QA(filt_test, fhash; fname="/Users/saydjari/Dropbox/GradSchool_AKS/Doug/Projects/DHC/scratch_AKS/images/filter_bank_QA_csafe_C2_wide.png")

#Now, only the smallest FS wavelet with wd=2 is bumped up to 3... visually seems a good choice.

fhash = fink_filter_hash_csafe(1,8,wd=2,pc=1)
filt_test = fink_filter_bank_csafe(1,8,wd=2,pc=1)[1]

plot_filter_bank_QA(filt_test, fhash; fname="/Users/saydjari/Dropbox/GradSchool_AKS/Doug/Projects/DHC/scratch_AKS/images/filter_bank_QA_csafe_C1_wide.png")

fhash = fink_filter_hash_csafe(1,8,wd=1,pc=1)
filt_test = fink_filter_bank_csafe(1,8,wd=1,pc=1)[1]

plot_filter_bank_QA(filt_test, fhash; fname="/Users/saydjari/Dropbox/GradSchool_AKS/Doug/Projects/DHC/scratch_AKS/images/filter_bank_QA_csafe_wd1_C1_wide.png")

# with wd=1, smallest is widened to 3, next to 2, rest to 1

heatmap(fftshift(dropdims(sum(filt_test[:,:,:].^2,dims=3),dims=3)))

fhash = fink_filter_bank(3,16,wd=2,pc=1)

heatmap(fftshift(dropdims(sum(fhash[1].^2,dims=3),dims=3)))

fhash = fink_filter_bank_csafe(4,16,wd=2,pc=1)

heatmap(fftshift(dropdims(sum(fhash[1].^2,dims=3),dims=3)))

fhash = fink_filter_hash_csafe(1,8,wd=1,pc=1,safety_on=false)
filt_test = fink_filter_bank_csafe(1,8,wd=1,pc=1,safety_on=false)[1]

plot_filter_bank_QA(filt_test, fhash; fname="/Users/saydjari/Dropbox/GradSchool_AKS/Doug/Projects/DHC/scratch_AKS/images/filter_bank_QA_csafe_wd1_C1_wide_unsafe.png")

filt_test = fink_filter_bank_csafe(1,8,wd=1,pc=2,safety_on=false)[1]
heatmap(fftshift(dropdims(sum(filt_test[:,:,:].^2,dims=3),dims=3)))

filt_test = fink_filter_bank_csafe(2,8,wd=4,pc=2)[1]
heatmap(fftshift(dropdims(sum(filt_test[:,:,:].^2,dims=3),dims=3)))

filt_test = fink_filter_bank_csafe(3,8,wd=4,pc=2)[1]
heatmap(fftshift(dropdims(sum(filt_test[:,:,:].^2,dims=3),dims=3)))

function fink_filter_bank_csafe(c, L; nx=256, wd=1, pc=1, shift=false, Omega=false, safety_on=true, wd_cutoff=2)
    #c     - sets the scale sampling rate (1 is dyadic, 2 is half dyadic)
    #L     - number of angular bins (usually 8*pc or 16*pc)
    #wd    - width of the wavelets (default 1, wd=2 for a double covering)
    #pc    - plane coverage (default 1, full 2pi 2)
    #shift - shift in θ by 1/2 of the θ spacing
    #Omega - true= append Omega filter (all power beyond Nyquist) so the sum of filters is 1.0

    # -------- assertion errors to make sure arguments are reasonable
    @test wd <= L/2

    # -------- set parameters
    dθ   = pc*π/L
    θ_sh = shift ? dθ/2 : 0.0
    dx   = nx/2-1

    im_scale = convert(Int8,log2(nx))
    # -------- number of bins in radial direction (size scales)
    J = (im_scale-3)*c + 1
    normj = 1/sqrt(c)

    # -------- allocate output array of zeros
    filt      = zeros(nx, nx, J*L+(Omega ? 2 : 1))
    psi_index = zeros(Int32, J, L)
    psi_ind_in= zeros(Int32, J*L+(Omega ? 2 : 1), 2)
    theta     = zeros(Float64, L)
    j_value   = zeros(Float64, J)

    # -------- compute the required wd
    j_rad_exp = zeros(J)
    for j_ind = 1:J
        j = (j_ind-1)/c
        jrad  = im_scale-j-2
        j_rad_exp[j_ind] = 2^(jrad)
    end

    wd_j = max.(ceil.(wd_cutoff.*L./(pc.*π.*j_rad_exp)),wd)

    if !safety_on
        wd_j.=wd
    end

    # loop over wd from small to large
    ## there is some uneeded redundancy added by doing this esp in l loop
    for wd in sort(unique(wd_j))
        # -------- allocate theta and logr arrays
        θ    = zeros(nx, nx)
        logr = zeros(nx, nx)

        wdθ  = wd*dθ
        norm = 1.0/(sqrt(wd))
        # -------- loop over l
        for l = 0:L-1
            θ_l        = dθ*l+θ_sh
            theta[l+1] = θ_l

        # -------- allocate anggood BitArray
            anggood = falses(nx, nx)

        # -------- loop over pixels
            for x = 1:nx
                sx = mod(x+dx,nx)-dx -1    # define sx,sy so that no fftshift() needed
                for y = 1:nx
                    sy = mod(y+dx,nx)-dx -1
                    θ_pix  = mod(atan(sy, sx)+π -θ_l, 2*π)
                    θ_good = abs(θ_pix-π) <= wdθ

                    # If this is a pixel we might use, calculate log2(r)
                    if θ_good
                        anggood[y, x] = θ_good
                        θ[y, x]       = θ_pix
                        r2            = sx^2 + sy^2
                        logr[y, x]    = 0.5*log2(max(1,r2))
                    end
                end
            end
            angmask = findall(anggood)
        # -------- compute the wavelet in the Fourier domain
        # -------- the angular factor is the same for all j
            F_angular = norm .* cos.((θ[angmask].-π).*(L/(2*wd*pc)))

        # -------- loop over j for the radial part
        #    for (j_ind, j) in enumerate(1/c:1/c:im_scale-2)
            j_ind_w_wd = findall(wd_j.==wd)
            for j_ind in j_ind_w_wd
                j = (j_ind-1)/c
                j_value[j_ind] = 1+j  # store for later
                jrad  = im_scale-j-2
                Δj    = abs.(logr[angmask].-jrad)
                rmask = (Δj .<= 1) #deprecating the 1/c to 1, constant width

        # -------- radial part
                F_radial = normj .* cos.(Δj[rmask] .* (π/2)) #deprecating c*π/2 to π/2
                ind      = angmask[rmask]
        #      Let's have these be (J,L) if you reshape...
        #        f_ind    = (j_ind-1)*L+l+1
                f_ind    = j_ind + l*J
                filt[ind, f_ind] = F_radial .* F_angular[rmask]
                psi_index[j_ind,l+1] = f_ind
                psi_ind_in[f_ind,:] = [j_ind-1,l]
            end
        end
    end

    # -------- phi contains power near k=0 not yet accounted for
    filter_power = (sum(filt.*filt, dims=3))[:,:,1]

    # -------- for plane half-covered (pc=1), add other half-plane
    if pc == 1
        filter_power .+= circshift(filter_power[end:-1:1,end:-1:1],(1,1))
    end

    # -------- compute power required to sum to 1.0
    i0 = round(Int16,nx/2-2)
    i1 = round(Int16,nx/2+4)
    center_power = 1.0 .- fftshift(filter_power)[i0:i1,i0:i1]
    zind = findall(center_power .< 1E-15)
    center_power[zind] .= 0.0  # set small numbers to zero
    phi_cen = zeros(nx, nx)
    phi_cen[i0:i1,i0:i1] = sqrt.(center_power)

    # -------- before adding ϕ to filter bank, renormalize ψ if pc=1
    if pc==1 filt .*= sqrt(2.0) end  # double power for half coverage

    # -------- add result to filter array
    phi_index  = J*L+1
    filt[:,:,phi_index] .= fftshift(phi_cen)
    psi_ind_in[phi_index,:] = [J,0]

    # -------- metadata dictionary
    info=Dict{String,Any}()
    info["npix"]         = nx
    info["j_value"]      = j_value
    info["theta_value"]  = theta
    info["psi_index"]    = psi_index
    info["phi_index"]    = phi_index
    info["J_L"]          = psi_ind_in
    info["pc"]           = pc
    info["wd"]           = wd_j
    info["wd_cutoff"]    = wd_cutoff
    info["fs_center_r"]  = j_rad_exp

    if Omega     # append a filter containing the rest (outside Nyquist)
        filter_power += filt[:,:,phi_index].^2
        edge_power    = 1.0 .- filter_power
        zind          = findall(edge_power .< 1E-15)
        edge_power[zind]     .= 0.0  # set small numbers to zero
        Omega_index           = J*L+2
        info["Omega_index"]   = Omega_index
        filt[:,:,Omega_index] = sqrt.(edge_power)
    end

    return filt, info
end

filt_test = fink_filter_bank_csafe(1,8,wd=1,pc=1)[1]
heatmap(fftshift(dropdims(sum(filt_test[:,:,:].^2,dims=3),dims=3)))

filter_power = (sum(filt.*filt, dims=3))[:,:,1]

# -------- for plane half-covered (pc=1), add other half-plane
if pc == 1
    filter_power .+= circshift(filter_power[end:-1:1,end:-1:1],(1,1))
end


w11 = [0.1, 6.8, -3.5, 2.0, 4.1, 3.1, -0.8, 0.9, 5.0, 3.9];
w12 = [1.1, 7.1, -4.1, 2.7, 2.8, 5.0, -1.3, 1.2, 6.4, 4.0];
w21 = [7.1, -1.4, 4.5, 6.3, 4.2, 1.4, 2.4, 2.5, 8.4, 4.1];
w22 = [4.2, -4.3, 0.0, 1.6, 1.9, -3.2, -4.0, -6.1, 3.7, -2.2];
w31 = [-3.0, 0.5, 2.9, -0.1, -4.0, -1.3, -3.4, -4.1, -5.1, 1.9];
w32 = [-2.9, 8.7, 2.1, 5.2, 2.2, 3.7, 6.2, 3.4, 1.6, 5.1];
w41 = [2.0, 8.9, 4.2, 8.5, 6.7, 0.5, 5.3, 8.7, 7.1, 8.0] * -1;
w42 = [8.4, -0.2, 7.7, 3.2, 4.0, 9.2, 6.7, 6.4, 9.7, 6.3] * -1;
const SAMPLE = [w11 w12 w21 w22 w31 w32 w41 w42];

function normalize_data(positive, negative)
    " normalization adds additional one column to data "
    normalize = data -> [data ones(size(data)[1])]

    positive = normalize(positive)
    negative = normalize(negative) * -1
    assert(size(positive) == size(negative))
    data = transpose([positive; negative])

    d, n = size(data)
    weight = zeros(d)

    data, weight
end

function pseudo_inverse(positive, negative)
    data, weight = normalize_data(positive, negative)
    d, n = size(data)

    weight = pinv(data * transpose(data)) * data * ones(n)
end

" least mean square method to computer weight vector "
function lms(positive, negative; eta=1, margin=1, theta=0.01)
    gradient(y) = (1 - dot(y, weight)) / dot(y, y) * y
    loss(weight) = sum([1 - dot(data[:, i], weight) for i = 1:n])

    data, weight = normalize_data(positive, negative)
    d, n = size(data)
    weight = [0.08276694629366121,0.12681539660924512,0.4783531495651045]

    function compute_delta(data, step, weight, eta, margin)
        i = (step % n + 1)
        for i = 1:n
            delta = eta / i * gradient(data[:, i])
            if dot(delta, delta) >= theta^2
                return delta
            end
        end
        nothing
    end

    step, delta = 1, compute_delta(data, 1, weight, eta, margin)
    while delta != nothing
        println("$step,$(loss(weight))")

        weight += delta
        step += 1
        delta = compute_delta(data, step, weight, eta, margin)
    end

    weight, step
end

function single_sample_relax(positive, negative; eta=1, margin=1, epsilon=0.00001)
    data, weight = normalize_data(positive, negative)
    d, n = size(data)

    function find_error(data, weight, eta, margin)
        for i = 1:n
            if dot(weight, data[:, i]) <= margin - epsilon
                return data[:, i]
            end
        end
        nothing
    end

    step, y = 1, find_error(data, weight, eta, margin)
    while y != nothing
        gradient = (margin - dot(weight, y)) / dot(y, y) * y
        weight += eta * gradient

        step += 1
        y = find_error(data, weight, eta, margin)
    end

    weight, step
end

function relax_procedure(positive, negative; eta=1, margin=1, epsilon=0.00001)
    data, weight = normalize_data(positive, negative)
    d, n = size(data)

    println("data = $data")

    function compute_error(data, weight, eta, margin)
        error_samples = [y for y in filter([data[:, i] for i = 1:n]) do y
            dot(y, weight) <= margin - epsilon
        end]
    end

    step, error_samples = 1, compute_error(data, weight, eta, margin)
    while length(error_samples) > 0
        println("== $(length(error_samples)) ==========================================")
        println("error_samples = $error_samples")
        println("weight = $weight")

        gradient = sum(error_samples) do y
            #println(y);
            #println(margin - dot(weight, y) / dot(y, y))
            (margin - dot(weight, y)) / dot(y, y) * y
        end

        weight += eta * gradient
        println("new weight $weight\t<= $gradient")

        step += 1
        error_samples = compute_error(data, weight, eta, margin)
    end

    weight, step
end

function batch_perception(positive, negative)
    data, weight = normalize_data(positive, negative)
    d, n = size(data)

    function compute_error(data, weight)
        error_samples = [y for y in filter([data[:, i] for i = 1:n]) do y
            dot(weight, y) <= 0
        end]
    end

    step, error_samples = 1, compute_error(data, weight)

    while length(error_samples) > 0
        gradient = sum(error_samples)
        weight += gradient
        #println("new weight $weight\t<= $gradient")

        step += 1
        error_samples = compute_error(data, weight)
    end

    weight, step
end

"""
Write a program to implement the “batch perception” algorithm
(see page 44 or 45 in PPT).

(a). Starting with a = 0, apply your program to the training data from 1 and 2.
Note that the number of iterations required for convergence.

(b). Apply your program to the training data from 3 and 2.
Again, note that the number of iterations required for convergence.

(c). Explain the difference between the iterations required in the above two cases.

写一个基于松驰准则的感知函数学习的代码，并采用 w1 和 w2 的数据进行训练。对程序运算过程和结果进行分析。

写一个最小平方准则的线性判别函数学习的代码，并分别采用 w1 和 w2、w2 和 w3、w3 和 w4 的数据进行训练。
"""
function hw2_1()
    weight, step = batch_perception(SAMPLE[:, 1:2], SAMPLE[:, 3:4])
    println("batch perception w1 and w2: iteration steps: $step, final weights: $weight")
    weight, step = batch_perception(SAMPLE[:, 3:4], SAMPLE[:, 5:6])
    println("batch perception w2 and w3: iteration steps: $step, final weights: $weight")

    weight, step = single_sample_relax(SAMPLE[:, 1:2], SAMPLE[:, 3:4])
    println("single sample relax w1 and w2: iteration steps: $step, final weights: $weight")

    weight = pseudo_inverse(SAMPLE[:, 1:2], SAMPLE[:, 3:4])
    println("pseudo_inverse MSE w1 and w2: final weights: $weight")
    weight = pseudo_inverse(SAMPLE[:, 3:4], SAMPLE[:, 5:6])
    println("pseudo_inverse MSE w2 and w3: final weights: $weight")
    weight = pseudo_inverse(SAMPLE[:, 5:6], SAMPLE[:, 7:8])
    println("pseudo_inverse MSE w3 and w4: final weights: $weight")

end

if !isinteractive()
    hw2_1()
end


#
include("mesh.jl")
include("melem.jl")

function buildmatrices_k0(msh::mesh, conn::Connectivity)
    #=
    Builds the matrix in full format (because I wanted to use the exact EV
    solver provided by Arpack).
    =#
    e2dof = conn.e2dof
    ndof  = conn.ndof
    # 1st: preallocate
    A = zeros(Float64, ndof, ndof)
    B = zeros(Float64, ndof, ndof)
    # 2nd: main loop
    for e=1:length(msh.elt)
        # @inbounds removes the check on the bounds of the array
        @inbounds verts  = msh.vtx[:, msh.elt[e]] # get the vertices
        # Choose if you want to compute the elementary matrix by projecting on
        # ∇P1 or on ∇P2 (for example for comparison with the RT0 :) )
        AE, BE = melemk0P1(verts)
        # AE, BE = melemk0P2(verts)
        nloc = length(e2dof[e])
        for i = 1:nloc
            @inbounds tmpI = e2dof[e][i]
            absI, sgnI = abs(tmpI), sign(tmpI)
            for j=1:nloc
                @inbounds tmpJ = e2dof[e][j]
                absJ, sgnJ = abs(tmpJ), sign(tmpJ)
                @inbounds A[absI, absJ] += sgnI*sgnJ*(AE[i,j] + BE[i,j])
                @inbounds B[absI, absJ] += sgnI*sgnJ*BE[i,j]
            end
        end
    end
    return A, B
end


# Find the Most Frequent Words with Mismatches in a String

const NUCLEOTIDES = ('A', 'C', 'G', 'T')
const NUCLEOTIDE_ORDER = Dict{Char, Int64}('A' => 0, 'C' => 1, 'G' => 2, 'T' => 3)

convert_to_intlist(line) = map(x -> parse(Int, x), split(line))

pattern_to_number(pattern) = foldl((acc, nucleotide) -> 4 * acc + NUCLEOTIDE_ORDER[nucleotide], pattern; init=0)

function number_to_pattern(encoding, k)
    pattern = Char[]
    for _ in 1:k
        encoding, remainder = divrem(encoding, 4)
        push!(pattern, NUCLEOTIDES[remainder+1])
    end
    join(reverse(pattern))
end

function calc_frequencies_for_approximate_occurrences(text, k, d)
    frequencies = zeros(Int32, 4 ^ k)
    for ix in 1:(length(text)-k+1)
        exact_pattern = text[ix:ix+k-1]
        for pattern in generate_neighbourhood(exact_pattern, d)
            code = pattern_to_number(pattern)
            frequencies[code+1] += 1
        end
    end
    frequencies
end

function calc_immediate_neighbours(pattern)
    neighbours = Set{String}()
    for (ix, letter) in enumerate(pattern)
        for nucleotide in NUCLEOTIDES
            if nucleotide != letter
                pattern_array = collect(pattern)
                pattern_array[ix] = nucleotide
                push!(neighbours, join(pattern_array))
            end
        end
    end
    neighbours
end

function generate_neighbourhood(text, d)
    neighbourhood = Set{String}([text])
    for _ in 1:d
        neighbours_of_neighbours = Set{String}()
        for pattern in neighbourhood
            immediate_neighbours = calc_immediate_neighbours(pattern)
            union!(neighbours_of_neighbours, immediate_neighbours)
        end
        union!(neighbourhood, neighbours_of_neighbours)
    end
    neighbourhood
end

function most_frequent_approximate_k_mers(text, k, d)
    frequencies = calc_frequencies_for_approximate_occurrences(text, k, d)
    max_occurrence = maximum(frequencies)
    patterns = String[]
    for (ix, freq) in enumerate(frequencies)
        if freq == max_occurrence
            pattern = number_to_pattern(ix - 1, k)
            push!(patterns, pattern)
        end
    end
    patterns
end

function main()
    text = readline()
    k, d = convert_to_intlist(readline())
    result = most_frequent_approximate_k_mers(text, k, d)
    println(join(result, " "))
end

main()


export newtoncombusca
export tentacholesky

"""
newtoncombusca(nlp, options...)

This package implements the idea of find the minimum τ > 0 such that
∇²f(xₖ)+τ I is positive definite.

Options:
- atol: absolute tolerance for the first order condition (default: 1e-6)
- rtol: relative tolerance for the first order condition (default: 1e-6)
- max_eval: maximum number of [functions] evaluations, use ≤ 0 for unlimited (default: 1000)
- max_iter: maximum number of iterations, use ≤ 0 for unlimited (default: 0)
- max_time: maximum elapsed time in seconds, use ≤ 0 for unlimited (default: 10)

Disclaimers for the developer:
  - nlp should be the only mandatory argument
  - these five options are the current default for other JSO-compliant solvers
  - always return a GenericExecutionStats
  """
  function newtoncombusca(
    nlp::AbstractNLPModel;
    atol::Real = 1e-6,
    rtol::Real = 1e-6,
    max_eval::Int = 1000,
    max_iter::Int = 0,
    max_time::Float64 = 10.0
    )

    if !unconstrained(nlp)
      error("Problem is not unconstrained")
    end

    x = copy(nlp.meta.x0)

    f(x) = obj(nlp, x)
    ∇f(x) = grad(nlp, x)
    H(x) = Symmetric(hess(nlp, x), :L)
    fx = f(x)
    ∇fx = ∇f(x)

    ϵ = atol + rtol * norm(∇fx)
    t₀ = time()

    x⁺ = similar(x) #estava faltando isso!

    iter = 0
    Δt = time() - t₀
    solved = norm(∇fx) < ϵ # First order stationary
    tired = neval_obj(nlp) ≥ max_eval > 0||
    iter ≥ max_iter > 0 || Δt ≥ max_time > 0 # Excess time, iteration, evaluations

    α = 1.0
    η = 1e-2
    Β = 1.0e-3

    status = :unknown

    # log_header is up for some rewrite in the future. For now, it simply prints the column names with some spacing
    @info log_header(
    [:iter, :fx, :ngx, :nf, :Δt],
    [Int, Float64, Float64, Int, Float64],
    hdr_override=Dict(:fx => "f(x)", :ngx => "‖∇f(x)‖", :nf => "#f")
    )
    # log_row uses the type information of each value, thus we use `Any` here.
    @info log_row(
    Any[iter, fx, norm(∇fx), neval_obj(nlp), Δt]
    )

    while !(solved || tired)
      h = H(x)
      F, status = tentacholesky(h, Β, status)
      if status != :unknown #small_step
        break
      end

      d = -(F \ ∇fx)
      slope = dot(d, ∇fx)

      # Armijo
      α = 1.0
      x⁺ = x + α * d
      f⁺ = f(x⁺)
      while f⁺ ≥ fx + η * α * slope
        α = α / 2
        x⁺ = x + α * d
        f⁺ = f(x⁺)
        if α < 1e-8
          status =:small_step
          break
        end
      end
      if status != :unknown #small_step
        break
      end
      x .= x⁺
      fx = f⁺
      ∇fx = ∇f(x)
      iter += 1

      solved = norm(∇fx) < ϵ # First order stationary

      Δt = time() - t₀
      tired = neval_obj(nlp) ≥ max_eval > 0||
      iter ≥ max_iter > 0 || Δt ≥ max_time > 0 # Excess time, iteration, evaluations

      @info log_row(
      Any[iter, fx, norm(∇fx), neval_obj(nlp), Δt]
      )
    end

    if solved
      status = :first_order
    elseif tired
      if neval_obj(nlp) ≥ max_eval > 0
        status = :max_eval
      elseif iter ≥ max_iter > 0
        status = :max_iter
      elseif Δt ≥ max_time > 0
        status = :max_time
      end
    end

    return GenericExecutionStats(
    status,
    nlp,
    solution=x,
    objective=fx,
    dual_feas=norm(∇fx),
    elapsed_time=Δt,
    iter=iter
    )
  end

  function tentacholesky(h, Β, status)
    F = copy(h)
    fatorou=:inicio
    ρ = minimum(diag(h))
    if  ρ > 0
      τ = 0.0
    else
      τ = -ρ + Β
    end
    k = 0
    while fatorou!=:fatorou
      F = cholesky(h, check = false)
      if !issuccess(F)
        τ = max(2τ, Β)
        h = h+τ*I
        k +=1
        if k > 250
          status=:user
          break
        end
      else
        fatorou =:fatorou
      end
    end
    return F, status
  end


using AbstractOperators
using ProximalOperators

A1 = [ 1.0  -2.0   3.0  -4.0  5.0;
       2.0  -1.0   0.0  -1.0  3.0]
A2 = [-1.0   0.0   4.0  -3.0  2.0;
      -1.0  -1.0  -1.0   1.0  3.0]
A = [A1; A2]
opA = VCAT(MatrixOp(A1), MatrixOp(A2))

b1 = [1.0, 2.0]
b2 = [3.0, 4.0]
b = [b1; b2]

f = SeparableSum(Translate(SqrNormL2(), -b1), Translate(SqrNormL2(), -b2))
lam = 0.1*norm(A'*b, Inf)
g = NormL1(lam)

x_star = [-3.877278911564627e-01, 0, 0, 2.174149659863943e-02, 6.168435374149660e-01]

# Nonfast/Nonadaptive

x0 = zero(x_star)
@time it, x, sol = ProximalAlgorithms.FBS(x0; fq=f, Aq=opA, g=g, gamma=1.0/opnorm(A)^2)
@test norm(x .- x_star, Inf) <= 1e-4
@test it < 150
println(sol)

# Nonfast/Adaptive

x0 = zero(x_star)
@time it, x, sol = ProximalAlgorithms.FBS(x0; fq=f, Aq=opA, g=g, adaptive=true)
@test norm(x .- x_star, Inf) <= 1e-4
@test it < 300
println(sol)

# Fast/Nonadaptive

x0 = zero(x_star)
@time it, x, sol = ProximalAlgorithms.FBS(x0; fq=f, Aq=opA, g=g, gamma=1.0/opnorm(A)^2, fast=true)
@test norm(x .- x_star, Inf) <= 1e-4
@test it < 100
println(sol)

# Fast/Adaptive

x0 = zero(x_star)
@time it, x, sol = ProximalAlgorithms.FBS(x0; fq=f, Aq=opA, g=g, adaptive=true, fast=true)
@test norm(x .- x_star, Inf) <= 1e-4
@test it < 200
println(sol)

# ZeroFPR/Nonadaptive

x0 = zero(x_star)
@time it, x, sol = ProximalAlgorithms.ZeroFPR(x0; fq=f, Aq=opA, g=g, gamma=1.0/opnorm(A)^2)
@test norm(x - x_star, Inf) <= 1e-4
@test it < 15
println(sol)

# ZeroFPR/Adaptive

x0 = zero(x_star)
@time it, x, sol = ProximalAlgorithms.ZeroFPR(x0; fq=f, Aq=opA, g=g, adaptive=true)
@test norm(x - x_star, Inf) <= 1e-4
@test it < 20

# PANOC/Nonadaptive

x0 = zero(x_star)
@time it, x, sol = ProximalAlgorithms.PANOC(x0; fq=f, Aq=opA, g=g, gamma=1.0/opnorm(A)^2)
@test norm(x - x_star, Inf) <= 1e-4
@test it < 20
println(sol)

# PANOC/Adaptive

x0 = zero(x_star)
@time it, x, sol = ProximalAlgorithms.PANOC(x0; fq=f, Aq=opA, g=g, adaptive=true)
@test norm(x - x_star, Inf) <= 1e-4
@test it < 20
println(sol)


function FixJanFirst(t,data)
    for yr in Dates.year(t[1]):Dates.year(t[end])
        try
            Jan1st = findin(t.==Date(yr,1,1),true)[1]
            adjacentValues = data[[-1 1]+Jan1st]
            if maximum(adjacentValues)>0 && data[Jan1st]==0
                data[Jan1st] = mean(adjacentValues)
            end
        catch
        end
    end
    return data
end

function FixMissedKwhReading(dataSample)
    ## ASSIGN GLOBAL SCOPE TO RETURN RESULT
    global dataVector
    ## MAKE VECTOR OUT OF DATA TO CAPTURE MISSED VALUES CLOSE TO 00:00
    dataVector = dataSample[:]
    ## IDENTIFY WHERE TO PERFORM MISSED VALUE TEST (STARTING POINT FOR TEST IS A ZERO VALUE)
    indZero = findall(dataVector.==0)
    indZero = indZero[findall(indZero.>=2)]
    indZero = indZero[findall(indZero.<=(length(dataVector)-2))]
    ## LOOP THROUGH ALL ZERO VALUES TO PERFORM TEST AT EACH LOCATION
    for iZ in indZero
        ## ASSIGN GLOBAL SCOPE TO RETURN RESULT
        global dataVector
        ## PERFORN TEST: CHECK TO SEE IF THE FOLLOWING DATAPOINT (n+1) IS LARGER THAT ⨯1.5 THE PRECEDING POINT (n-1) OR SUBSEQUENT POINT (n+2)
        if dataVector[iZ+1]>1.5*(maximum([dataVector[iZ-1] dataVector[iZ+2]]))
            dataVector[iZ] = Statistics.mean([dataVector[iZ-1] dataVector[iZ+2]])
            dataVector[iZ+1] = Statistics.mean([dataVector[iZ-1] dataVector[iZ+2]])
        end
    end
    dataSample = reshape(dataVector,size(dataSample,1),size(dataSample,2))
    return dataSample
end


module MultiCellMapsTests

using Test
using Gridap
import Gridap: ∇

u1fun(x) = x[1] + x[2]
u2fun(x) = x[1] - x[2]

u1fun_grad(x) = VectorValue(1.0,1.0)
u2fun_grad(x) = VectorValue(1.0,-1.0)

∇(::typeof(u1fun)) = u1fun_grad
∇(::typeof(u2fun)) = u2fun_grad

b1fun(x) = u2fun(x)
b2fun(x) = 0.0

model = CartesianDiscreteModel(domain=(0.0,1.0,0.0,1.0), partition=(4,4))

order = 1
diritag = "boundary"
fespace = H1ConformingFESpace(Float64,model,order,diritag)

V1 = TestFESpace(fespace)
V2 = V1

V = [V1,V2]
U = V

trian = Triangulation(model)
quad = CellQuadrature(trian,degree=2)

b1field = CellField(trian,b1fun)
b2field = CellField(trian,b2fun)

function a(v,u)
  a11 = varinner(∇(v[1]), ∇(u[1]))
  a12 = varinner(v[1],u[2])
  a22 = varinner(∇(v[2]), ∇(u[2]))
  return CellMap[a11,a12,a22], [(1,1),(1,2),(2,2)]
end

function b(v)
  b1 = varinner(v[1],b1field)
  b2 = varinner(v[2],b2field)
  return CellMap[b1,b2], [(1,),(2,)]
end

v = [ CellBasis(Vi) for Vi in V ]
u = [ CellBasis(Ui) for Ui in U ]

blocks, fieldids = a(v,u)

mcm = MultiCellMap(blocks,fieldids)

mca = integrate(mcm,trian,quad)

@test isa(mca,MultiCellArray{Float64,2})

u1 = CellField(trian,u1fun)
u2 = CellField(trian,u2fun)
u = [u1,u2]

blocks, fieldids = a(v,u)

fieldids = [ (fi[1],) for fi in fieldids ]

mcm = MultiCellMap(blocks,fieldids)
mcm1 = mcm

mca = integrate(mcm,trian,quad)

@test isa(mca,MultiCellArray{Float64,1})

blocks, fieldids = b(v)

mcm = MultiCellMap(blocks,fieldids)
mcm2 = mcm

mca = integrate(mcm,trian,quad)

@test isa(mca,MultiCellArray{Float64,1})

mcm = mcm1 + mcm2
mca = integrate(mcm,trian,quad)
@test isa(mca,MultiCellArray{Float64,1})

mcm = mcm1 - mcm2
mca = integrate(mcm,trian,quad)
@test isa(mca,MultiCellArray{Float64,1})

end


#=
  Cantor distribution
  from Foundations of Probabilistic Programming,
  page 5
  """
  Finally, the program in Fig. 1.4 generates a real number between [0, 1] whose
  expansion in base 3 does not contain any 1’s. This program is not like the others
  in that it does not halt (nor is it meant to). The program generates a sample from
  a curious and in many respects counterintuitive distribution called the Cantor
  distribution. It cannot be described using discrete probability distributions (i.e.
  finite or countable weighted sums of point masses), although the program only
  uses a discrete fair coin as a source. The Cantor distribution is also an example of
  continuous probability distribution, which assigns probability zero to every element
  of the state space. It is also an example of a so-called singular distribution, since
  it can be shown that the set of all its possible outcomes—that is to say the set of
  all real numbers whose base-3 expansion contains no 1’s—has measure 0 in the
  Lebesgue measure on [0, 1]
  """
=#

using Turing, StatsPlots, DataFrames
include("../jl_utils.jl")

# Plain simulation
function cantor_plain()
     x = 0
     d = 1
     i = 0
     while i < 10  # true
         d = d / 3
         x = x + 2*rand(DiscreteUniform(0,1))*d
         println("d:$d x:$x")
         i+=1
     end
end

# cantor_plain()

# Using Turing
@model cantor() = begin
    x ~ Uniform(0,1)
    d ~ Uniform(0,1)
    d = 1
    i = 0
    while i < 100  # true
        d /= 3
        t ~ DiscreteUniform(0,1)
        x += 2*t*d
        # println("d:$d x:$x")
        i += 1
    end
    dfinal ~ Uniform(0,1)
    dfinal = d
end

model = cantor()

num_chains = 4
# chains = sample(model, MH(), MCMCThreads(), 1000, num_chains)
chains = sample(model, MH(), 100)

display(chains)
show_var_dist_pct(chains, :d,20)
show_var_dist_pct(chains, :dfinal,20)
# show_var_dist_pct(chains, :y)
# show_var_dist_pct(chains, :u,20)
# show_var_dist_pct(chains, :v,20)


struct UnionFind
    par::Vector{Int}
    size::Vector{Int}
    UnionFind(n) = new(collect(1:n), ones(Int, n))
end

root!(uf, x) = uf.par[x] == x ? x : uf.par[x] = root!(uf, uf.par[x])

same(uf, x, y) = root!(uf, x) == root!(uf, y)

function unite!(uf, x, y)
    x = root!(uf, x)
    y = root!(uf, y)
    x == y && return false

    if uf.size[x] > uf.size[y]
        uf.par[y] = x
        uf.size[x] += uf.size[y]
    else
        uf.par[x] = y
        uf.size[y] += uf.size[x]
    end
    return true
end
size!(uf, x) = uf.size[root!(uf, x)]

struct Edge
    s::Int
    t::Int
    c::Int
end

import Base: isless
isless(e₁::Edge, e₂::Edge) = isless(e₁.c, e₂.c) # For sorting

function solve(n, m)
    edges = [Edge((parse.(Int, split(readline())))...) for _ = 1:m]

    sort!(edges)
    uf = UnionFind(n)
    cnt_edges = 0
    ans = 0
    for e ∈ edges
        if !same(uf, e.s, e.t)
            unite!(uf, e.s, e.t)
            cnt_edges += 1
        end

        if cnt_edges == (n + 1) ÷ 2
            ans = e.c
            break
        end
    end

    println(ans)
end

function main()
    while true
        n, m = parse.(Int, split(readline()))
        if n == 0 && m == 0
            break
        else
            solve(n, m)
        end
    end
end

main()


# Paulis
@test X * Id == X
@test Id * X == X
@test Y * Id == Y
@test Z * Id == Z

@test X * X == Id
@test Y * Y == Id
@test Z * Z == Id

@test X * Y == im∘Z
@test Y * Z == im∘X
@test Z * X == im∘Y
@test Y * X == -im∘Z

@test Id < X
@test X < Y
@test Y < Z

@test II < IX
@test IX < XX
@test XI < XX
@test IX < IY
@test XI < YI
@test XX < YY
@test XX < XY
@test YX > XX

# generators
@test generators(X) == [X]
@test generators(Z) == [Z]
@test generators(Y) == [im∘X, Z]

@test generators(XX) == [XI, IX]
@test generators(ZZ) == [ZI, IZ]
@test generators(XY) == [im∘XI, IX, IZ]
@test generators(YX) == [im∘XI, ZI, IX]
@test generators(YY) == [-XI, ZI, IX, IZ]

XII = kron(X,Id,Id)
IXI = kron(Id,X,Id)
IIX = kron(Id,Id,X)
ZII = kron(Z,Id,Id)
IZI = kron(Id,Z,Id)
IIZ = kron(Id,Id,Z)

@test generators(kron(X,X,X)) == [XII, IXI, IIX]
@test generators(kron(Z,Z,Z)) == [ZII, IZI, IIZ]
@test generators(kron(X,Y,Z)) == [im∘XII, IXI, IZI, IIZ]

@test Pauli([1 0; 0 1]) == Id
@test Pauli([0 1; 1 0]) == X
@test Pauli([0 -im; im 0]) == Y
@test Pauli([1 0; 0 -1]) == Z

@test complex(paulieye(2)) == eye(4)

# Factors
@test factor(XII)  == [ X,Id,Id]
@test factor(XII*IZI)  == [ X,Z,Id]
@test factor(-XII)  == [-X,Id,Id]
@test factor(-XII*IZI)  == [-X,Z,Id]
@test factor(-IXI)  == [-Id,X,Id]
@test factor(-IXI*IIZ)  == [-Id,X,Z]
@test factor(im∘XII)  == [im∘X,Id,Id]
@test factor(im∘XII*IZI)  == [im∘X,Z,Id]
@test factor(im∘IXI)  == [im∘Id,X,Id]
@test factor(im∘IXI*IIZ)  == [im∘Id,X,Z]

# Weight
for i=1:20
  v = rand(10)
  vX = (v .> .25) .* (v .< .5)
  vY = 2*(v .> .5) .* (v .< .75)
  vZ = 3*(v .> .75)
  r = rand()
  rS = (r > .25) + (r > .5) + (r > .75)
  @test weight(Pauli(vX + vY + vZ,rS)) == sum(v .> .25)
end

# Matrix conversions
@test complex(X) == [0 1; 1 0]
@test complex(Y) == [0 -1im; 1im 0]
@test complex(Z) == [1 0; 0 -1]

# Matrix arthimetic
@test iszero(X - [0 1; 1 0])
@test iszero(Y - [0 -1im;1im 0])
@test iszero(Z - [1 0; 0 -1])

# weight returns Int
@test isa(weight(Pauli(1)), Int)

@testset "Construct from matrix" begin
    xmat = Complex{Float64}[4.702883518586134e-7 + 0.0im 0.9999991609182552 + 0.0im; 0.9999990516115099 + 0.0im 5.793263216649176e-7 + 0.0im]
    @test Pauli(xmat) == X
    ymat = -im * Complex{Float64}[7.6121754928998e-7 + 0.0im 5.577787991615943e-7 - 0.9999984033382637im; 2.066866029117562e-7 + 0.9999984033382637im 8.745277847991395e-8 + 0.0im]
    @test Pauli(ymat) == -im ∘ Y
end


# SimLynx.jl Repeatable Process-Based Discrete-Event Simulation Example

using SimLynx
SimLynx.greet()

using Distributions: Exponential, Uniform
using Random

const N_TELLERS = 2
const MEAN_INTERARRIVAL_TIME = 4.0
const MIN_SERVICE_TIME = 2.0
const MAX_SERVICE_TIME = 10.0

const GENERATOR_RNG = 1
const CUSTOMER_RNG = 2

tellers = nothing

"Process to generate n customers arriving into the system."
@process generator(n::Integer) begin
    rng = MersenneTwister(seed(GENERATOR_RNG))
    dist = Exponential(MEAN_INTERARRIVAL_TIME)
    for i = 1:n
        @schedule now customer(i)
        work(rand(rng, dist))
    end
end

"The ith customer into the system."
@process customer(i::Integer) begin
    rng = MersenneTwister(seed(CUSTOMER_RNG, i))
    dist = Uniform(MIN_SERVICE_TIME, MAX_SERVICE_TIME)
    @with_resource tellers begin
        work(rand(rng, dist))
    end
end

"Run the simulation for n customers."
function run_simulation(n::Integer)
    println("SimLynx.jl Repeatable Process-Based Discrete-Event Simulation Example")
    println("$N_TELLERS Teller, Single Queue Bank Model")
    println("  Inter-arrival time = Exponential($MEAN_INTERARRIVAL_TIME)")
    println("  Service time = Uniform($MIN_SERVICE_TIME, $MAX_SERVICE_TIME)")
    println("  Number of customers = $n")
    @simulation begin
        global tellers = Resource(N_TELLERS, "tellers")
        @schedule at 0.0 generator(n)
        start_simulation()
        println("--- Simulation results after $(current_time()) minutes ---")
        print_stats(tellers.allocated, title="Teller Allocation Statistics")
        print_stats(tellers.queue_length, title="Teller Queue Length Statistics")
        plot_history(tellers.queue_length, title="Teller Queue Length History")
        print_stats(tellers.wait, title="Teller Queue Wait Time Statistics")
        plot_history(tellers.wait, title="Teller Queue Wait Time History")
    end
end

run_simulation(100)


using Parameters
using Blink
using Interact
using Plots
gr()

include("lv_model.jl")

function iso1(u2, p)
    @unpack α12, K1 = p
    return K1 - α12 * u2 / K1
end

function iso2(u2, p)
    @unpack α21, K2 = p
    return (K2 - u2) / α21
end

let
    u2s = range(0, 5, length = 100)
    ui = @manipulate for α12 in slider(0.1:0.01:1.0; label = "α12", value = 0.8),
                         α21 in slider(0.1:0.01:1.0; label = "α21", value = 0.3),
                         K1 in slider(0.1:0.01:2.0; label = "K1", value = 1.5),
                         K2 in slider(0.1:0.01:2.0; label = "K2", value = 1.0)
        p = LVPar(α12 = α12, α21 = α21, K1 = K1, K2 = K2)

        plot(u2s, [iso1(u2, p) for u2 in u2s], label = "u1 = 0")
        plot!(u2s, [iso2(u2, p) for u2 in u2s], label = "u2 = 0")
        xlims!(0, 3)
        ylims!(0, 5)
        xlabel!("u2")
        ylabel!("u1")
    end

    w = Blink.AtomShell.Window()
    body!(w, ui)
end