"""
BigFloat operations modifying a buffer. The first argument is overwritten by the result.
All operations of MPRF returning a BigFloat are simulated.
According to the MPFR C-functions, the result variable may be identical to one of the arguments.
"""

module MPFR_MODIFY

export
    dot!, rotate!, neg!,
    mul!, add!, sub!, dv!, pow!, ceil!, copysign!, div!,
    exp!, exp2!, exponent!, factorial!, floor!, fma!, fms!, hypot!,
    ldexp!, log!, log2!, log10!, max!, min!, mod!, modf!,
    nextfloat!, prevfloat!, rem!, rem2pi!, round!,
    sqrt!, trunc!, exp10!, expm1!,
    gamma!, lgamma!, log1p!,
    sin!, cos!, tan!, sec!, csc!, cot!, acos!, asin!, atan!,
    cosh!, sinh!, tanh!, sech!, csch!, coth!, acosh!, asinh!, atanh!, atan2!,
    cbrt!, significand, frexp!

import Base: broadcast!, copy!, sum!, A_mul_B!
import Base.GMP: ClongMax, CulongMax, CdoubleMax, Limb
import Base.MPFR.ROUNDING_MODE

# Basic arithmetic without promotion
for (fJ, fC) in ((:add!,:add), (:mul!,:mul))
    @eval begin
        # BigFloat
        function ($fJ)(z::BigFloat, x::BigFloat, y::BigFloat)
            ccall(($(string(:mpfr_,fC)),:libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Ptr{BigFloat}, Int32), &z, &x, &y, ROUNDING_MODE[])
            return z
        end

        # Unsigned Integer
        function ($fJ)(z::BigFloat, x::BigFloat, c::CulongMax)
            ccall(($(string(:mpfr_,fC,:_ui)), :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Culong, Int32), &z, &x, c, ROUNDING_MODE[])
            return z
        end
        ($fJ)(z::BigFloat, c::CulongMax, x::BigFloat) = ($fJ)(z,x,c)

        # Signed Integer
        function ($fJ)(z::BigFloat, x::BigFloat, c::ClongMax)
            ccall(($(string(:mpfr_,fC,:_si)), :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Clong, Int32), &z, &x, c, ROUNDING_MODE[])
            return z
        end
        ($fJ)(z::BigFloat, c::ClongMax, x::BigFloat) = ($fJ)(z,x,c)

        # Float32/Float64
        function ($fJ)(z::BigFloat, x::BigFloat, c::CdoubleMax)
            ccall(($(string(:mpfr_,fC,:_d)), :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Cdouble, Int32), &z, &x, c, ROUNDING_MODE[])
            return z
        end
        ($fJ)(z::BigFloat, c::CdoubleMax, x::BigFloat) = ($fJ)(z,x,c)

        # BigInt
        function ($fJ)(z::BigFloat, x::BigFloat, c::BigInt)
            ccall(($(string(:mpfr_,fC,:_z)), :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Ptr{BigInt}, Int32), &z, &x, &c, ROUNDING_MODE[])
            return z
        end
        ($fJ)(z::BigFloat, c::BigInt, x::BigFloat) = ($fJ)(z,x,c)
    end
end

for (fJ, fC) in ((:sub!,:sub), (:dv!,:div))
    @eval begin
        # BigFloat
        function ($fJ)(z::BigFloat, x::BigFloat, y::BigFloat)
            ccall(($(string(:mpfr_,fC)),:libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Ptr{BigFloat}, Int32), &z, &x, &y, ROUNDING_MODE[])
            return z
        end

        # Unsigned Int
        function ($fJ)(z::BigFloat, x::BigFloat, c::CulongMax)
            ccall(($(string(:mpfr_,fC,:_ui)), :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Culong, Int32), &z, &x, c, ROUNDING_MODE[])
            return z
        end
        function ($fJ)(z::BigFloat, c::CulongMax, x::BigFloat)
            ccall(($(string(:mpfr_,:ui_,fC)), :libmpfr), Int32, (Ptr{BigFloat}, Culong, Ptr{BigFloat}, Int32), &z, c, &x, ROUNDING_MODE[])
            return z
        end

        # Signed Integer
        function ($fJ)(z::BigFloat, x::BigFloat, c::ClongMax)
            ccall(($(string(:mpfr_,fC,:_si)), :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Clong, Int32), &z, &x, c, ROUNDING_MODE[])
            return z
        end
        function ($fJ)(z::BigFloat, c::ClongMax, x::BigFloat)
            ccall(($(string(:mpfr_,:si_,fC)), :libmpfr), Int32, (Ptr{BigFloat}, Clong, Ptr{BigFloat}, Int32), &z, c, &x, ROUNDING_MODE[])
            return z
        end

        # Float32/Float64
        function ($fJ)(z::BigFloat, x::BigFloat, c::CdoubleMax)
            ccall(($(string(:mpfr_,fC,:_d)), :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Cdouble, Int32), &z, &x, c, ROUNDING_MODE[])
            return z
        end
        function ($fJ)(z::BigFloat, c::CdoubleMax, x::BigFloat)
            ccall(($(string(:mpfr_,:d_,fC)), :libmpfr), Int32, (Ptr{BigFloat}, Cdouble, Ptr{BigFloat}, Int32), &z, c, &x, ROUNDING_MODE[])
            return z
        end

        # BigInt
        function ($fJ)(z::BigFloat, x::BigFloat, c::BigInt)
            ccall(($(string(:mpfr_,fC,:_z)), :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Ptr{BigInt}, Int32), &z, &x, &c, ROUNDING_MODE[])
            return z
        end
        # no :mpfr_z_div function
    end
end

function sub!(z::BigFloat, c::BigInt, x::BigFloat)
    ccall((:mpfr_z_sub, :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigInt}, Ptr{BigFloat}, Int32), &z, &c, &x, ROUNDING_MODE[])
    return z
end

function fma!(r::BigFloat, x::BigFloat, y::BigFloat, z::BigFloat)
    ccall(("mpfr_fma",:libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Ptr{BigFloat}, Ptr{BigFloat}, Int32), &r, &x, &y, &z, ROUNDING_MODE[])
    return r
end

function fms!(r::BigFloat, x::BigFloat, y::BigFloat, z::BigFloat)
    ccall(("mpfr_fms",:libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Ptr{BigFloat}, Ptr{BigFloat}, Int32), &r, &x, &y, &z, ROUNDING_MODE[])
    return r
end

# div
# BigFloat
function div!(z::BigFloat, x::BigFloat, y::BigFloat)
    ccall((:mpfr_div,:libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Ptr{BigFloat}, Int32), &z, &x, &y, to_mpfr(RoundToZero))
    ccall((:mpfr_trunc, :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}), &z, &z)
    return z
end

# Unsigned Int
function div!(z::BigFloat, x::BigFloat, c::CulongMax)
    ccall((:mpfr_div_ui, :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Culong, Int32), &z, &x, c, to_mpfr(RoundToZero))
    ccall((:mpfr_trunc, :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}), &z, &z)
    return z
end
function div!(z::BigFloat, c::CulongMax, x::BigFloat)
    ccall((:mpfr_ui_div, :libmpfr), Int32, (Ptr{BigFloat}, Culong, Ptr{BigFloat}, Int32), &z, c, &x, to_mpfr(RoundToZero))
    ccall((:mpfr_trunc, :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}), &z, &z)
    return z
end

# Signed Integer
function div!(z::BigFloat, x::BigFloat, c::ClongMax)
    ccall((:mpfr_div_si, :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Clong, Int32), &z, &x, c, to_mpfr(RoundToZero))
    ccall((:mpfr_trunc, :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}), &z, &z)
    return z
end
function div!(z::BigFloat, c::ClongMax, x::BigFloat)
    ccall((:mpfr_si_div, :libmpfr), Int32, (Ptr{BigFloat}, Clong, Ptr{BigFloat}, Int32), &z, c, &x, to_mpfr(RoundToZero))
    ccall((:mpfr_trunc, :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}), &z, &z)
    return z
end

# Float32/Float64
function div!(z::BigFloat, x::BigFloat, c::CdoubleMax)
    ccall((:mpfr_div_d, :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Cdouble, Int32), &z, &x, c, to_mpfr(RoundToZero))
    ccall((:mpfr_trunc, :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}), &z, &z)
    return z
end
function div!(z::BigFloat, c::CdoubleMax, x::BigFloat)
    ccall((:mpfr_d_div, :libmpfr), Int32, (Ptr{BigFloat}, Cdouble, Ptr{BigFloat}, Int32), &z, c, &x, to_mpfr(RoundToZero))
    ccall((:mpfr_trunc, :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}), &z, &z)
    return z
end

# BigInt
function div!(z::BigFloat, x::BigFloat, c::BigInt)
    ccall((:mpfr_div_z, :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Ptr{BigInt}, Int32), &z, &x, &c, to_mpfr(RoundToZero))
    ccall((:mpfr_trunc, :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}), &z, &z)
    return z
end

# More efficient commutative operations
for (fJ, fC, fI) in ((:add!, :add, 0), (:mul!, :mul, 1))
    @eval begin
        function ($fJ)(z::BigFloat, a::BigFloat, b::BigFloat, c::BigFloat)
            ccall(($(string(:mpfr_,fC)), :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Ptr{BigFloat}, Int32), &z, &a, &b, ROUNDING_MODE[])
            ccall(($(string(:mpfr_,fC)), :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Ptr{BigFloat}, Int32), &z, &z, &c, ROUNDING_MODE[])
            return z
        end
        function ($fJ)(z::BigFloat, a::BigFloat, b::BigFloat, c::BigFloat, d::BigFloat)
            ccall(($(string(:mpfr_,fC)), :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Ptr{BigFloat}, Int32), &z, &a, &b, ROUNDING_MODE[])
            ccall(($(string(:mpfr_,fC)), :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Ptr{BigFloat}, Int32), &z, &z, &c, ROUNDING_MODE[])
            ccall(($(string(:mpfr_,fC)), :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Ptr{BigFloat}, Int32), &z, &z, &d, ROUNDING_MODE[])
            return z
        end
        function ($fJ)(z::BigFloat, a::BigFloat, b::BigFloat, c::BigFloat, d::BigFloat, e::BigFloat)
            ccall(($(string(:mpfr_,fC)), :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Ptr{BigFloat}, Int32), &z, &a, &b, ROUNDING_MODE[])
            ccall(($(string(:mpfr_,fC)), :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Ptr{BigFloat}, Int32), &z, &z, &c, ROUNDING_MODE[])
            ccall(($(string(:mpfr_,fC)), :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Ptr{BigFloat}, Int32), &z, &z, &d, ROUNDING_MODE[])
            ccall(($(string(:mpfr_,fC)), :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Ptr{BigFloat}, Int32), &z, &z, &e, ROUNDING_MODE[])
            return z
        end
    end
end

function neg!(z::BigFloat)
    ccall((:mpfr_neg, :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Int32), &z, &z, ROUNDING_MODE[])
    return z
end

function neg!(z::BigFloat, x::BigFloat)
    ccall((:mpfr_neg, :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Int32), &z, &x, ROUNDING_MODE[])
    return z
end

function sqrt!(z::BigFloat, x::BigFloat)
    isnan(x) && return x
    ccall((:mpfr_sqrt, :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Int32), &z, &x, ROUNDING_MODE[])
    if isnan(z)
        throw(DomainError())
    end
    return z
end

sqrt!(z::BigFloat, x::BigInt) = sqrt!(z, BigFloat(x))

function pow!(z::BigFloat, x::BigFloat, y::BigFloat)
    ccall((:mpfr_pow, :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Ptr{BigFloat}, Int32), &z, &x, &y, ROUNDING_MODE[])
    return z
end

function pow!(z::BigFloat, x::BigFloat, y::CulongMax)
    ccall((:mpfr_pow_ui, :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Culong, Int32), &z, &x, y, ROUNDING_MODE[])
    return z
end

function pow!(z::BigFloat, x::BigFloat, y::ClongMax)
    ccall((:mpfr_pow_si, :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Clong, Int32), &z, &x, y, ROUNDING_MODE[])
    return z
end

function pow!(z::BigFloat, x::BigFloat, y::BigInt)
    ccall((:mpfr_pow_z, :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Ptr{BigInt}, Int32), &z, &x, &y, ROUNDING_MODE[])
    return z
end

pow!(z::BigFloat, x::BigFloat, y::Integer)  = typemin(Clong)  <= y <= typemax(Clong)  ? pow!(z, x, Clong(y))  : pow!(z, x, BigInt(y))

pow!(z::BigFloat, x::BigFloat, y::Unsigned) = typemin(Culong) <= y <= typemax(Culong) ? pow!(z, x, Culong(y)) : pow!(z, x, BigInt(y))

for f in (:exp, :exp2, :exp10, :expm1, :cosh, :sinh, :tanh, :sech, :csch, :coth, :cbrt)
@eval function $(Symbol(f, "!"))(z::BigFloat, x::BigFloat)
        ccall(($(string(:mpfr_,f)), :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Int32), &z, &x, ROUNDING_MODE[])
        return z
    end
end

# return log(2)
function big_ln2!(c::BigFloat)
    ccall((:mpfr_const_log2, :libmpfr), Cint, (Ptr{BigFloat}, Int32),
          &c, MPFR.ROUNDING_MODE[])
    return c
end

function ldexp!(z::BigFloat, x::BigFloat, n::Clong)
    ccall((:mpfr_mul_2si, :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Clong, Int32), &z, &x, n, ROUNDING_MODE[])
    return z
end
function ldexp!(z::BigFloat, x::BigFloat, n::Culong)
    ccall((:mpfr_mul_2ui, :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Culong, Int32), &z, &x, n, ROUNDING_MODE[])
    return z
end
ldexp!(z::BigFloat, x::BigFloat, n::ClongMax) = ldexp!(z, x, convert(Clong, n))
ldexp!(z::BigFloat, x::BigFloat, n::CulongMax) = ldexp!(z, x, convert(Culong, n))
ldexp!(z::BigFloat, x::BigFloat, n::Integer) = mul!(z, x, exp2!(BigFloat(n)))

function factorial!(z::BigFloat, x::BigFloat)
    if x < 0 || !isinteger(x)
        throw(DomainError())
    end
    ui = convert(Culong, x)
    ccall((:mpfr_fac_ui, :libmpfr), Int32, (Ptr{BigFloat}, Culong, Int32), &z, ui, ROUNDING_MODE[])
    return z
end

function hypot!(z::BigFloat, x::BigFloat, y::BigFloat)
    ccall((:mpfr_hypot, :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Ptr{BigFloat}, Int32), &z, &x, &y, ROUNDING_MODE[])
    return z
end

for f in (:log, :log2, :log10)
  @eval function $(Symbol(f, "!"))(z::BigFloat, x::BigFloat)
        if x < 0
            throw(DomainError())
        end
        ccall(($(string(:mpfr_,f)), :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Int32), &z, &x, ROUNDING_MODE[])
        return z
    end
end

function log1p!(z::BigFloat, x::BigFloat)
    if x < -1
        throw(DomainError())
    end
    ccall((:mpfr_log1p, :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Int32), &z, &x, ROUNDING_MODE[])
    return z
end

function max!(z::BigFloat, x::BigFloat, y::BigFloat)
    isnan(x) && return x
    isnan(y) && return y
    ccall((:mpfr_max, :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Ptr{BigFloat}, Int32), &z, &x, &y, ROUNDING_MODE[])
    return z
end

function min!(z::BigFloat, x::BigFloat, y::BigFloat)
    isnan(x) && return x
    isnan(y) && return y
    ccall((:mpfr_min, :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Ptr{BigFloat}, Int32), &z, &x, &y, ROUNDING_MODE[])
    return z
end

function modf!(zint::BigFloat, zfloat::BigFloat, x::BigFloat)
  if isinf(x)
    copy!(zint, x)
    copy!(zfloat, BigFloat(NaN))
  end
  ccall((:mpfr_modf, :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Ptr{BigFloat}, Int32), &zint, &zfloat, &x, ROUNDING_MODE[])
end

function rem!(z::BigFloat, x::BigFloat, y::BigFloat)
    ccall((:mpfr_fmod, :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Ptr{BigFloat}, Int32), &z, &x, &y, ROUNDING_MODE[])
    return z
end

function rem!(z::BigFloat, x::BigFloat, y::BigFloat, ::RoundingMode{:Nearest})
    ccall((:mpfr_remainder, :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Ptr{BigFloat}, Int32), &z, &x, &y, ROUNDING_MODE[])
    return z
end

# TODO: use a higher-precision BigFloat(pi) here?
function rem2pi!(z::BigFloat, x::BigFloat, r::RoundingMode)
  mul!(z, BigFloat(pi), 2)
  rem!(z, x, z, r)
end

function sum!(z::BigFloat, arr::AbstractArray{BigFloat})
copy!(z, BigFloat(0))
    for i in arr
        ccall((:mpfr_add, :libmpfr), Int32,
            (Ptr{BigFloat}, Ptr{BigFloat}, Ptr{BigFloat}, Cint),
            &z, &z, &i, 0)
    end
    return z
end

# Functions for which NaN results are converted to DomainError, following Base
for f in (:sin,:cos,:tan,:sec,:csc,
          :acos,:asin,:atan,:acosh,:asinh,:atanh, :gamma)
  @eval begin
    function $(Symbol(f, "!"))(z::BigFloat, x::BigFloat)
            if isnan(x)
                return x
            end
            ccall(($(string(:mpfr_,f)), :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Int32), &z, &x, ROUNDING_MODE[])
            if isnan(z)
                throw(DomainError())
            end
            return z
        end
    end
end

# log of absolute value of gamma function
const lgamma_signp2 = Ref{Cint}()
function lgamma!(z::BigFloat, x::BigFloat)
    ccall((:mpfr_lgamma,:libmpfr), Cint, (Ptr{BigFloat}, Ptr{Cint}, Ptr{BigFloat}, Int32), &z, lgamma_signp2, &x, ROUNDING_MODE[])
    return z
end

lgamma_r!(z::BigFloat, x::BigFloat) = (lgamma!(z, x), lgamma_signp2[])

function atan2!(z::BigFloat, y::BigFloat, x::BigFloat)
    ccall((:mpfr_atan2, :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Ptr{BigFloat}, Int32), &z, &y, &x, ROUNDING_MODE[])
    return z
end

function copysign!(z::BigFloat, x::BigFloat, y::BigFloat)
    ccall((:mpfr_copysign, :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Ptr{BigFloat}, Int32), &z, &x, &y, ROUNDING_MODE[])
    return z
end

function frexp!(z::BigFloat, x::BigFloat)
    c = Ref{Clong}()
    ccall((:mpfr_frexp, :libmpfr), Int32, (Ptr{Clong}, Ptr{BigFloat}, Ptr{BigFloat}, Cint), c, &z, &x, ROUNDING_MODE[])
    return (z, c[])
end

function significand!(z::BigFloat, x::BigFloat)
    c = Ref{Clong}()
    ccall((:mpfr_frexp, :libmpfr), Int32, (Ptr{Clong}, Ptr{BigFloat}, Ptr{BigFloat}, Cint), c, &z, &x, ROUNDING_MODE[])
    # Double the significand to make it work as Base.significand
    ccall((:mpfr_mul_si, :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Clong, Int32), &z, &z, 2, ROUNDING_MODE[])
    return z
end

function round!(z::BigFloat, x::BigFloat)
    ccall((:mpfr_rint, :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Cint), &z, &x, ROUNDING_MODE[])
    return z
end
function round!(z::BigFloat, x::BigFloat,::RoundingMode{:NearestTiesAway})
    ccall((:mpfr_round, :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}), &z, &x)
    return z
end

function copy!(z::BigFloat, x::BigFloat)
    ccall((:mpfr_set, :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Int32),
          &z, &x, ROUNDING_MODE[])
    return z
end

function copy!(z::BigFloat, x::ClongMax)
    ccall((:mpfr_set_si, :libmpfr), Int32, (Ptr{BigFloat}, Clong, Int32),
          &z, x, ROUNDING_MODE[])
    return z
end

function copy!(z::BigFloat, x::CulongMax)
    ccall((:mpfr_set_ui, :libmpfr), Int32, (Ptr{BigFloat}, Culong, Int32),
          &z, x, ROUNDING_MODE[])
    return z
end

function copy!(z::BigFloat, x::CdoubleMax)
    ccall((:mpfr_set_d, :libmpfr), Int32, (Ptr{BigFloat}, Cdouble, Int32),
          &z, x, ROUNDING_MODE[])
    return z
end

function copy!(z::BigFloat, x::BigInt)
    ccall((:mpfr_set_z, :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigInt}, Int32),
          &z, &x, ROUNDING_MODE[])
    return z
end

copy!(z::BigFloat, x::Real) = copy!(z, BigFloat(x))

function nextfloat!(z::BigFloat, x::BigFloat)
  copy!(z, x)
  ccall((:mpfr_nextabove, :libmpfr), Int32, (Ptr{BigFloat},), &z) != 0
  return z
end

function prevfloat!(z::BigFloat, x::BigFloat)
  copy!(z, x)
  ccall((:mpfr_nextbelow, :libmpfr), Int32, (Ptr{BigFloat},), &z) != 0
  return z
end

# broadcast!(::typeof(identity), z::BigFloat, x::Real) = copy!(z, x)

function checklength(n::Int, m::Int)
  n != m &&
    throw(DimensionMismatch("first array has length $(n) which does not match the length of the second, $(m)."))
  n
end

checklength(a, b) = checklength(length(a), length(b))
checkmult(a::AbstractArray{T,2}, b) where T = checklength(size(a, 2), length(b))
checkmult(a, b::AbstractArray{T,2}) where T = checklenght(length(a), size(b, 2))

function dot!(z::BigFloat, a::AbstractArray{T,1}, b::AbstractArray{U,1}) where T <: Real where U <: Real

  n = checklength(a, b)
  _dot!(z, n, a, b)
end

function _dot!(z::BigFloat, n::Int, a::AbstractArray{T,1}, b::AbstractArray{T,1}) where T <: BigFloat
  copy!(z, 0)
  for i in 1:n
    fma!(z, a[i], b[i], z)
  end
  z
end

function _dot!(z::BigFloat, n::Int, a::AbstractArray{T,1}, b::AbstractArray{U,1}) where T <: BigFloat where U <: Real

  copy!(z, 0)
  c = BigFloat()
  for i in 1:n
    copy!(c, b[i])
    fma!(z, a[i], c, z)
  end
  z
end

function _dot!(z::BigFloat, n::Int, a::AbstractArray{T,1}, b::AbstractArray{U,1}) where T <: Real where U <: BigFloat
  _dot!(z, n, b, a)
end

function _dot!(z::BigFloat, n::Int, a::AbstractArray{T,1}, b::AbstractArray{U,1}) where T <: Real where U <: Real

  copy!(z, 0)
  c = BigFloat()
  for i in 1:n
    copy!(c, a[i])
    mul!(c, c, b[i])
    add!(z, z, c)
  end
  z
end

function mul!(z::AbstractVector{BigFloat}, a::AbstractArray{BigFloat,2}, b::AbstractVector{BigFloat})

  checkmult(a, b)
  n = checkmult(z, a)
  for i = 1:n
    isassigned(z, i) || ( z[i] = BigFloat() )
    dot!(z[i], view(a, i,:), b)
  end
  z
end

function mul!(z::AbstractVector{BigFloat}, a::AbstractVector{BigFloat}, b::Real)
  n = checklength(z, a)
  for i = 1:n
    isassigned(z, i) || ( z[i] = BigFloat() )
    mul!(z[i], a[i], b)
  end
  z
end

function add!(z::AbstractVector{BigFloat}, a::AbstractVector{BigFloat}, b::AbstractVector{<:Real})
  n = checklength(z, a)
  for i = 1:n
    isassigned(z, i) || ( z[i] = BigFloat() )
    add!(z[i], a[i], b[i])
  end
  z
end

function sub!(z::AbstractVector{BigFloat}, a::AbstractVector{BigFloat}, b::AbstractVector{<:Real})
  n = checklength(z, a)
  for i = 1:n
    isassigned(z, i) || ( z[i] = BigFloat() )
    sub!(z[i], a[i], b[i])
  end
  z
end

"""
Apply givens rotation, defined by `(c, s)` to `[a1, a2]`.
Use temporary work space `t[1:2]`. No aliasing of any used varables allowed.
"""
function rotate!(a1::BigFloat, a2::BigFloat, c::BigFloat, s::BigFloat, t::AbstractVector{BigFloat})

  mul!(t[1], a1, c)
  mul!(t[2], a1, s)
  fma!(a1, a2, s, t[1])
  fms!(a2, a2, c, t[2])
  nothing
end

end # module


using SharedArrays
using Distributed
export Fitness, GRN, evolve

struct Fitness
    nin::Int64
    nout::Int64
    func::Function
    cacheable::Bool
end

function Fitness(nin::Int64, nout::Int64, func::Function)
    Fitness(nin, nout, func, true)
end

function tournament(fits::Array{Float64}; tsize=3)
    # return the index of the winner of a n-way tournament
    if length(fits) == 1
        return 1
    else
        n = min(tsize, length(fits))
        fshuffle = randperm(length(fits))[1:n]
        winner = indmax(fits[fshuffle])
        return fshuffle[winner]
    end
end

function speciation(population::Array, reprs::Array, config::Config;
                    dfunc::Function=AGRN.distance)
    # return a vector of ints corresponding to the species of each individual in population
    ptype = typeof(population[1])
    species = Array{Int64}(length(population))
    for p in eachindex(population)
        distances = Array{Float64}(length(reprs))
        for r in eachindex(reprs)
            distances[r] = dfunc(population[p], reprs[r], config)
        end
        # distances = [distance(population[p], reprs[r]) for r in eachindex(reprs)]
        if minimum(distances) < config.speciation_thresh
            species[p] = indmin(distances)
        else
            species[p] = length(reprs)+1
            append!(reprs, [ptype(population[p])])
        end
    end
    species_set = sort(unique(species))
    for s in eachindex(species_set)
        species[species.==species_set[s]] = s
    end
    species
end

function species_sizes(fits::Array{Float64}, species::Array{Int64}, config::Config)
    nspecies = length(unique(species))
    spec_fits = map(x->mean(fits[species.==x])-minimum(fits), 1:nspecies)
    @debug("spec fits: $spec_fits")
    spec_sizes = map(x->spec_fits[x]/sum(spec_fits), 1:nspecies)
    spec_sizes = spec_sizes./sum(spec_sizes).*config.ga_population
    spec_sizes[isnan.(spec_sizes)] = 0
    spec_sizes = Int64.(round.(spec_sizes))

    while sum(spec_sizes) > config.ga_population
        spec_sizes[indmax(spec_sizes)] -= 1
    end
    while sum(spec_sizes) < config.ga_population
        spec_sizes[indmin(spec_sizes)] += 1
    end
    @debug("spec sizes: $spec_sizes")
    spec_sizes
end

function evolve(fitness::Fitness, config::Config)
    population = Array{GRN}(config.ga_population)
    fits = Array{Float64}(config.ga_population)
    fits .= -Inf
    species = Array{Int64}(config.ga_population)
    for p in eachindex(population)
        if p <= config.init_species
            population[p] = GRN(fitness.nin, fitness.nout, 1, config)
            species[p] = p
        else
            repr = rand(1:config.init_species)
            population[p] = mutate(population[repr], config)
            species[p] = repr
        end
    end
    best = population[1]
    max_fit = -Inf
    eval_count = 0

    for generation in 1:config.ga_num_generations
        # evaluation
        @debug("evaluation $generation")
        dfits = SharedArray{Float64}(config.ga_population)
        dfits .= fits
        @sync @distributed for p=1:config.ga_population
            dfits[p] = fitness.func(population[p])
        end
        fits .= dfits

        new_best = false
        for p in eachindex(population)
            fit = fits[p]
            eval_count += 1
            if fit > max_fit
                max_fit = fit
                best = population[p]
                new_best = true
            end
        end

        @info(@sprintf("E: %d %d %0.5f %0.5f",
                              generation, eval_count, mean(fits), max_fit))
        if new_best
            @info(@sprintf("B: %d %0.5f|%s", generation, max_fit,
                                  JSON.json(best)))
        end

        # representatives
        @debug("representatives $generation")
        nspecies = length(unique(species))
        reprs = Array{GRN}(nspecies)
        for s in 1:nspecies
            reprs[s] = GRN(population[rand(find(species.==s))])
        end

        # species sizes
        @debug("species sizes $generation $nspecies")
        spec_sizes = species_sizes(fits, species, config)
        new_pop = Array{GRN}(config.ga_population)
        new_fits = -Inf*ones(Float64, config.ga_population)
        popi = 1

        # create new population
        @debug("new population $generation $spec_sizes")
        for s in 1:nspecies
            @debug("species $s")
            sfits = fits[species.==s]
            spec = population[species.==s]
            ncross = round(spec_sizes[s] * config.ga_crossover_rate)
            nmut = round(spec_sizes[s] * config.ga_mutation_rate)
            ncopy = spec_sizes[s] - (ncross+nmut)
            while ncopy < 0
                if nmut > ncross
                    nmut -= 1
                else
                    ncross -= 1
                end
                ncopy = spec_sizes[s] - (ncross+nmut)
            end

            if popi > length(new_pop)
                break
            end

            # crossover
            @debug("Crossover $s popi: $popi, ncross: $ncross")
            for i in 1:ncross
                p1 = spec[tournament(sfits)]
                p2 = spec[tournament(sfits)]
                new_pop[popi] = crossover(p1, p2, config)
                popi += 1
            end

            if popi > length(new_pop)
                break
            end

            # mutation
            @debug("Mutation $s popi: $popi, nmut: $nmut")
            for i in 1:nmut
                parent = spec[tournament(sfits)]
                child = mutate(parent, config)
                if fitness.cacheable
                    if distance(parent, child, config) == 0
                        new_fits[popi] = fits[popi]
                    end
                end
                new_pop[popi] = child
                popi += 1
            end

            if popi > length(new_pop)
                break
            end

            # copy
            @debug("Copy $s popi: $popi, ncopy: $ncopy")
            for i in 1:ncopy
                new_pop[popi] = GRN(spec[tournament(sfits)])
                if fitness.cacheable
                    new_fits[popi] = fits[popi]
                end
                popi += 1
            end

            if popi > length(new_pop)
                break
            end
        end

        @debug("variable set $generation")
        species = speciation(new_pop, reprs, config)
        population = new_pop
        fits = new_fits

        @debug("done $generation")
    end

    max_fit, best
end


using Rayuela, HDF5

function get_dataset_numbers(dataset_name::String)
  nquery, nbase, ntrain = 0, 0, 0
  if dataset_name == "MNIST"
    nquery, nbase, ntrain = Int(10e3), Int(60e3), Int(60e3)
  elseif dataset_name == "labelme"
    nquery, nbase, ntrain = Int(2e3), Int(20019), Int(20019)
  elseif dataset_name in ["SIFT1M", "Convnet1M"]
    nquery, nbase, ntrain = Int(1e4), Int(1e6), Int(1e5)
  else
    error("dataset unknown")
  end
  return nquery, nbase, ntrain
end

function test_experiment(
  dataset_name::String,
  m::Integer,
  trial::Integer)

  @show dataset_name, m, trial
  bpath = joinpath("/home/julieta/Desktop/CQ/build/results/", lowercase(dataset_name) * "_$m", "trial_$trial")
  h = 256

  # Read the codes
  B = read_cq_bvecs(joinpath(bpath, "B"))
  B = convert(Matrix{Int16}, B)
  B .+= 1

  # Read the codebooks
  K = read_cq_fvecs(joinpath(bpath, "D"))
  C = Vector{Matrix{Cfloat}}(m)
  indices = splitarray(1:(m*h), m)
  for i = 1:m
    C[i] = K[:, indices[i]]
  end

  # Load the data
  nquery, nbase, ntrain = get_dataset_numbers(dataset_name)
  verbose = true
  Xt = Rayuela.read_dataset(dataset_name, ntrain)
  Xb = Rayuela.read_dataset(dataset_name * "_base", nbase)
  Xq = Rayuela.read_dataset(dataset_name * "_query", nquery, verbose)[:,1:nquery]
  gt = Rayuela.read_dataset(dataset_name * "_groundtruth", nquery, verbose)
  if dataset_name == "SIFT1M" || dataset_name == "GIST1M"
    gt = gt .+ 1
  end
  gt = convert( Vector{UInt32}, gt[1,1:nquery] )

  qerr = qerror(Xt, B, C) * nbase
  @printf("Qerror: %e\n", qerr )

  if dataset_name in ["SIFT1M", "Convnet1M"]
    # Encode the base
    B_base = convert(Matrix{Int16}, rand(1:h, m, size(Xb,2)))
    ilsiter, icmiter, randord, npert, cpp, V = 4, 4, false, 0, true, true
    B_base = Rayuela.encoding_icm(Xb, B_base, C, ilsiter, icmiter, randord, npert, cpp, V)

    qerr = qerror(Xb, B_base, C)
    @printf("%e\n", qerr)

    # Put the base codes into B
    B = B_base
  end

  # Fast search
  knn = 1000
  if verbose; print("Querying m=$m ... "); end
  @time dists, idx = linscan_cq(B, Xq, C, knn)
  if verbose; println("done"); end
  rec = eval_recall(gt, idx, knn)

  h5write(joinpath(bpath, "recall.h5"), "recall", rec)
end

function run_experiments(
  dataset_name::String,
  dictionaries_count::Int=8)

  ntrials = 10
  for trial = 1:1

    @show "trial", trial

    a = CQ_parameters()
    a.output_file_prefix = "/home/julieta/Desktop/CQ/build/results/sift_$(dictionaries_count)/trial_$trial/"
    a.dictionaries_count = dictionaries_count
    # a.max_iter=1

    if dataset_name == "SIFT1M"
      # do nothing
    elseif dataset_name == "MNIST"
      a.points_count     = Int(60e3)
      a.space_dimension  = 28 * 28
      a.mu               = 0.00001f0
      a.queries_count    = Int(10e3)
      a.groundtruth_length = 1
      a.output_file_prefix = "/home/julieta/Desktop/CQ/build/results/mnist_$(dictionaries_count)/trial_$trial/"
      a.points_file        = "/home/julieta/Desktop/CQ/build/data/mnist/mnist_learn.fvecs"
      a.queries_file       = "/home/julieta/Desktop/CQ/build/data/mnist/mnist_query.fvecs"
      a.groundtruth_file   = "/home/julieta/Desktop/CQ/build/data/mnist/mnist_groundtruth.ivecs"
    elseif dataset_name == "labelme"
      a.points_count     = 20019
      a.space_dimension  = 512
      a.mu               = 100f0
      a.queries_count    = 2000
      a.groundtruth_length = 1
      a.output_file_prefix = "/home/julieta/Desktop/CQ/build/results/labelme_$(dictionaries_count)/trial_$trial/"
      a.points_file        = "/home/julieta/Desktop/CQ/build/data/labelme/labelme_learn.fvecs"
      a.queries_file       = "/home/julieta/Desktop/CQ/build/data/labelme/labelme_query.fvecs"
      a.groundtruth_file   = "/home/julieta/Desktop/CQ/build/data/labelme/labelme_groundtruth.ivecs"
    else
      error("dataset unknown")
    end

    a.trained_dictionary_file = joinpath(a.output_file_prefix, "D")
    a.trained_binary_codes_file = joinpath(a.output_file_prefix, "B")
    a.output_retrieved_results_file = joinpath(a.output_file_prefix, "recall")

    # Prepare output folder
    if !Base.Filesystem.ispath(a.output_file_prefix)
      mkpath(a.output_file_prefix)
    end

    # Write parameters
    pname = "/home/julieta/Desktop/CQ/build/config.txt"
    dump_CQ_parameters(a, pname)

    # Run the experiment
    cmd = joinpath("/home/julieta/Desktop/CQ/build/CompositeQuantization")
    run(`$cmd $pname`)

  end
end

function dump_dataset(dataset_name, verbose=true)
  nquery, nbase, ntrain = get_dataset_numbers(dataset_name)
  Xt = Rayuela.read_dataset(dataset_name * "_base", ntrain)
  Xb = Rayuela.read_dataset(dataset_name * "_base", nbase)
  Xq = Rayuela.read_dataset(dataset_name * "_query", nquery, verbose)[:,1:nquery]
  gt = Rayuela.read_dataset(dataset_name * "_groundtruth", nquery, verbose)

  bpath = joinpath("/home/julieta/Desktop/CQ/build/data/", lowercase(dataset_name))
  if !Base.Filesystem.ispath(bpath)
    mkpath(bpath)
  end

  fvecs_write(Xt, joinpath(bpath, "$(lowercase(dataset_name))_learn.fvecs"))
  fvecs_write(Xb, joinpath(bpath, "$(lowercase(dataset_name))_base.fvecs"))
  fvecs_write(Xq, joinpath(bpath, "$(lowercase(dataset_name))_query.fvecs"))
  ivecs_write(convert(Matrix{Int32},gt.-=1), joinpath(bpath, "$(lowercase(dataset_name))_groundtruth.ivecs"))
end

# dump_dataset("MNIST")
# dump_dataset("labelme")
# dump_dataset("Convnet1M")

# @show("run_experiments")
# run_experiments("MNIST", 16)
run_experiments("labelme", 8)
# run_experiments("SIFT1M", 16)
# run_experiments("Convnet1M")

for trial = 3
  # test_experiment("labelme", 8, trial)
  # test_experiment("labelme", 16, trial)
  # test_experiment("MNIST", 8, trial)
  # test_experiment("MNIST", 16, trial)

  # test_experiment("SIFT1M", 8, trial)
  # test_experiment("SIFT1M", 16, trial)
end


using Random

n = 256

function apply_seed(seed)
    if seed === missing
        # should result in different numbers across launches
        Random.seed!()
        # XXX: this currently doesn't work, because of the definition in Base,
        #      `seed!(r::MersenneTwister=default_rng())`, which breaks overriding
        #      `default_rng` with a non-MersenneTwister RNG.
    elseif seed !== nothing
        # should result in the same numbers
        Random.seed!(seed)
    elseif seed === nothing
        # should result in different numbers across launches,
        # as determined by the seed set during module loading.
    end
end

@testset "rand($T), seed $seed" for T in (Int32, UInt32, Int64, UInt64, Int128, UInt128,
                                          Float16, Float32, Float64),
                                    seed in (nothing, #=missing,=# 1234)
    # different kernel invocations should get different numbers
    @testset "across launches" begin
        function kernel(A::AbstractArray{T}, seed) where {T}
            apply_seed(seed)
            tid = threadIdx().x
            A[tid] = rand(T)
            return nothing
        end

        a = CUDA.zeros(T, n)
        b = CUDA.zeros(T, n)

        @cuda threads=n kernel(a, seed)
        @cuda threads=n kernel(b, seed)

        if seed === nothing || seed === missing
            @test Array(a) != Array(b)
        else
            @test Array(a) == Array(b)
        end
    end

    # multiple calls to rand should get different numbers
    @testset "across calls" begin
        function kernel(A::AbstractArray{T}, B::AbstractArray{T}, seed) where {T}
            apply_seed(seed)
            tid = threadIdx().x
            A[tid] = rand(T)
            B[tid] = rand(T)
            return nothing
        end

        a = CUDA.zeros(T, n)
        b = CUDA.zeros(T, n)

        @cuda threads=n kernel(a, b, seed)

        @test Array(a) != Array(b)
    end

    # different threads should get different numbers
    @testset "across threads" for active_dim in 1:6
        function kernel(A::AbstractArray{T}, seed) where {T}
            apply_seed(seed)
            id = threadIdx().x*threadIdx().y*threadIdx().z*blockIdx().x*blockIdx().y*blockIdx().z
            A[id] = rand(T)
            return nothing
        end

        tx, ty, tz, bx, by, bz = [dim == active_dim ? 2 : 1 for dim in 1:6]
        a = CUDA.zeros(T, 2)

        @cuda threads=(tx, ty, tz) blocks=(bx, by, bz) kernel(a, seed)

        @test Array(a)[1] != Array(a)[2]
    end
end

@testset "basic randn($T), seed $seed" for T in (Float16, Float32, Float64),
                                           seed in (nothing, #=missing,=# 1234)
    function kernel(A::AbstractArray{T}, seed) where {T}
        apply_seed(seed)
        tid = threadIdx().x
        A[tid] = randn(T)
        return
    end

    a = CUDA.zeros(T, n)
    b = CUDA.zeros(T, n)

    @cuda threads=n kernel(a, seed)
    @cuda threads=n kernel(b, seed)

    if seed === nothing || seed === missing
        @test Array(a) != Array(b)
    else
        @test Array(a) == Array(b)
    end
end

@testset "basic randexp($T), seed $seed" for T in (Float16, Float32, Float64),
                                           seed in (nothing, #=missing,=# 1234)
    function kernel(A::AbstractArray{T}, seed) where {T}
        apply_seed(seed)
        tid = threadIdx().x
        A[tid] = randexp(T)
        return
    end

    a = CUDA.zeros(T, n)
    b = CUDA.zeros(T, n)

    @cuda threads=n kernel(a, seed)
    @cuda threads=n kernel(b, seed)

    if seed === nothing || seed === missing
        @test Array(a) != Array(b)
    else
        @test Array(a) == Array(b)
    end
end


# Implements Transition to represent a named X-ray transition.
"""
    Transition

Represents an inner and outer shell that describe an X-ray transition. Only
transitions for which one or more element has a characteristic x-ray are supported
according to the default line weight database (weight > 0 for one or more Z).
This structure does not contain the Element information necessary to specify
a characteristic X-ray.

Data items:

    innerShell::SubShell
    outerShell::SubShell

Example:

    tr1 = Transition(n"K",n"L3")
    tr2   = Transition(SubShell(1),SubShell(4))
    @assert tr1==tr2
"""
struct Transition
    innershell::SubShell
    outershell::SubShell
    function Transition(inner::SubShell, outer::SubShell)
        if !(haskey(transitions(), (inner.index, outer.index)))
            error("$(inner)-$(outer) does not represent a known Transition.")
        end
        return new(inner, outer)
    end
end

"""
    has(elm::Element, tr::Transition)::Bool

Is the specified Transition available for the specified element?

Example:

    @assert has(n"Fe L3-M5)
    @assert !has(n"Fe N7-O9)
"""
has(elm::Element, tr::Transition)::Bool =
    charactericXRayAvailable(z(elm), tr.innershell.index, tr.outershell.index)

"""
    shell(tr::Transition)

Return the shell (Shell(1), Shell(2),...) associated with the transition's inner shell.

Example:

    @assert shell(n"K-L3")==Shell(1)
    @assert shell(n"M5-N7")==Shell(3)
"""
shell(tr::Transition) = shell(tr.innershell)

inner(tr::Transition) = tr.innershell
outer(tr::Transition) = tr.outershell

function everytransition(trs)
    lttr(tr1, tr2) = (tr1[1] == tr2[1] ? isless(tr1[2], tr2[2]) : isless(tr1[1], tr2[1]))
    return map(
        tr -> Transition(SubShell(tr[1]), SubShell(tr[2])),
        sort([keys(trs)...], lt = lttr),
    )
end

"""
    alltransitions

A complete list of all the transitions present in one or more elements.
"""
const alltransitions =
    tuple(filter(tr -> n(shell(tr)) <= 3, everytransition(transitions()))...)

const transitionnames =
    tuple(map(tr -> "$(tr.innershell)-$(tr.outershell)", alltransitions)...)

"""
    transition(str::AbstractString)::Transition

Constructs a Transition structure from a string representation of the form \"K-L3\"
or \"L3-M5\".  Asserts if the transition is not a known transition.
"""
function Base.parse(::Type{Transition}, str::AbstractString)
    ff = findfirst(name -> name == str, transitionnames)
    if isnothing(ff)
        error("$(str) does not represent a known transition.")
    end
    return alltransitions[ff]
end

transition(str::AbstractString) = parse(Transition, str)

"""
    ktransitions

A complete list of all the K-shell transitions.
"""
const ktransitions = tuple(filter(tr -> shell(tr) == Shell(1), collect(alltransitions))...)

"""
    kalpha

A list of K-L? transitions.
"""
const kalpha =
    tuple(filter(tr -> shell(tr.outershell) == Shell(2), collect(ktransitions))...)

"""
    kbeta

A list of K-M? transitions.
"""
const kbeta =
    tuple(filter(tr -> shell(tr.outershell) == Shell(3), collect(ktransitions))...)

"""
    kother

A list of K-!L? transitions.
"""
const kother =
    tuple(filter(tr -> shell(tr.outershell) ≠ Shell(2), collect(ktransitions))...)

"""
    ltransitions

A complete list of all the L-shell transitions.
"""
const ltransitions = tuple(filter(tr -> shell(tr) == Shell(2), collect(alltransitions))...)

const lalpha = transition.(("L3-M4", "L3-M5"))
const lbeta = transition.(("L2-M4", "L1-M2", "L1-M3", "L3-N4", "L3-N5", "L3-N1", "L3-O1"))

"""
    mtransitions

A complete list of all the M-shell transitions.
"""
const mtransitions = tuple(filter(tr -> shell(tr) == Shell(3), collect(alltransitions))...)

const malpha = transition.(("M5-N6", "M5-N7"))
const mbeta = transition.(("M4-N6",))
"""
    ntransitions

A complete list of all the N-shell transitions.
"""
const ntransitions = tuple(filter(tr -> shell(tr) == Shell(4), collect(alltransitions))...)

"""
    otransitions
A complete list of all the O-shell transitions.
"""
const otransitions = tuple(filter(tr -> shell(tr) == Shell(5), collect(alltransitions))...)

"""
    transitionsbygroup

A Dict{String,Tuple{Transition}} mapping group name into a list of transitions.
Keys are "K","L","M","N","O" and "Kα", "Ka", "Kβ", "Kb" and "Kother".
"""
const transitionsbygroup = Dict(
    "K" => ktransitions,
    "Kα" => kalpha,
    "Ka" => kalpha,
    "Kβ" => kbeta,
    "Kb" => kbeta,
    "Kother" => kother,
    "L" => ltransitions,
    "Lα" => lalpha,
    "La" => lalpha,
    "Lβ" => lbeta,
    "M" => mtransitions,
    "Mα" => malpha,
    "Ma" => malpha,
    "Mβ" => mbeta,
    "N" => ntransitions,
    "O" => otransitions,
)

"""
    transitionsbyshell

A Dict{Char,Tuple{Transition}} mapping shell name into a list of transitions.
Keys are Shell objects.
"""
const transitionsbyshell = Dict(
    Shell(1) => ktransitions,
    Shell(2) => ltransitions,
    Shell(3) => mtransitions,
    Shell(4) => ntransitions,
    Shell(5) => otransitions,
)

const transitionsbysubshell =
    Dict(ss => filter(tr -> tr.innershell == ss, alltransitions) for ss in allsubshells)

Base.hash(tr::Transition, h::UInt)::UInt =
    hash(hash(tr.innershell), hash(hash(tr.outershell), h))

Base.isequal(tr1::Transition, tr2::Transition) =
    isequal(tr1.innershell, tr2.innershell) && isequal(tr1.outershell, tr2.outershell)

function Base.isless(tr1::Transition, tr2::Transition)
    return isequal(tr1.innershell, tr2.innershell) ?
           isless(tr1.outershell, tr2.outershell) : isless(tr1.innershell, tr2.innershell)
end

"""
    transition(inner::SubShell, outer::SubShell)::Transition

Return a Transition structure from an inner and outer shell. This function tests
to ensure that the Transition is a known transition.
"""
function transition(inner::SubShell, outer::SubShell)::Transition
    ff = findfirst(
        tr -> (tr.innershell == inner) && (tr.outershell == outer),
        alltransitions,
    )
    @assert !isnothing(ff) "$(inner)-$(outer) does not represent a known Transition."
    alltransitions[ff]
end

"""
    exists(inner::SubShell, outer::SubShell)::Bool

Does a transition exist in our database for this pair of shells?
"""
exists(inner::SubShell, outer::SubShell)::Bool =
    !isnothing(
        findfirst(
            tr -> (tr.innershell == inner) && (tr.outershell == outer),
            alltransitions,
        ),
    )

Base.show(io::IO, tr::Transition) = print(io, tr.innershell, "-", tr.outershell)


function warmup()
    t1 = now()
    ds = Dataset(x1 = rand(1:2, 10000), x2 = rand(1:2, 10000), x3 = rand(1:2, 10000) * 1.1, x4 = rand([Date(1), Date(2)], 10000), x5 = string.(rand(1:2), 10000),
                 x6 = PooledArray(Characters{5, UInt8}.(rand(1:2, 10000))), x7 = Int32.(rand(1:2, 10000)), x8 = Float32.(rand(1:2, 10000) * 1.1), x9 = PooledArray(Characters{3, UInt8}.(rand(1:2, 10000))), x10 = PooledArray(Characters{12, UInt8}.(rand(1:2, 10000))))
    for i in 1:ncol(ds)
        sortperm(ds, i)
    end
    for i in 1:ncol(ds)
        sortperm(ds, i, alg = QuickSort)
    end
    for i in 1:ncol(ds)
        sortperm(ds, 1:i)
    end
    for i in 1:ncol(ds)-1
        sortperm(ds, [ncol(ds), i])
    end
    for i in 1:ncol(ds)
        groupby(ds, i)
        gatherby(ds, i)
    end
    for i in 1:ncol(ds)
        groupby(ds, 1:i)
        gatherby(ds, 1:i)
    end
    byrow(ds, all, :, by = isequal(1))
    byrow(ds, sum)
    for op in (+, *, -, /)
        byrow(ds, op, 1:2)
    end
    combine(groupby(ds, [6,1]), (1,2)=>cor)
    combine(groupby(ds,1), Ref([1,2,3,7,8]) .=> [sum, mean, length, maximum, minimum, var, std, median, median!, sort])
    combine(groupby(ds,1), r"x1$" .=> [sum, mean, length, maximum, minimum, var, std, median, sort])
    ds2 = ds[1:2, [1,3]]
    combine(gatherby(ds,1), Ref([1,2,3,7,8]) .=> [median, sort])
    combine(gatherby(ds,1), Ref([1,2,3,7,8]) .=> [sum, mean, length, maximum, minimum, var, std])
    combine(gatherby(ds,1), r"x1$" .=> [sum, mean, length, maximum, minimum, var, std])
    IMD._gather_groups_hugeds_multicols(ds, 1:6, Val(Int32), threads = true)

    ds2 = ds[1:2, [1,3,7]]
    innerjoin(ds, ds2, on = [:x1, :x3, :x7])
    leftjoin(ds, ds2, on = [:x1, :x3, :x7])
    innerjoin(ds, ds2, on = [:x1, :x3, :x7], method = :hash)
    leftjoin(ds, ds2, on = [:x1, :x3, :x7], method = :hash)
    leftjoin(ds, ds2, on = [:x1, :x3, :x7], accelerate = true)
    ds3 = unique(ds, 1)[:, [1,3,5,7]]
    leftjoin(ds, ds3, on = :x1, makeunique = true)
    innerjoin(ds, ds3, on = :x1, makeunique = true)
    leftjoin(ds, ds3, on = :x1, makeunique = true, method = :hash)
    innerjoin(ds, ds3, on = :x1, makeunique = true, method = :hash)
    transpose(ds, 1:ncol(ds))
    transpose(groupby(ds,1:8), [2,3])
    show(IOBuffer(), MIME("text/html"), ds);
    # views
    ds = Dataset("A"=> ["A", "A" ,"A", "A", "B", "B","G"],
        "B"=> ["C", "D", "E", "B", "F", "G","N"]
        )
    select!(ds, 2,1)
    rename!(ds, [:child, :parent])

    function ff(ds, i)
        newds = leftjoin(ds, unique(dropmissing(ds,ncol(ds), view=true)[!, reverse(ncol(ds):-1:ncol(ds)-1)], [1,2] , view=true), on = [i+1=>1], makeunique = true)
        rename!(x->replace(x,"1"=>"grand"), newds)
        newds
    end

    ff(ff(ff(ds, 1),2), 3)

    ds = Dataset("A"=> ["A", "A" ,"A", "A", "B", "B","G"],
        "B"=> ["C", "D", "E", "B", "F", "G","N"]
        )
    select!(ds, 2,1)
    rename!(ds, [:child, :parent])

    function ff_hash(ds, i)
        newds = leftjoin(ds, unique(dropmissing(ds,ncol(ds), view=true)[!, reverse(ncol(ds):-1:ncol(ds)-1)], [1,2] , view=true), on = [i+1=>1], makeunique = true, method = :hash)
        rename!(x->replace(x,"1"=>"grand"), newds)
        newds
    end

    ff_hash(ff_hash(ff_hash(ds, 1),2), 3)
    @assert ff(ff(ff(ds, 1),2), 3) == ff_hash(ff_hash(ff_hash(ds, 1),2), 3)

    # some from test
    store = Dataset([[Date("2019-10-05"), Date("2019-10-04"), Date("2019-10-02"), Date("2020-01-01"), Date("2019-10-01"), Date("2019-10-02"), Date("2019-10-05"), Date("2019-10-04"), Date("2019-10-03"), Date("2019-10-03")],
        ["A", "A", "B", "A", "B", "A", "B", "B", "A", "B"]], [:date, :store])
    roster = Dataset([Union{Missing, String}["A", "A", "B", "A", "B", "A", "B", "B"],
             Union{Missing, Int64}[4, 1, 8, 2, 5, 3, 6, 7],
             Union{Missing, Date}[Date("2019-10-04"), Date("2019-09-30"), Date("2019-10-04"), Date("2019-10-02"), Date("2019-09-30"), Date("2019-10-03"), Date("2019-10-02"), Date("2019-10-03")],
             Union{Missing, Date}[Date("2019-10-06"), Date("2019-10-04"), Date("2019-10-06"), Date("2019-10-04"), Date("2019-10-04"), Date("2019-10-05"), Date("2019-10-04"), Date("2019-10-05")]],
             ["store", "employee_ID", "start_date", "end_date"])
    inn_r1 =  innerjoin(store, roster, on = [:date => (:start_date, nothing)], makeunique = true, stable = true)
    inn_r1_v =  innerjoin(store, view(roster, :, :), on = [:date => (:start_date, nothing)], makeunique = true, stable = true)
    inn_r1_a =  innerjoin(store, roster, on = [:date => (:start_date, nothing)], makeunique = true, stable = true, accelerate = true)
    inn_r1_v_a =  innerjoin(store, view(roster, :, :), on = [:date => (:start_date, nothing)], makeunique = true, stable = true, accelerate = true)

    inn_r1 =  innerjoin(store, roster, on = [:store => :store, :date => (nothing, :start_date)], stable = true)
    inn_r1_a =  innerjoin(store, roster, on = [:store => :store, :date => (nothing, :start_date)], stable = true, accelerate = true)
    inn_r1 =  innerjoin(store, roster, on = [:store => :store, :date => (:end_date, :start_date)], stable = true)
    inn_r1_a =  innerjoin(store, roster, on = [:store => :store, :date => (:end_date, :start_date)], stable = true, accelerate = true)

    inn_r1 =  innerjoin(view(store, :, :), roster, on = [:store => :store, :date => (nothing, :start_date)], stable = true)
    inn_r1_a =  innerjoin(view(store, :, :), roster, on = [:store => :store, :date => (nothing, :start_date)], stable = true, accelerate = true)
    inn_r1 =  innerjoin(view(store, :, :), roster, on = [:store => :store, :date => (:end_date, :start_date)], stable = true)
    inn_r1_a =  innerjoin(view(store, :, :), roster, on = [:store => :store, :date => (:end_date, :start_date)], stable = true, accelerate = true)

    inn_r1 =  innerjoin(store, roster, on = [:date => (:start_date, nothing)], makeunique = true, stable = true, method = :hash)
    inn_r1_v =  innerjoin(store, view(roster, :, :), on = [:date => (:start_date, nothing)], makeunique = true, stable = true, method = :hash)
    inn_r1_a =  innerjoin(store, roster, on = [:date => (:start_date, nothing)], makeunique = true, stable = true, accelerate = true, method = :hash)
    inn_r1_v_a =  innerjoin(store, view(roster, :, :), on = [:date => (:start_date, nothing)], makeunique = true, stable = true, accelerate = true, method = :hash)

    inn_r1 =  innerjoin(store, roster, on = [:store => :store, :date => (nothing, :start_date)], stable = true, method = :hash)
    inn_r1_a =  innerjoin(store, roster, on = [:store => :store, :date => (nothing, :start_date)], stable = true, accelerate = true, method = :hash)
    inn_r1 =  innerjoin(store, roster, on = [:store => :store, :date => (:end_date, :start_date)], stable = true, method = :hash)
    inn_r1_a =  innerjoin(store, roster, on = [:store => :store, :date => (:end_date, :start_date)], stable = true, accelerate = true, method = :hash)

    inn_r1 =  innerjoin(view(store, :, :), roster, on = [:store => :store, :date => (nothing, :start_date)], stable = true, method = :hash)
    inn_r1_a =  innerjoin(view(store, :, :), roster, on = [:store => :store, :date => (nothing, :start_date)], stable = true, accelerate = true, method = :hash)
    inn_r1 =  innerjoin(view(store, :, :), roster, on = [:store => :store, :date => (:end_date, :start_date)], stable = true, method = :hash)
    inn_r1_a =  innerjoin(view(store, :, :), roster, on = [:store => :store, :date => (:end_date, :start_date)], stable = true, accelerate = true, method = :hash)

    ds = Dataset(foo = ["one", "one", "one", "two", "two","two"],
                      bar = ['A', 'B', 'C', 'A', 'B', 'C'],
                      baz = [1, 2, 3, 4, 5, 6],
                      zoo = ['x', 'y', 'z', 'q', 'w', 't'])
    ds2 = transpose(groupby(ds, :foo, stable = true), :baz, id = :bar)
    ds = Dataset(g1 = ["g2", "g1", "g1", "g2", "g1", "g3"], c1 = ["c1", "c1", "c1", "c1", "c1", "c1"],
               id1 = ["id2_g2", "id1_g1", "id2_g1", "id1_g2", "id3_g1", "id4_g3"], id2 = [2, 3, 1, 5, 4, 6],
               val1 = [1, 2, 3, 4, 5, 6], val2 = [1.1, 2.1, 0.5, 1.3, 1.0, 2])
   ds2 = transpose(ds, [:val1, :val2])
   ds3 = transpose(ds, [:val1, :val2], id = :id2)
   ds4 = transpose(ds, [:val1, :val2], id = [:id1, :id2])
   ds5 = transpose(groupby(ds, 1, stable = true), [:val1, :val2])
   ds6 = transpose(groupby(ds, 1, stable = true), [:val1, :val2], id = :id1)
   ds7 = transpose(groupby(ds, 1, stable = true), [:val1, :val2], id = [:id1, :id2])
   ds8 = transpose(groupby(ds, 1:2, stable = true), [:val1, :val2], id = :id2)
   ds9 = transpose(groupby(ds, 1:2, stable = true), [:val1], id = [:id2, :id1])

   ds = Dataset([Union{Missing, Int64}[1, 1, 1, 2, 2, 2],
                Union{Missing, String}["foo", "bar", "monty", "foo", "bar", "monty"],
                Union{Missing, String}["a", "b", "c", "d", "e", "f"],
                Union{Missing, Int64}[1, 2, 3, 4, 5, 6]], [:g, :key, :foo, :bar])
    dst = transpose(groupby(ds, :g), (:foo, :bar), id = :key, variable_name = "_variables_")

    ds = Dataset(g = [1, 1, 1, 2, 2],
                        x1_int = [0, 0, 1, missing, 2],
                        x2_int = [3, 2, 1, 3, -2],
                        x1_float = [1.2, missing, -1.0, 2.3, 10],
                        x2_float = [missing, missing, 3.0, missing, missing],
                        x3_float = [missing, missing, -1.4, 3.0, -100.0])
    isequal(byrow(ds, argmin, r"float"), [:x1_float, missing, :x3_float, :x1_float, :x3_float])
    ds1 = Dataset(a = Union{String, Missing}["a", "b", "a", "b", "a", "b"],
                    b = Vector{Union{Int, Missing}}(1:6),
                    c = Union{Int, Missing}[1:3;1:3])
    ds = vcat(ds1, ds1)
    findall(duplicates(ds)) == collect(7:12)
    fmt(x)=1
    setformat!(ds, :a=>fmt)
    findall(duplicates(ds, :a, mapformats = true)) == 2:12
    unique(ds) == ds1
    unique(ds, 2:3) == ds1

    t2 = now()
    Dataset(x1 = "Finished warmup in", x2 = t2-t1)
end


## read_bin function with full list of argument

function read_bin(fil::String,kt,kk,prec::DataType);

  if ~isempty(kt);
    error("non-empty kt option not implemented yet");
  end;

  if ~isempty(kk);
    error("non-empty kk option not implemented yet");
  end;

  (n1,n2)=MeshArrays.ioSize;

  if prec==Float64;
    reclen=8;
  else;
    reclen=4;
  end;
  tmp1=stat(fil);
  n3=Int64(tmp1.size/n1/n2/reclen);

  fid = open(fil);
  fld = Array{prec,1}(undef,(n1*n2*n3));
  read!(fid,fld);
  fld = hton.(fld);

  n3>1 ? s=(n1,n2,n3) : s=(n1,n2)
  v0=reshape(fld,s);
  convert2gcmfaces(v0);

end

## read_bin function with reduced list of argument

function read_bin(fil::String,prec::DataType);
  read_bin(fil,[],[],prec);
end

## read_bin function with reduced list of argument

function read_bin(fil::String);
  read_bin(fil,[],[],Float32);
end


# this file has not been completed yet

export BDD2

"""
type alias
"""

const UniqueTable2{Ts} = Dict{Tuple{HeaderID,NodeID,NodeID,Bool},AbstractBDDNode{Ts}}
const BinCache2{Ts} = Dict{Tuple{NodeID,Bool,NodeID,Bool},Tuple{AbstractBDDNode{Ts},Bool}}

"""
Node2
"""

struct BDD2Node{Ts} <: AbstractBDDNode{Ts}
    id::NodeID
    header::BDDNodeHeader{Ts}
    low::AbstractBDDNode{Ts}
    high::AbstractBDDNode{Ts}
    neg::Bool
end

"""
BDD2{Ts}

The structure for BDD. This includes all the nodes and caches.
"""

struct BDD2{Ts}
    manager::BDDManager
    headers::Dict{Ts,BDDNodeHeader{Ts}}
    utable::UniqueTable2{Ts}
    zero::BDDTerminal{Ts}
    andcache::BinCache2{Ts}
    orcache::BinCache2{Ts}
    xorcache::BinCache2{Ts}
end

function BDD2(::Type{Ts} = Symbol) where Ts
    manager = BDDManager()
    zero = BDDTerminal{Ts}(nodeid!(manager))
    BDD2{Ts}(
        manager,
        Dict{Ts,BDDNodeHeader{Ts}}(),
        UniqueTable2{Ts}(),
        zero,
        BinCache2{Ts}(),
        BinCache2{Ts}(),
        BinCache2{Ts}()
    )
end

function var(b::BDD2{Ts}, label::Ts)::Tuple{AbstractBDDNode{Ts},Bool} where Ts
    h = get(b.headers, label) do
        h = BDDNodeHeader(headerid!(b.manager), level!(b.manager), label)
        b.headers[label] = h
    end
    _node(b, h, b.zero, false, b.zero, true)
end

function _node(b::BDD2{Ts}, h::BDDNodeHeader{Ts}, low::AbstractBDDNode{Ts}, blow::Bool, high::AbstractBDDNode{Ts}, bhigh::Bool)::Tuple{AbstractBDDNode{Ts},Bool} where Ts
    if blow == bhigh && low.id == high.id
        (low, blow)
    else
        flag = blow != bhigh
        key = (h.id, low.id, high.id, flag)
        node = get(b.utable, key) do
            b.utable[key] = BDD2Node(nodeid!(b.manager), h, low, high, flag)
        end
        (node, blow)
    end
end

### binoperator

function not(b::BDD2{Ts}, f::Tuple{AbstractBDDNode{Ts},Bool}) where Ts
    (f[1], !f[2])
end

function and(b::BDD2{Ts}, f::Vararg{Tuple{AbstractBDDNode{Ts},Bool}}) where Ts
    ans, bans = (b.zero, true)
    for x = f
        ans, bans = _binapply(BDDAnd(), b.andcache, b, ans, bans, x[1], x[2])
    end
    (ans, bans)
end

function or(b::BDD2{Ts}, f::Vararg{Tuple{AbstractBDDNode{Ts},Bool}}) where Ts
    ans, bans = (b.zero, false)
    for x = f
        ans, bans = _binapply(BDDOr(), b.orcache, b, ans, bans, x[1], x[2])
    end
    (ans, bans)
end

function xor(b::BDD2{Ts}, f::Tuple{AbstractBDDNode{Ts},Bool}, g::Tuple{AbstractBDDNode{Ts},Bool}) where Ts
    _binapply(BDDXor(), b.xorcache, b, f, g)
end

function imp(b::BDD2{Ts}, f::Tuple{AbstractBDDNode{Ts},Bool}, g::Tuple{AbstractBDDNode{Ts},Bool}) where Ts
    or(b, bddnot(b, f), g)
end

function ite(b::BDD2{Ts}, f::Tuple{AbstractBDDNode{Ts},Bool}, g::Tuple{AbstractBDDNode{Ts},Bool}, h::Tuple{AbstractBDDNode{Ts},Bool}) where Ts
    or(b, and(b, f, g), and(b, not(b, f), h))
end

### primitive

function getzero(f::BDD2Node{Ts}, bf::Bool) where Ts
    (f.low, bf)
end

function getone(f::BDD2Node{Ts}, bf::Bool) where Ts
    (f.high, bf != f.neg)
end

function _binapply(op::AbstractBDDOperator, cache::BinCache2{Ts}, b::BDD2{Ts}, f::BDD2Node{Ts}, bf::Bool, g::BDD2Node{Ts}, bg::Bool)::Tuple{AbstractBDDNode{Ts},Bool} where Ts
    key = (f.id, bf, g.id, bg)
    get(cache, key) do
        if f.header.level > g.header.level
            n0, bn0 = _binapply(op, cache, b, getzero(f, bf)..., g, bg)
            n1, bn1 = _binapply(op, cache, b, getone(f, bf)..., g, bg)
            ans, bans = _node(b, f.header, n0, bn0, n1, bn1)
        elseif f.header.level < g.header.level
            n0, bn0 = _binapply(op, cache, b, f, bf, getzero(g, bg)...)
            n1, bn1 = _binapply(op, cache, b, f, bf, getone(g, bg)...)
            ans, bans = _node(b, g.header, n0, bn0, n1, bn1)
        else
            n0, bn0 = _binapply(op, cache, b, getzero(f, bf)..., getzero(g, bg)...)
            n1, bn1 = _binapply(op, cache, b, getone(f, bf)..., getone(g, bg)...)
            ans, bans = _node(b, f.header, n0, bn0, n1, bn1)
        end
        cache[key] = (ans, bans)
    end
end

## and

function _binapply(::BDDAnd, ::BinCache2{Ts}, b::BDD2{Ts}, f::BDDTerminal{Ts}, bf::Bool, g::BDD2Node{Ts}, bg::Bool)::Tuple{AbstractBDDNode{Ts},Bool} where Ts
    if bf == true
        (g, bg)
    else
        (b.zero, false)
    end
end

function _binapply(::BDDAnd, ::BinCache2{Ts}, b::BDD2{Ts}, f::BDD2Node{Ts}, bf::Bool, g::BDDTerminal{Ts}, bg::Bool)::Tuple{AbstractBDDNode{Ts},Bool} where Ts
    if bg == true
        (f, bf)
    else
        (b.zero, false)
    end
end

function _binapply(::BDDAnd, ::BinCache2{Ts}, b::BDD2{Ts}, f::BDDTerminal{Ts}, bf::Bool, g::BDDTerminal{Ts}, bg::Bool)::Tuple{AbstractBDDNode{Ts},Bool} where Ts
    if bf == true && bg == true
        (b.zero, true)
    else
        (b.zero, false)
    end
end

## or

function _binapply(::BDDOr, ::BinCache2{Ts}, b::BDD2{Ts}, f::BDDTerminal{Ts}, bf::Bool, g::BDD2Node{Ts}, bg::Bool)::Tuple{AbstractBDDNode{Ts},Bool} where Ts
    if bf == true
        (b.zero, true)
    else
        (g, bg)
    end
end

function _binapply(::BDDOr, ::BinCache2{Ts}, b::BDD2{Ts}, f::BDD2Node{Ts}, bf::Bool, g::BDDTerminal{Ts}, bg::Bool)::Tuple{AbstractBDDNode{Ts},Bool} where Ts
    if bg == true
        (b.zero, true)
    else
        (f, bf)
    end
end

function _binapply(::BDDOr, ::BinCache2{Ts}, b::BDD2{Ts}, f::BDDTerminal{Ts}, bf::Bool, g::BDDTerminal{Ts}, bg::Bool)::Tuple{AbstractBDDNode{Ts},Bool} where Ts
    if bf == false && bg == false
        (b.zero, false)
    else
        (b.zero, true)
    end
end

## xor

function _binapply!(::BDDXor, ::BinCache2{Ts}, b::BDD2{Ts}, f::BDDTerminal{Ts}, bf::Bool, g::BDD2Node{Ts}, bg::Bool)::Tuple{AbstractBDDNode{Ts},Bool} where Ts
    if bf == true
        (g, !bg)
    else
        (g, bg)
    end
end

function _binapply!(::BDDXor, ::BinCache2{Ts}, b::BDD2{Ts}, f::BDD2Node{Ts}, bf::Bool, g::BDDTerminal{Ts}, bg::Bool)::Tuple{AbstractBDDNode{Ts},Bool} where Ts
    if bg == true
        (f, !bf)
    else
        (f, bf)
    end
end

function _binapply!(::BDDXor, ::BinCache2{Ts}, b::BDD2{Ts}, f::BDDTerminal{Ts}, bf::Bool, g::BDDTerminal{Ts}, bg::Bool)::Tuple{AbstractBDDNode{Ts},Bool} where Ts
    if bf == bg
        (b.zero, false)
    else
        (b.zero, true)
    end
end

##

# function fteval(b::BDD{Ts}, f::Tuple{AbstractNode{Ts},Bool}, env::Dict{Ts,Tx})::Tx where {Ts,Tx}
#     cache = Dict{Tuple{UInt,Bool},Tx}()
#     _fteval(b, f, env, cache)
# end

# function _fteval(b::BDD{Ts}, f::Tuple{Node{Ts},Bool}, env::Dict{Ts,Tx}, cache::Dict{Tuple{UInt,Bool},Tx})::Tx where {Ts,Tx}
#     get(cache, (f[1].id, f[2])) do
#         p = env[f[1].header.label]
#         fprob = (1-p) * _fteval(b, getzero(f), env, cache) + p * _fteval(b, getone(f), env, cache)
#         cache[(f[1].id,!f[2])] = 1 - fprob
#         cache[(f[1].id,f[2])] = fprob
#     end
# end

# function _fteval(b::BDD{Ts}, f::Tuple{Terminal{Ts},Bool}, env::Dict{Ts,Tx}, cache::Dict{Tuple{UInt,Bool},Tx})::Tx where {Ts,Tx}
#     (f[2] == false) ? Tx(0) : Tx(1)
# end

# ##

# mutable struct MinPath
#     len::Int
#     set::Vector{Vector{Bool}}
# end

# function ftmcs(b::BDD{Ts}, f::Tuple{AbstractNode{Ts},Bool}) where Ts
#     result = Vector{Ts}[]
#     r = f
#     while r != (b.zero, false)
#         f, s = _ftmcs(b, r)
#         append!(result, s)
#         r = bddnot(b, bddimp(b, r, f))
#     end
#     result
# end

# function _ftmcs(b::BDD{Ts}, f::Tuple{AbstractNode{Ts},Bool}) where Ts
#     vars = Dict([x.level => var!(b, x.label) for (k,x) = b.headers])
#     path = [false for i = 1:Int(b.totalvarid)]
#     s = MinPath(Int(b.totalvarid), Vector{Bool}[])
#     _ftmcs(b, f, path, s)
#     result = (b.zero, false)
#     result2 = Vector{Ts}[]
#     for x = s.set
#         tmp = (b.zero, true)
#         tmp2 = Ts[]
#         for i = 1:length(x)
#             if x[i] == true
#                 tmp = bddand(b, tmp, vars[i])
#                 push!(tmp2, vars[i][1].header.label)
#             end
#         end
#         result = bddor(b, result, tmp)
#         push!(result2, tmp2)
#     end
#     return result, result2
# end

# function _ftmcs(b::BDD{Ts}, f::Tuple{Node{Ts},Bool}, path::Vector{Bool}, s::MinPath) where Ts
#     if s.len < sum(path)
#         return
#     end
#     path[f[1].header.level] = false
#     _ftmcs(b, getzero(f), path, s)
#     path[f[1].header.level] = true
#     _ftmcs(b, getone(f), path, s)
#     path[f[1].header.level] = false
#     nothing
# end

# function _ftmcs(b::BDD{Ts}, f::Tuple{Terminal{Ts},Bool}, path::Vector{Bool}, s::MinPath) where Ts
#     if f[2] == true
#         if s.len > sum(path)
#             s.len = sum(path)
#             s.set = [copy(path)]
#         elseif s.len == sum(path)
#             push!(s.set, copy(path))
#         end
#     end
#     nothing
# end

##

function todot(b::BDD2{Ts}, f::Tuple{AbstractBDDNode{Ts},Bool}) where Ts
    io = IOBuffer()
    visited = Set{AbstractBDDNode{Ts}}()
    println(io, "digraph { layout=dot; overlap=false; splines=true; node [fontsize=10];")
    _todot!(b, f[1], visited, io)
    println(io, "}")
    return String(take!(io))
end

function _todot!(b::BDD2{Ts}, f::BDDTerminal{Ts}, visited::Set{AbstractBDDNode{Ts}}, io::IO)::Nothing where Ts
    if in(f, visited)
        return
    end
    println(io, "\"obj$(f.id)\" [shape = square, label = \"0\"];")
    push!(visited, f)
    nothing
end

function _todot!(b::BDD2{Ts}, f::BDD2Node{Ts}, visited::Set{AbstractBDDNode{Ts}}, io::IO)::Nothing where Ts
    if in(f, visited)
        return
    end

    println(io, "\"obj$(f.id)\" [shape = circle, label = \"$(f.header.label)\"];")
    _todot!(b, f.low, visited, io)
    println(io, "\"obj$(f.id)\" -> \"obj$(f.low.id)\" [label = \"0\"];")
    _todot!(b, f.high, visited, io)
    if f.neg == false
        println(io, "\"obj$(f.id)\" -> \"obj$(f.high.id)\" [label = \"1\"];")
    else
        println(io, "\"obj$(f.id)\" -> \"obj$(f.high.id)\" [label = \"1\", arrowhead = odot];")
    end
    push!(visited, f)
    nothing
end


"""
    OpConnectionIdentity

An iterable structure containing all the changes from one state to another, and
an associated value.
This effectively encodes a full row of an operator.
"""
struct OpConnection{A<:AbstractArray,B<:AbstractArray,C<:AbstractArray} <: AbsOpConnection
    mel::A
    changes::Vector{StateChanges{B,C}}

    length::Ref{Int}
end

function OpConnection(op::AbsLinearOperator)
    t = conn_type(op)
    v1, v2, v3 = t.parameters
    return OpConnection(v1(), Vector{StateChanges{v2,v3}}(), Ref(0))
end

function OpConnection{A,B,C}() where {A,B,C}
    return OpConnection(A(), Vector{typeof(StateChanges(B(),C()))}(), Ref(0))
end

function Base.resize!(c::OpConnection, n)
    # diminish size
    if n <= length(c)
        c.length[] = n
    else
        # increase size
        c.length[] = n
        resize!(c.mel,     n)
        resize!(c.changes, n)
    end

    return c
end

Base.eltype(::Type{OpConnection{A,B,C}}) where {A,B,C} = Tuple{eltype(A), StateChanges{B,C}}
Base.length(c::OpConnection) = c.length[]
Base.size(c::OpConnection) = (length(c), )
capacity(c::OpConnection) = length(c.mel)

Base.:(==)(a::OpConnection, b::OpConnection) = (a.mel == b.mel &&
                                              a.changes == b.changes)

@inline Base.push!(c::OpConnection, (m_els, to_change, new_values)) =
    push!(c, (m_els, StateChanges(to_change, new_values)))

function Base.push!(c::OpConnection, (mel, cngs)::Tuple{Number, StateChanges})
    if length(c) < capacity(c)
        resize!(c, length(c)+1)
        @inbounds setindex!(c, length(c), (mel, cngs))
    else
        c.length[] += 1
        push!(c.mel, mel)
        push!(c.changes, cngs)
    end
    return c
end

@inline function Base.setindex!(c::OpConnection, i, (mel, cngs)::Tuple{Number, StateChanges})
    @boundscheck checkbounds(1:length(c), i)
    @inbounds c.mel[i]     = mel
    @inbounds c.changes[i] = cngs
end

function Base.append!(c::OpConnection, (m_els, to_change, new_values)::Tuple{Vector, Vector, Vector})
    cngs = Vector{StateChanges{eltype(to_change), eltype(new_values)}}()
    for (tcng, nwvls) = zip(to_change, new_values)
        push!(cngs, StateChanges(tcng, nwvls))
    end
    return append!(c, (m_els, cngs))
end

function Base.append!(c::OpConnection, (m_els, cngs)::Tuple)
    l = length(c)
    resize!(c, l+length(m_els))

    for i=1:length(m_els)
        @inbounds setindex!(c, l+i, (m_els[i], cngs[i]))
    end
    return c
end

function Base.append!(c::OpConnection, new::OpConnection)
    L = length(new)

    L == 0 && return c

    # If this accumulator is empty, then simply append
    if length(c) == 0
        append!(c, (new.mel, new.changes))
    else
        # We assume that the first element is always for len(statechanges) = 0
        c.mel[1] += new.mel[1]

        L == 1 && return c

        append!(c, (uview(new.mel, 2:L), uview(new.changes, 2:L )))
    end

    return c
end

Base.@propagate_inbounds Base.getindex(c::OpConnection, i) = (c.mel[i], c.changes[i])

function Base.iterate(iter::OpConnection, state=1)
    state > length(iter) && return nothing
    @inbounds conn = iter[state]
    return (conn, state + 1)
end

function add!(c::OpConnection, mel, cngs::StateChanges)
    i = findfirst(isequal(cngs), c.changes)
    isnothing(i) ? push!(c, (mel, cngs)) : c.mel[i] += mel
    return c
end

function add!(c::OpConnection, o::OpConnection)
    for (mel, cngs)=o
        add!(c, mel, cngs)
    end
end

Base.conj!(op::OpConnection) = conj!(op.mel)

function clear_duplicates(c::OpConnection)
    u_cngs = unique(c.changes)
    u_mel  = similar(c.mel, 0)
    for el=u_cngs
        ids = findall(isequal(el), c.changes)
        push!(u_mel, sum(c.mel[ids]))
    end
    return OpConnection(u_mel, u_cngs)
end

#showing
function Base.show(io::IO, c::OpConnection{A,B,C}) where {A,B,C}
    print(io, "$(length(c))-elements - OpConnection{$(eltype(A)),StateChanges{$B,$C}}:\n")
    Base.print_matrix(IOContext(io, :compact=>true), collect(c))
    return io
end


function DiagonalMatrix(arg0::Vector{jdouble})
    return DiagonalMatrix((Vector{jdouble},), arg0)
end

function DiagonalMatrix(arg0::Vector{jdouble}, arg1::jboolean)
    return DiagonalMatrix((Vector{jdouble}, jboolean), arg0, arg1)
end

function DiagonalMatrix(arg0::jint)
    return DiagonalMatrix((jint,), arg0)
end

function add(obj::DiagonalMatrix, arg0::DiagonalMatrix)
    return jcall(obj, "add", DiagonalMatrix, (DiagonalMatrix,), arg0)
end

function add_to_entry(obj::DiagonalMatrix, arg0::jint, arg1::jint, arg2::jdouble)
    return jcall(obj, "addToEntry", void, (jint, jint, jdouble), arg0, arg1, arg2)
end

function copy(obj::DiagonalMatrix)
    return jcall(obj, "copy", RealMatrix, ())
end

function create_matrix(obj::DiagonalMatrix, arg0::jint, arg1::jint)
    return jcall(obj, "createMatrix", RealMatrix, (jint, jint), arg0, arg1)
end

function get_column_dimension(obj::DiagonalMatrix)
    return jcall(obj, "getColumnDimension", jint, ())
end

function get_data(obj::DiagonalMatrix)
    return jcall(obj, "getData", Vector{Vector{jdouble}}, ())
end

function get_data_ref(obj::DiagonalMatrix)
    return jcall(obj, "getDataRef", Vector{jdouble}, ())
end

function get_entry(obj::DiagonalMatrix, arg0::jint, arg1::jint)
    return jcall(obj, "getEntry", jdouble, (jint, jint), arg0, arg1)
end

function get_row_dimension(obj::DiagonalMatrix)
    return jcall(obj, "getRowDimension", jint, ())
end

function inverse(obj::DiagonalMatrix)
    return jcall(obj, "inverse", DiagonalMatrix, ())
end

function inverse(obj::DiagonalMatrix, arg0::jdouble)
    return jcall(obj, "inverse", DiagonalMatrix, (jdouble,), arg0)
end

function is_singular(obj::DiagonalMatrix, arg0::jdouble)
    return jcall(obj, "isSingular", jboolean, (jdouble,), arg0)
end

function multiply(obj::DiagonalMatrix, arg0::DiagonalMatrix)
    return jcall(obj, "multiply", DiagonalMatrix, (DiagonalMatrix,), arg0)
end

function multiply(obj::DiagonalMatrix, arg0::RealMatrix)
    return jcall(obj, "multiply", RealMatrix, (RealMatrix,), arg0)
end

function multiply_entry(obj::DiagonalMatrix, arg0::jint, arg1::jint, arg2::jdouble)
    return jcall(obj, "multiplyEntry", void, (jint, jint, jdouble), arg0, arg1, arg2)
end

function multiply_transposed(obj::DiagonalMatrix, arg0::DiagonalMatrix)
    return jcall(obj, "multiplyTransposed", DiagonalMatrix, (DiagonalMatrix,), arg0)
end

function multiply_transposed(obj::DiagonalMatrix, arg0::RealMatrix)
    return jcall(obj, "multiplyTransposed", RealMatrix, (RealMatrix,), arg0)
end

function operate(obj::DiagonalMatrix, arg0::Vector{jdouble})
    return jcall(obj, "operate", Vector{jdouble}, (Vector{jdouble},), arg0)
end

function pre_multiply(obj::DiagonalMatrix, arg0::RealVector)
    return jcall(obj, "preMultiply", RealVector, (RealVector,), arg0)
end

function pre_multiply(obj::DiagonalMatrix, arg0::Vector{jdouble})
    return jcall(obj, "preMultiply", Vector{jdouble}, (Vector{jdouble},), arg0)
end

function set_entry(obj::DiagonalMatrix, arg0::jint, arg1::jint, arg2::jdouble)
    return jcall(obj, "setEntry", void, (jint, jint, jdouble), arg0, arg1, arg2)
end

function subtract(obj::DiagonalMatrix, arg0::DiagonalMatrix)
    return jcall(obj, "subtract", DiagonalMatrix, (DiagonalMatrix,), arg0)
end

function transpose_multiply(obj::DiagonalMatrix, arg0::DiagonalMatrix)
    return jcall(obj, "transposeMultiply", DiagonalMatrix, (DiagonalMatrix,), arg0)
end

function transpose_multiply(obj::DiagonalMatrix, arg0::RealMatrix)
    return jcall(obj, "transposeMultiply", RealMatrix, (RealMatrix,), arg0)
end


using CompilerPluginTools: rm_code_coverage_effect
using IBMQClient.Schema
using YaoCompiler
using YaoCompiler.Intrinsics
using YaoTargetQobj
using Test

@device function basic()
    1 => X
    2 => Y
    3 => Z
    4 => H
    4 => T
    4 => S
    4 => Rx(1.2)
    4 => Ry(1.2)
    4 => Rz(1.2)
    4 => shift(1.2)
end

@testset "basic" begin
    ci, type = @yao_code_typed(basic())[1]
    ci = rm_code_coverage_effect(ci)
    @test measure_ssa_uses(ci) == Set{Int}()
    mi = MemoryInfo(ci)
    display(mi)
    @test mi.qubits == Dict(1=>0, 2=>1, 3=>2, 4=>3)
    @test mi.n_qubits == 4

    @test code_qobj(typeof(basic())) == Experiment(;
        header = Dict{String, Any}(
            "description" => "Operation{typeof(basic), Tuple{}}",
            "n_qubits" => 4,
        ),
        config = ExpConfig(;
            memory_slots = 0,
            seed = 1,
            max_credits = 3,
        ),
        instructions = [
            Gate(;
                name = "x",
                qubits = [0],
            ),
            Gate(;
                name = "y",
                qubits = [1],
            ),
            Gate(;
                name = "z",
                qubits = [2],
            ),
            Gate(;
                name = "h",
                qubits = [3],
            ),
            Gate(;
                name = "t",
                qubits = [3],
            ),
            Gate(;
                name = "s",
                qubits = [3],
            ),
            Gate(;
                name = "rx",
                qubits = [3],
                params = [1.2],
            ),
            Gate(;
                name = "ry",
                qubits = [3],
                params = [1.2],
            ),
            Gate(;
                name = "rz",
                qubits = [3],
                params = [1.2],
            ),
            Gate(;
                name = "u1",
                qubits = [3],
                params = [1.2],
            ),
        ],
    )
end

@device function just_measure()
    1 => H
    c = @measure 1
    return c
end

@testset "just_measure" begin
    ci, type = @yao_code_typed(just_measure())[1]
    ci = rm_code_coverage_effect(ci)
    @test measure_ssa_uses(ci) == Set{Int}()
    mi = MemoryInfo(ci)
    display(mi)
    @test isempty(mi.registers)

    @test code_qobj(typeof(just_measure())) == Experiment(;
        header = Dict{String, Any}(
            "description" => "Operation{typeof(just_measure), Tuple{}}",
            "n_qubits" => 1,
        ),
        config = ExpConfig(;
            memory_slots = 1,
            seed = 1,
            max_credits = 3,
        ),
        instructions = [
            Gate(;
                name = "h",
                qubits = [0],
            ),
            Measure(;
                qubits = [0],
                memory = [0],
            ),
        ],
    )
end

@device function reset_qubit()
    1 => H
    c = @measure 1
    if c == 1
        1 => X
    end
    return
end

@testset "reset_qubit" begin
    ci, type = @yao_code_typed(reset_qubit())[1]
    ci = rm_code_coverage_effect(ci)
    @test measure_ssa_uses(ci) == Set(2)
    @test measure_ssa_returns(ci) == Set()

    mi = MemoryInfo(ci)
    display(mi)
    @test mi.registers == Dict(2=>[0])
    @test mi.n_qubits == 1
    @test mi.qubits == Dict(1=>0)
    @test code_qobj(typeof(reset_qubit())) == Experiment(;
        header = Dict{String, Any}(
            "description" => "Operation{typeof(reset_qubit), Tuple{}}",
            "n_qubits" => 1,
        ),
        config = ExpConfig(;
            memory_slots = 0,
            seed = 1,
            max_credits = 3,
        ),
        instructions = [
            Gate(;
                name = "h",
                qubits = [0],
            ),
            Reset(;
                qubits = [0],
            ),
        ],
    )
end

@device function reset_qubit_return()
    1 => H
    c = @measure 1
    if c == 1
        1 => X
    end
    return c
end

@testset "reset_qubit_return" begin
    ci, type = @yao_code_typed(reset_qubit_return())[1]
    ci = rm_code_coverage_effect(ci)
    @test measure_ssa_uses(ci) == Set(2)
    # @test measure_ssa_returns(ci) == Set(2)
    @test code_qobj(typeof(reset_qubit_return())) == Experiment(;
        header = Dict{String, Any}(
            "description" => "Operation{typeof(reset_qubit_return), Tuple{}}",
            "n_qubits" => 1,
        ),
        config = ExpConfig(;
            memory_slots = 1,
            seed = 1,
            max_credits = 3,
        ),
        instructions = [
            Gate(;
                name = "h",
                qubits = [0],
            ),
            Measure(;
                qubits = [0],
                memory = [0],
                register = [0],
            ),
            Gate(;
                name = "x",
                qubits = [0],
                conditional = 0,
            ),
        ],
    )
end

@device function reset_with_return()
    1 => H
    m1 = @measure 2
    m2 = @measure 1
    if m1 == 1
        1 => X
    end
    return m1, m2
end

@testset "reset_with_return" begin
    ci, type = @yao_code_typed(reset_with_return())[1]
    ci = rm_code_coverage_effect(ci)
    @test measure_ssa_uses(ci) == Set(2)
    # @test measure_ssa_returns(ci) == Set([2, 3])
    mi = MemoryInfo(ci)
    @test mi.n_qubits == 2
    @test mi.registers == Dict(2=>[0])
    @test mi.qubits == Dict(1=>0, 2=>1)
    @test code_qobj(typeof(reset_with_return())) == Experiment(;
        header = Dict{String, Any}(
            "description" => "Operation{typeof(reset_with_return), Tuple{}}",
            "n_qubits" => 2,
        ),
        config = ExpConfig(;
            memory_slots = 2,
            seed = 1,
            max_credits = 3,
        ),
        instructions = [
            Gate(;
                name = "h",
                qubits = [0],
            ),
            Measure(;
                qubits = [1],
                memory = [0],
                register = [0],
            ),
            Measure(;
                qubits = [0],
                memory = [1],
            ),
            Gate(;
                name = "x",
                qubits = [0],
                conditional = 0,
            ),
        ],
    )
end

@device function multi_qubit_measure()
    1 => H
    3 => X
    c = @measure (1, 2)
    return (c=c)
end

@testset "multi_qubit_measure" begin
    ci, type = @yao_code_typed(multi_qubit_measure())[1]
    ci = rm_code_coverage_effect(ci)
    mi = MemoryInfo(ci)
    @test mi.n_qubits == 3
    @test isempty(mi.registers)
    @test mi.qubits == Dict(1=>0, 2=>2, 3=>1)
    @test code_qobj(typeof(multi_qubit_measure())) == Experiment(;
        header = Dict{String, Any}(
            "description" => "Operation{typeof(multi_qubit_measure), Tuple{}}",
            "n_qubits" => 3,
        ),
        config = ExpConfig(;
            memory_slots = 2,
            seed = 1,
            max_credits = 3,
        ),
        instructions = [
            Gate(;
                name = "h",
                qubits = [0],
            ),
            Gate(;
                name = "x",
                qubits = [1],
            ),
            Measure(;
                qubits = [0, 2],
                memory = [0, 1],
            ),
        ],
    )
end

@device function multi_if_measure()
    1 => H
    3 => X
    c = @measure 1
    if c == 1
        1 => Y
        2 => Z
        3 => H
    end
    return
end

@testset "multi if stmt" begin
    @test code_qobj(typeof(multi_if_measure())) == Experiment(;
        header = Dict{String, Any}(
            "description" => "Operation{typeof(multi_if_measure), Tuple{}}",
            "n_qubits" => 3,
        ),
        config = ExpConfig(;
            memory_slots = 1,
            seed = 1,
            max_credits = 3,
        ),
        instructions = [
            Gate(;
                name = "h",
                qubits = [0],
            ),
            Gate(;
                name = "x",
                qubits = [1],
            ),
            Measure(;
                qubits = [0],
                memory = [0],
                register = [0],
            ),
            Gate(;
                name = "y",
                qubits = [0],
                conditional = 0,
            ),
            Gate(;
                name = "z",
                qubits = [2],
                conditional = 0,
            ),
            Gate(;
                name = "h",
                qubits = [1],
                conditional = 0,
            ),
        ],
    )
end

@device function conditional_ctrl()
    1 => H
    3 => X
    c = @measure 1
    if c == 1
        @ctrl 2 1 => Y
    end
    return
end

@testset "conditional ctrl" begin
    @test code_qobj(typeof(conditional_ctrl())) == Experiment(;
        header = Dict{String, Any}(
            "description" => "Operation{typeof(conditional_ctrl), Tuple{}}",
            "n_qubits" => 3,
        ),
        config = ExpConfig(;
            memory_slots = 1,
            seed = 1,
            max_credits = 3,
        ),
        instructions = [
            Gate(;
                name = "h",
                qubits = [0],
            ),
            Gate(;
                name = "x",
                qubits = [1],
            ),
            Measure(;
                qubits = [0],
                memory = [0],
                register = [0],
            ),
            Gate(;
                name = "cy",
                qubits = [0, 2],
                conditional = 0,
            ),
        ],
    )
end

@device function test_barrier()
    1 => H
    3 => X
    @barrier 1,3
    c = @measure 1
    if c == 1
        @ctrl 2 1 => Y
    end
    return
end

@testset "test_barrier" begin
    @test code_qobj(typeof(test_barrier())) == Experiment(;
        header = Dict{String, Any}(
            "description" => "Operation{typeof(test_barrier), Tuple{}}",
            "n_qubits" => 3,
        ),
        config = ExpConfig(;
            memory_slots = 1,
            seed = 1,
            max_credits = 3,
        ),
        instructions = [
            Gate(;
                name = "h",
                qubits = [0],
            ),
            Gate(;
                name = "x",
                qubits = [1],
            ),
            Barrier(;
                qubits = [0, 1],
            ),
            Measure(;
                qubits = [0],
                memory = [0],
                register = [0],
            ),
            Gate(;
                name = "cy",
                qubits = [0, 2],
                conditional = 0,
            ),
        ],
    )
end

@device function test_ctrl()
    @ctrl 4 1 => H
    @ctrl 4 3 => X
    @ctrl 4 3 => Y
    @ctrl 4 3 => Z
    @ctrl (4,5) 3 => X
    @ctrl 4 3 => Rx(2.1)
    @ctrl 4 3 => Ry(2.1)
    @ctrl 4 3 => Rz(2.1)
    @ctrl 4 3 => shift(2.1)
    @ctrl 4 3 => Intrinsics.UGate(1.2, 2.2, 3.3)
    return
end

@testset "test_ctrl" begin
    @test code_qobj(typeof(test_ctrl())) == Experiment(;
        header = Dict{String, Any}(
            "description" => "Operation{typeof(test_ctrl), Tuple{}}",
            "n_qubits" => 4,
        ),
        config = ExpConfig(;
            memory_slots = 0,
            seed = 1,
            max_credits = 3,
        ),
        instructions = [
            Gate(;
                name = "ch",
                qubits = [0, 1],
            ),
            Gate(;
                name = "cx",
                qubits = [2, 1],
            ),
            Gate(;
                name = "cy",
                qubits = [2, 1],
            ),
            Gate(;
                name = "cz",
                qubits = [2, 1],
            ),
            Gate(;
                name = "ccx",
                qubits = [2, 1, 3],
            ),
            Gate(;
                name = "crx",
                qubits = [2, 1],
                params = [2.1],
            ),
            Gate(;
                name = "cry",
                qubits = [2, 1],
                params = [2.1],
            ),
            Gate(;
                name = "crz",
                qubits = [2, 1],
                params = [2.1],
            ),
            Gate(;
                name = "cu1",
                qubits = [2, 1],
                params = [2.1],
            ),
            Gate(;
                name = "cu3",
                qubits = [2, 1],
                params = [1.2, 2.2, 3.3],
            ),
        ],
    )
end


using Revise, Parameters, Logging, Test
using CUDA
includet("C:\\GitHub\\GitHub\\NuclearMedEval\\test\\includeAllUseFullForTest.jl")
using ..CUDAGpuUtils ,..IterationUtils,..ReductionUtils , ..MemoryUtils,..CUDAAtomicUtils
using ..ResultListUtils, ..MetadataAnalyzePass,..MetaDataUtils,..WorkQueueUtils,..ProcessMainDataVerB,..HFUtils, ..ScanForDuplicates

resList = allocateResultList(100,100);
ind= (1,2,3)
getResLinIndex([ind[1] ind[2] ind[3] 1 1 1 ; 0 0 0 0 0 0],1, (200,200,200))

carts = CartesianIndices(zeros(200,200,200))

mappedA = map(ind-> getResLinIndex(ind[1],ind[2],ind[3],1, (200,200,200)) ,carts)
mappedB = map(ind-> getResLinIndex(ind[1],ind[2],ind[3],0, (200,200,200)) ,carts)

@test length(unique(mappedA))== 200*200*200
@test length(unique(vec(vcat(mappedA,mappedB ))))== 200*200*200*2

#################  addResult
using Revise, Parameters, Logging, Test
using CUDA
includet("C:\\GitHub\\GitHub\\NuclearMedEval\\test\\includeAllUseFullForTest.jl")
using ..CUDAGpuUtils ,..IterationUtils,..ReductionUtils , ..MemoryUtils,..CUDAAtomicUtils
using ..MetadataAnalyzePass,..MetaDataUtils,..WorkQueueUtils,..ProcessMainDataVerB,..HFUtils,..ResultListUtils

threads=(32,5);
blocks =2;
mainArrDims= (67,177,90);
dataBdim = (43,21,17);
mainArrCPU= falses(mainArrDims);
mainArrCPU[5,5,5]= true;
mainArrCPU[33,10,7]= true;
mainArrGPU = CuArray(mainArrCPU);
metaData = MetaDataUtils.allocateMetadata(mainArrDims,dataBdim);
metaDataDims= size(metaData);
totalFp,totalFn= 500,500
resList,resListIndicies,maxResListIndex= allocateResultLists(totalFp,totalFn)

xMeta,yMeta,zMeta= 1,1,1 #aaume 0 based
x,y,z = 1,2,3
dir= 5
queueNumber = 1
isGold=4
iterNumb=6
metaData[xMeta+1,yMeta+1,zMeta+1,getResOffsetsBeg()+queueNumber]=7
function addResKernel(metaData ,xMeta,yMeta,zMeta, resList,resListIndicies,maxResListIndex,x,y,z, dir,iterNumb,queueNumber,metaDataDims ,mainArrDims,isGold )
    @ifXY 1 1 @addResult(metaData ,xMeta,yMeta,zMeta, resList,resListIndicies,maxResListIndex,x,y,z, dir,iterNumb,queueNumber,metaDataDims,mainArrDims ,isGold  )
    @ifXY 2 1 @addResult(metaData ,xMeta,yMeta,zMeta, resList,resListIndicies,maxResListIndex,x,y,z, dir,iterNumb,queueNumber,metaDataDims ,mainArrDims,isGold  )
    @ifXY 3 1 @addResult(metaData ,xMeta,yMeta,zMeta, resList,resListIndicies,maxResListIndex,x,y,z, dir,iterNumb,queueNumber,metaDataDims,mainArrDims ,isGold  )
    return
end

@cuda threads=threads blocks=blocks addResKernel(metaData ,xMeta,yMeta,zMeta, resList,resListIndicies,maxResListIndex,x,y,z, dir,iterNumb,queueNumber,metaDataDims ,mainArrDims,isGold )
@test length(filter(it-> it>0,Array(resListIndicies)))==6

# @test  Int64.(Array(resList[7,:]))==[1,2,3,4,5,6]
@test  Int64.(Array(resList[8,:]))==[1,2,3,4,5,6]
@test  Int64.(Array(resList[9,:]))==[1,2,3,4,5,6]
@test  Int64.(Array(resList[10,:]))==[1,2,3,4,5,6]
@test  Int64.(Array(resList[11,:]))==[1,2,3,4,5,6]
@test  Int64.(Array(resList[12,:]))==[1,2,3,4,5,6]
@test  Int64.(Array(resList[14,:]))==[0,0,0,0,0,0]

Int64.(Array(resListIndicies[5:15]))

filter(it-> it>0,Array(resListIndicies))
Int64(sum(resList))


"""
GenX: An Configurable Capacity Expansion Model
Copyright (C) 2021,  Massachusetts Institute of Technology
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.
A complete copy of the GNU General Public License v2 (GPLv2) is available
in LICENSE.txt.  Users uncompressing this from an archive may not have
received this license file.  If not, see <http://www.gnu.org/licenses/>.
"""

@doc raw"""
	load_reserves(setup::Dict,path::AbstractString, inputs_res::Dict)

Function for reading input parameters related to frequency regulation and operating reserve requirements
"""
function load_reserves(setup::Dict,path::AbstractString, inputs_res::Dict)

	res_in = DataFrame(CSV.File(joinpath(path, "Reserves.csv"), header=true), copycols=true)

	# Regulation requirement as a percent of hourly load; here load is the total across all model zones
	inputs_res["pReg_Req_Load"] = convert(Float64, res_in[!,:Reg_Req_Percent_Load][1])
	# Regulation requirement as a percent of hourly wind and solar generation (summed across all model zones)
	inputs_res["pReg_Req_VRE"] = convert(Float64, res_in[!,:Reg_Req_Percent_VRE][1])
	# Spinning up reserve requirement as a percent of hourly load (which is summed across all zones)
	inputs_res["pRsv_Req_Load"] = convert(Float64, res_in[!,:Rsv_Req_Percent_Load][1])
	# Spinning up reserve requirement as a percent of hourly wind and solar generation (which is summed across all zones)
	inputs_res["pRsv_Req_VRE"] = convert(Float64, res_in[!,:Rsv_Req_Percent_VRE][1])

	if setup["ParameterScale"] == 1  # Parameter scaling turned on - adjust values of subset of parameter values
		# Penalty for not meeting hourly spinning reserve requirement
		inputs_res["pC_Rsv_Penalty"] = convert(Float64, res_in[!,:Unmet_Rsv_Penalty_Dollar_per_MW][1])/ModelScalingFactor # convert to million $/GW with objective function in millions
		inputs_res["pStatic_Contingency"] = convert(Float64, res_in[!,:Static_Contingency_MW][1])/ModelScalingFactor # convert to GW
	else
		# Penalty for not meeting hourly spinning reserve requirement
		inputs_res["pC_Rsv_Penalty"] = convert(Float64, res_in[!,:Unmet_Rsv_Penalty_Dollar_per_MW][1])
		inputs_res["pStatic_Contingency"] = convert(Float64, res_in[!,:Static_Contingency_MW][1])
	end
	if setup["UCommit"] >= 1
		inputs_res["pDynamic_Contingency"] = convert(Int8, res_in[!,:Dynamic_Contingency][1] )
		# Set BigM value used for dynamic contingencies cases to be largest possible cluster size
		# Note: this BigM value is only relevant for units in the COMMIT set. See reserves.jl for details on implementation of dynamic contingencies
		if inputs_res["pDynamic_Contingency"] > 0
			inputs_res["pContingency_BigM"] = zeros(Float64, inputs_res["G"])
			for y in inputs_res["COMMIT"]
				inputs_res["pContingency_BigM"][y] = inputs_res["dfGen"][!,:Max_Cap_MW][y]
				# When Max_Cap_MW == -1, there is no limit on capacity size
				if inputs_res["pContingency_BigM"][y] < 0
					# NOTE: this effectively acts as a maximum cluster size when not otherwise specified, adjust accordingly
					inputs_res["pContingency_BigM"][y] = 5000*inputs_res["dfGen"][!,:Cap_Size][y]
				end
			end
		end
	end

	println("Reserves.csv Successfully Read!")

	return inputs_res
end


"""
Sets the tolerance for the iLQR solver to the intermediate tolerances, until the last
outer loop iteration.
"""
function set_tolerances!(solver::ALSolver{T}, i::Int, cost_tol=solver.opts.cost_tolerance,
                         grad_tol=solver.opts.gradient_tolerance) where T
    opts = options(solver)
    if i != solver.opts.iterations_outer
        opts.cost_tolerance = opts.cost_tolerance_intermediate
        opts.gradient_tolerance = opts.gradient_tolerance_intermediate
    else
        opts.cost_tolerance = cost_tol
        opts.gradient_tolerance = grad_tol
    end

    return nothing
end

function solve!(solver::ALSolver)
    reset!(solver)

    conset = get_constraints(solver)
    if !is_constrained(solver)
        ilqr = get_ilqr(solver)
        solve!(ilqr)
        terminate!(solver)
        return solver
    end

    # Terminal tolerances
    cost_tol = solver.opts.cost_tolerance
    grad_tol = solver.opts.gradient_tolerance

    Z̄ = solver.ilqr.Z̄
    for al_iter = 1:solver.opts.iterations_outer
        # Set potentially looser tolerances for inner iLQR solve
        set_tolerances!(solver, al_iter, cost_tol, grad_tol)

        # Solve problem with iLQR
        solve!(solver.ilqr)

        # Check solver status
        status(solver) > SOLVE_SUCCEEDED && break

        # Evaluate the constraints and the cost for the current trajectory
        J = TO.cost(solver, Z̄)  # NOTE: this evaluates the constraints
        c_max = max_violation(conset)
        μ_max = max_penalty(conset)
        record_iteration!(solver, J, c_max, μ_max)

        # Check if it's converged
        isconverged = evaluate_convergence(solver)
        if isconverged
            break
        end

        # Outer loop updates
        dualupdate!(conset)
        penaltyupdate!(conset)

        # Reset iLQR solver
        # TODO: is this necessary? it gets reset at the beginning of the iLQR solve method
        reset!(solver.ilqr)
    end
    # Reset tolerances to their original values
    solver.opts.cost_tolerance = cost_tol
    solver.opts.gradient_tolerance = grad_tol
    terminate!(solver)
    return solver
end

function record_iteration!(solver::ALSolver, J, c_max, μ_max)
    stats = solver.stats
    record_iteration!(stats, c_max=c_max, penalty_max=μ_max, is_outer=true)
    lg = getlogger(solver)
    @log lg "iter" stats.iterations
    @log lg "AL iter" stats.iterations_outer
    @log lg "cost" J
    @log lg "||v||" c_max
    @log lg μ_max
end

function evaluate_convergence(solver::ALSolver)
    lg = getlogger(solver)
    iter = solver.stats.iterations
    isconverged = false
    if solver.stats.c_max[iter] < solver.opts.constraint_tolerance
        @log lg "info" "Constraint tolerance met."
        solver.stats.status = SOLVE_SUCCEEDED
        isconverged = true
    end
    if solver.opts.kickout_max_penalty && solver.stats.penalty_max[i] >= solver.opts.penalty_max
        @log lg "info" "Hit max penalty."
        isconverged = true
    end
    if iter >= solver.opts.iterations
        @log lg "info" "Hit max iterations."
        solver.stats.status = MAX_ITERATIONS
        isconverged = true
    end
    if solver.stats.iterations_outer >= solver.opts.iterations_outer
        @log lg "info" "Hit max AL iterations."
        solver.stats.status = MAX_ITERATIONS_OUTER
        isconverged = true
    end
    return isconverged
end