struct Instruction
    mod::Symbol
    instr::Symbol
end
Instruction(instr::Symbol) = Instruction(:LoopVectorization, instr)
Base.convert(::Type{Instruction}, instr::Symbol) = Instruction(instr)
lower(instr::Instruction) = Expr(:(.), instr.mod, QuoteNode(instr.instr))
Base.Expr(instr::Instruction, args...) = Expr(:call, lower(instr), args...)::Expr
Base.hash(instr::Instruction, h::UInt64) = hash(instr.instr, hash(instr.mod, h))

const LOOPCONSTANT = Instruction(gensym())

struct InstructionCost
    scaling::Float64 # sentinel values: -3 == no scaling; -2 == offset_scaling, -1 == linear scaling, >0 ->  == latency == reciprical throughput
    scalar_reciprical_throughput::Float64
    scalar_latency::Int
    register_pressure::Int
end
InstructionCost(sl::Int, srt::Float64, scaling::Float64 = -3.0) = InstructionCost(scaling, srt, sl, 0)

nocost(c::InstructionCost) = c.scalar_reciprical_throughput == 0.0
flatcost(c::InstructionCost) = c.scaling == -3.0
offsetscaling(c::InstructionCost) = c.scaling == -2.0
linearscaling(c::InstructionCost) = c.scaling == -1.0

function scalar_cost(ic::InstructionCost)#, ::Type{T} = Float64) where {T}
    @unpack scalar_reciprical_throughput, scalar_latency, register_pressure = ic
    scalar_reciprical_throughput, scalar_latency, register_pressure
end
function vector_cost(ic::InstructionCost, Wshift, sizeof_T)
    srt, sl, srp = scalar_cost(ic)
    if flatcost(ic) || Wshift == 0 || nocost(ic) # No scaling
        return srt, sl, srp
    elseif offsetscaling(ic) # offset scaling
        srt *= 1 << (Wshift + VectorizationBase.intlog2(sizeof_T) - 4)
        if (sizeof_T << Wshift) == 64 # VectorizationBase.REGISTER_SIZE # These instructions experience double latency with zmm
            sl += sl
        end
    elseif linearscaling(ic) # linear scaling
        W = 1 << Wshift
        extra_latency = sl - srt
        srt *= W
        sl = round(Int, srt + extra_latency)
    else # we assume custom cost, and that latency == recip_throughput
        scaling = ic.scaling
        sl, srt = round(Int,scaling), scaling
    end
    srt, sl, srp
end
instruction_cost(instruction::Instruction) = get(COST, instruction, OPAQUE_INSTRUCTION)
instruction_cost(instruction::Symbol) = instruction_cost(Instruction(instruction))
scalar_cost(instr::Instruction) = scalar_cost(instruction_cost(instr))
vector_cost(instr::Instruction, Wshift, sizeof_T) = vector_cost(instruction_cost(instr), Wshift, sizeof_T)
function cost(instruction::InstructionCost, Wshift, sizeof_T)
    Wshift == 0 ? scalar_cost(instruction) : vector_cost(instruction, Wshift, sizeof_T)
end

function cost(instruction::Instruction, Wshift, sizeof_T)
    cost(
        instruction_cost(instruction), Wshift, sizeof_T
    )
end

# Just a semi-reasonable assumption; should not be that sensitive to anything other than loads
const OPAQUE_INSTRUCTION = InstructionCost(50, 50.0, -1.0, VectorizationBase.REGISTER_COUNT)

# Comments on setindex!
# 1. Not a part of dependency chains, so not really twice as expensive as getindex?
# 2. getindex loads a register, not setindex!, but we place cost on setindex!
#    as a heuristic means of approximating register pressure, since many loads can be
#    consolidated into a single register. The number of LICM-ed setindex!, on the other
#    hand, should indicate how many registers we're keeping live for the sake of eventually storing.
const COST = Dict{Instruction,InstructionCost}(
    Instruction(:getindex) => InstructionCost(-3.0,0.5,3,1),
    Instruction(:setindex!) => InstructionCost(-3.0,1.0,3,0),
    Instruction(:zero) => InstructionCost(1,0.5),
    Instruction(:one) => InstructionCost(3,0.5),
    Instruction(:+) => InstructionCost(4,0.5),
    Instruction(:-) => InstructionCost(4,0.5),
    Instruction(:*) => InstructionCost(4,0.5),
    Instruction(:/) => InstructionCost(13,4.0,-2.0),
    Instruction(:vadd) => InstructionCost(4,0.5),
    Instruction(:vsub) => InstructionCost(4,0.5),
    Instruction(:vmul) => InstructionCost(4,0.5),
    Instruction(:vdiv) => InstructionCost(13,4.0,-2.0),
    Instruction(:abs2) => InstructionCost(4,0.5),
    Instruction(:vabs2) => InstructionCost(4,0.5),
    Instruction(:(==)) => InstructionCost(1, 0.5),
    Instruction(:isequal) => InstructionCost(1, 0.5),
    Instruction(:(&)) => InstructionCost(1, 0.5),
    Instruction(:(|)) => InstructionCost(1, 0.5),
    Instruction(:(>)) => InstructionCost(1, 0.5),
    Instruction(:(<)) => InstructionCost(1, 0.5),
    Instruction(:(>=)) => InstructionCost(1, 0.5),
    Instruction(:(<=)) => InstructionCost(1, 0.5),
    Instruction(:inv) => InstructionCost(13,4.0,-2.0,1),
    Instruction(:vinv) => InstructionCost(13,4.0,-2.0,1),
    Instruction(:muladd) => InstructionCost(4,0.5), # + and * will fuse into this, so much of the time they're not twice as expensive
    Instruction(:fma) => InstructionCost(4,0.5), # + and * will fuse into this, so much of the time they're not twice as expensive
    Instruction(:vmuladd) => InstructionCost(4,0.5), # + and * will fuse into this, so much of the time they're not twice as expensive
    Instruction(:vfma) => InstructionCost(4,0.5), # + and * will fuse into this, so much of the time they're not twice as expensive
    Instruction(:vfmadd) => InstructionCost(4,0.5), # + and * will fuse into this, so much of the time they're not twice as expensive
    Instruction(:vfmsub) => InstructionCost(4,0.5), # - and * will fuse into this, so much of the time they're not twice as expensive
    Instruction(:vfnmadd) => InstructionCost(4,0.5), # + and -* will fuse into this, so much of the time they're not twice as expensive
    Instruction(:vfnmsub) => InstructionCost(4,0.5), # - and -* will fuse into this, so much of the time they're not twice as expensive
    Instruction(:vfmadd_fast) => InstructionCost(4,0.5), # + and * will fuse into this, so much of the time they're not twice as expensive
    Instruction(:vfmsub_fast) => InstructionCost(4,0.5), # - and * will fuse into this, so much of the time they're not twice as expensive
    Instruction(:vfnmadd_fast) => InstructionCost(4,0.5), # + and -* will fuse into this, so much of the time they're not twice as expensive
    Instruction(:vfnmsub_fast) => InstructionCost(4,0.5), # - and -* will fuse into this, so much of the time they're not twice as expensive
    Instruction(:sqrt) => InstructionCost(15,4.0,-2.0),
    Instruction(:log) => InstructionCost(20,20.0,40.0,20),
    Instruction(:exp) => InstructionCost(20,20.0,20.0,18),
    Instruction(:sin) => InstructionCost(18,15.0,68.0,23),
    Instruction(:cos) => InstructionCost(18,15.0,68.0,26),
    Instruction(:sincos) => InstructionCost(25,22.0,70.0,26),
    Instruction(:identity) => InstructionCost(0,0.0,0.0,0),
    Instruction(:adjoint) => InstructionCost(0,0.0,0.0,0),
    Instruction(:transpose) => InstructionCost(0,0.0,0.0,0),
    # Symbol("##CONSTANT##") => InstructionCost(0,0.0)
)
# for (k, v) ∈ COST # so we can look up Symbol(typeof(function))
#     COST[Symbol("typeof(", lower(k), ")")] = v
# end

const CORRESPONDING_REDUCTION = Dict{Instruction,Instruction}(
    Instruction(:+) => Instruction(:vsum),
    Instruction(:-) => Instruction(:vsum),
    Instruction(:*) => Instruction(:vprod),
    Instruction(:vadd) => Instruction(:vsum),
    Instruction(:vsub) => Instruction(:vsum),
    Instruction(:vmul) => Instruction(:vprod),
    Instruction(:&) => Instruction(:vall),
    Instruction(:|) => Instruction(:vany),
    Instruction(:muladd) => Instruction(:vsum),
    Instruction(:fma) => Instruction(:vsum),
    Instruction(:vmuladd) => Instruction(:vsum),
    Instruction(:vfma) => Instruction(:vsum),
    Instruction(:vfmadd) => Instruction(:vsum),
    Instruction(:vfmsub) => Instruction(:vsum),
    Instruction(:vfnmadd) => Instruction(:vsum),
    Instruction(:vfnmsub) => Instruction(:vsum)
)
const REDUCTION_TRANSLATION = Dict{Instruction,Instruction}(
    Instruction(:+) => Instruction(:evadd),
    Instruction(:vadd) => Instruction(:evadd),
    Instruction(:*) => Instruction(:evmul),
    Instruction(:vmul) => Instruction(:evmul),
    Instruction(:-) => Instruction(:evadd),
    Instruction(:vsub) => Instruction(:evadd),
    Instruction(:/) => Instruction(:evmul),
    Instruction(:vfdiv) => Instruction(:evmul),
    Instruction(:muladd) => Instruction(:evadd),
    Instruction(:fma) => Instruction(:evadd),
    Instruction(:vmuladd) => Instruction(:evadd),
    Instruction(:vfma) => Instruction(:evadd),
    Instruction(:vfmadd) => Instruction(:evadd),
    Instruction(:vfmsub) => Instruction(:evadd),
    Instruction(:vfnmadd) => Instruction(:evadd),
    Instruction(:vfnmsub) => Instruction(:evadd)
)
const REDUCTION_ZERO = Dict{Instruction,Symbol}(
    Instruction(:+) => :zero,
    Instruction(:vadd) => :zero,
    Instruction(:*) => :one,
    Instruction(:vmul) => :one,
    Instruction(:-) => :zero,
    Instruction(:vsub) => :zero,
    Instruction(:/) => :one,
    Instruction(:vfdiv) => :one,
    Instruction(:muladd) => :zero,
    Instruction(:fma) => :zero,
    Instruction(:vmuladd) => :zero,
    Instruction(:vfma) => :zero,
    Instruction(:vfmadd) => :zero,
    Instruction(:vfmsub) => :zero,
    Instruction(:vfnmadd) => :zero,
    Instruction(:vfnmsub) => :zero
)

lv(x) = GlobalRef(LoopVectorization, x)
# Fast functions, because common pattern is
const REDUCTION_SCALAR_COMBINE = Dict{Instruction,GlobalRef}(
    Instruction(:+) => lv(:reduced_add),
    Instruction(:vadd) => lv(:reduced_add),
    Instruction(:*) => lv(:reduced_prod),
    Instruction(:vmul) => lv(:reduced_prod),
    Instruction(:-) => lv(:reduced_add),
    Instruction(:vsub) => lv(:reduced_add),
    Instruction(:/) => lv(:reduced_prod),
    Instruction(:vfdiv) => lv(:reduced_prod),
    Instruction(:muladd) => lv(:reduced_add),
    Instruction(:fma) => lv(:reduced_add),
    Instruction(:vmuladd) => lv(:reduced_add),
    Instruction(:vfma) => lv(:reduced_add),
    Instruction(:vfmadd) => lv(:reduced_add),
    Instruction(:vfmsub) => lv(:reduced_add),
    Instruction(:vfnmadd) => lv(:reduced_add),
    Instruction(:vfnmsub) => lv(:reduced_add)
)

const FUNCTIONSYMBOLS = Dict{Type{<:Function},Instruction}(
    typeof(+) => :(+),
    typeof(SIMDPirates.vadd) => :(+),
    typeof(Base.FastMath.add_fast) => :(+),
    typeof(-) => :(-),
    typeof(SIMDPirates.vsub) => :(-),
    typeof(Base.FastMath.sub_fast) => :(-),
    typeof(*) => :(*),
    typeof(SIMDPirates.vmul) => :(*),
    typeof(Base.FastMath.mul_fast) => :(*),
    typeof(/) => :(/),
    typeof(SIMDPirates.vdiv) => :(/),
    typeof(Base.FastMath.div_fast) => :(/),
    typeof(==) => :(==),
    typeof(isequal) => :isequal,
    typeof(&) => :(&),
    typeof(|) => :(|),
    typeof(>) => :(>),
    typeof(<) => :(<),
    typeof(>=) => :(>=),
    typeof(<=) => :(<=),
    typeof(inv) => :inv,
    typeof(muladd) => :muladd,
    typeof(fma) => :fma,
    typeof(SIMDPirates.vmuladd) => :vmuladd,
    typeof(SIMDPirates.vfma) => :vfma,
    typeof(SIMDPirates.vfmadd) => :vfmadd,
    typeof(SIMDPirates.vfmsub) => :vfmsub,
    typeof(SIMDPirates.vfnmadd) => :vfnmadd,
    typeof(SIMDPirates.vfnmsub) => :vfnmsub,
    typeof(sqrt) => :sqrt,
    typeof(Base.FastMath.sqrt_fast) => :sqrt,
    typeof(SIMDPirates.vsqrt) => :sqrt,
    typeof(log) => :log,
    typeof(Base.FastMath.log_fast) => :log,
    typeof(SIMDPirates.vlog) => :log,
    typeof(SLEEFPirates.log) => :log,
    typeof(exp) => :exp,
    typeof(Base.FastMath.exp_fast) => :exp,
    typeof(SIMDPirates.vexp) => :exp,
    typeof(SLEEFPirates.exp) => :exp,
    typeof(sin) => :sin,
    typeof(Base.FastMath.sin_fast) => :sin,
    typeof(SLEEFPirates.sin) => :sin,
    typeof(cos) => :cos,
    typeof(Base.FastMath.cos_fast) => :cos,
    typeof(SLEEFPirates.cos) => :cos,
    typeof(sincos) => :sincos,
    typeof(Base.FastMath.sincos_fast) => :sincos,
    typeof(SLEEFPirates.sincos) => :sincos
)


user = ENV["USER"]
host = readlines("/etc/hostname")[1]
kernel = split(readlines("/proc/version")[1])[3]
term = ENV["TERM"]
shell = ENV["SHELL"]
tasks = length(filter(x -> isa(tryparse(Int, x), Int), readdir("/proc")))

mem = readlines("/proc/meminfo")[1:3]
total, _, avail = map(x -> fld(parse(Int, split(x)[2]), 1000), mem)

uptime = parse(Int, split(readlines("/proc/uptime")[1], '.')[1])
d = trunc(Int, (uptime / 60 / 60 / 24))
h = trunc(Int, (uptime / 60 / 60 % 24))
m = trunc(Int, (uptime / 60 % 60))

print("$user@$host
kernel\t$kernel
term\t$term
shell\t$shell
tasks\t$tasks
mem\t$(avail)m / $(total)m
uptime\t$(d)d $(h)h $(m)m")


len(m::single_part_motif) = m.len;
# view(m::single_part_motif) = @pt :header = DNA_char_cap view(m.P);
sample_m(m::single_part_motif)::String = join([DNA_char_cap[i] for i in rand(m.P)]);
sample_whole(m::single_part_motif) = (motif=sample_m(m), mode=1);

function sample_contiguous_subset(m::single_part_motif, r::UnitRange{Int})::String
    if maximum(r) > m.len || minimum(r) < 1
        error("unit range specified is not covered by 1:$(m.len)");
    end
    sample_m(m)[r] # make it Vector{String} for concatenation purposes later (sample fake data)
end

view(m::k_parts_motif) = for v in m.P view(v) end;
sample_m(m::k_parts_motif)::String = join([sample_m(v) for v in m.P]);
sample_vec(m::k_parts_motif) = [sample_m(v) for v in m.P];
sample_whole(m::k_parts_motif) = (motif=sample_m(m), mode=1);

# distribution on the number of gap alphbet for now is assumed to be discrete uniform (for now)
function sample_m(m::gapped_k_parts_motif)::String
    s = [];
    parts = sample_vec(m.P_motifs);
    for i = 1:(m.K-1)
        push!(s, parts[i]);
        num_alphabet = rand(DiscreteUniform(0, m.gap_len[i]));
        push!(s, join([DNA_char_nocap[rand(m.P_gap)] for j = 1:num_alphabet]));
    end
    push!(s,parts[m.K]);
    return join(s);
end

sample_whole(m::gapped_k_parts_motif) = (motif=sample_m(m), mode=1);

"""
sample a (contiguous) subset of gapped_p_fam_k_motif_PC
according to a unitrange r
"""

function sample_contiguous_subset(m::gapped_k_parts_motif, r::UnitRange{Int})::String
    r_max = maximum(r); r_min = minimum(r);
    if r_max > m.K || r_min < 1
        error("unit range specified is not covered by 1:$(m.len)");
    end
    s = [];
    parts = sample_vec(m.P_motifs);
    for i = r_min:(r_max-1)
        push!(s, parts[i]);
        num_alphabet = rand(DiscreteUniform(0,m.gap_len[i]));
        push!(s, join([DNA_char_nocap[rand(m.P_gap)] for j = 1:num_alphabet]));
    end
    push!(s, parts[r_max]);
    return join(s);
end

view(m::gapped_k_parts_motif) = view(m.P_motifs);

# example ----------------------------------
# a = gapped_k_parts_motif([3,3,3],[2,2]);
# sample_m(a)
# view(a)
# ------------------------------------------

# Sample the ith mode of the motif, a gapped_p_fam_k_motif

function sample_m(m::mixture_gapped_k_parts_motifs)::String
    i = rand(m.mixture_weights);
    sample_contiguous_subset(m.motif, m.modes[i]);
end

function sample_whole(m::mixture_gapped_k_parts_motifs)
    i = rand(m.mixture_weights);
    str = sample_contiguous_subset(m.motif, m.modes[i]);
    (motif=str, mode=i);
end

# example ----------------------------------
# a=mixture_gapped_k_parts_motifs([7,7,7],[4,4],[1:2,2:3]);
# a.modes
# a.num_modes
# a.mixture_weights.p
# rand(a.mixture_weights)
# sample_m(a)

# b = mixture_gapped_k_parts_motifs([1:9, 6:12, 8:15], "uniform");
# b.modes
# b.num_modes
# b.mixture_weights.p
# rand(b.mixture_weights)
# sample_m(b)
# ------------------------------------------

"""
Sample the ith mode of the motif, a mixture_p_fam_motifs
"""
function sample_m(m::mixture_k_parts_motifs)::String
    i = rand(m.mixture_weights);
    sample_contiguous_subset(m.motif, m.modes[i]);
end

function sample_whole(m::mixture_k_parts_motifs)
    i = rand(m.mixture_weights);
    str = sample_contiguous_subset(m.motif, m.modes[i]);
    # println("mode $i");
    (motif=str, mode=i);
end

# example ----------------------------------
# a = mixture_k_parts_motifs([1:9, 3:8, 6:15]);
# a.modes
# a.num_modes
# a.mixture_weights.p
# rand(a.mixture_weights)
# sample_m(a)
# sample_whole(a)

# b = mixture_gapped_p_fam_k_motifs([1:7, 6:8, 6:10], "uniform");
# b.modes
# b.num_modes
# b.mixture_weights.p
# rand(b.mixture_weights)
# sample_m(b)
# ------------------------------------------

# Sample a background-string of length len (default: len = data_pt_len)
sample_background_single(len::Int = data_pt_len)::String =
        join([DNA_char_nocap[rand(background)] for _ in 1:len]);

#=
sample a background string of length len (default: len = data_pt_len)
with a motif embedded in it;

returns a named-tuple:
(str=dna_string, motif_where=motif_start:motif_end, mode=binding_mode),
which has a motif-embedded background string , an UnitRange that
indicates where motif is embedded, and the mode of binding
=#

function sample_backgound_with_motif_single(motif::motif_type,
                        len::Int=data_pt_len
                        )::sim_dna_str_w_motif

    motif_realization = sample_whole(motif);

    ########### either make it complement or not ###########
    motif_str = motif_realization.motif; motif_mode = motif_realization.mode;
    is_complement = rand() > percent_complement;
    motif_realization = is_complement ? (motif=reverse_complement(motif_str),mode=motif_mode) : motif_realization;
    ########################################################

    motif_len = length(motif_realization.motif);
    if motif_len > data_pt_len
        error("Please use a motif that has width shorter than $data_pt_len");
    end

    avail_positions::Int = (len-motif_len+1)-sample_near_center;
    at::Int = rand(DiscreteUniform(1+sample_near_center, avail_positions));
    before_at::Int = (at-1);
    dna_str = join([sample_background_single(before_at),
        motif_realization.motif::String,
        sample_background_single((data_pt_len-(before_at+motif_len)))]);

    return  (str=dna_str,
             motif_where=at:(at+motif_len-1),
             mode=motif_realization.mode,
             complement=is_complement);
end

# example ----------------------------------
# a = gapped_k_parts_motif([3,3,3],[2,2]);
# sample_a = sample_backgound_with_motif_single(a);
# sample_a
# sample_a.mode
# sample_a[1][sample_a.motif_where]

# b = mixture_gapped_k_parts_motifs([3,3,3],[2,2],[1:2,2:3]);
# sample_b = sample_backgound_with_motif_single(b);
# sample_b
# sample_b.mode
# sample_b[1][sample_b.motif_where]
# ------------------------------------------

#=
sample multiple background string of length len (default: len = data_pt_len)
with a motif embedded in it; see sample_backgound_with_motif_single.
Instead of returning an instance of sim_dna_str_w_motif, it returns an
instance of Vector{sim_dna_str_w_motif}.
=#

function sample_backgound_with_motif_multiple(
                            motif::motif_type,
                            how_many::Int,
                            len::Int=data_pt_len
                            )::Vector{sim_dna_str_w_motif}
    return [sample_backgound_with_motif_single(motif, len)
        for _ in 1:how_many];
end

# example ----------------------------------
# b = mixture_gapped_k_parts_motifs([3,3,3],[2,2],[1:2,2:3]);
# sample_b = sample_backgound_with_motif_multiple(b, 5);
# sample_b
# ------------------------------------------


#using Distributions
#include("constants.jl")
#include("orbit.jl")
#include("planet.jl")

if !isdefined(:PlanetarySystemAbstract)
  @compat abstract type PlanetarySystemAbstract end

  immutable PlanetarySystem{StarT<:StarAbstract} <: PlanetarySystemAbstract
    star::StarT
    planet::Vector{Planet}         # TODO OPT: If want to optimize, consider using  something like ImmmutableArrays?
    orbit::Vector{Orbit}           # TODO OPT: If want to optimize, consider using  something like ImmmutableArrays?
    # TODO DETAIL: Setup inner constructor to enforce equal number of planets & orbits
    #PlanetarySystem{StarT}(s::StarAbstract, p::Vector{Planet}, o::Vector{Orbit})
    #  @assert(length(p)==length(o)) # else error(string("Number of planets must match number of orbits: Np= ",length(p)," No= ",length(o)))
    #  new(s,p,o)
    #end
  end
  #typealias PlanetarySystemSingleStar  PlanetarySystem{SingleStar}
  PlanetarySystemSingleStar = PlanetarySystem{SingleStar}

end
PlanetarySystem{StarT<:StarAbstract}(s::StarT) = PlanetarySystem(s,Array{Planet}(0),Array{Orbit}(0))  # Constructor for a Planetary System with no planets

PlanetarySystem{StarT<:StarAbstract}(s::StarT, p::Planet, o::Orbit) = PlanetarySystem(s,[p],[o])  # Constructor for a single Planet System

function PlanetarySystem{StarT}(ps::PlanetarySystem{StarT}, keep::Vector{Int64}) # Why doesn't this work?
   PlanetarySystem{StarT}(ps.star,ps.planet[keep],ps.orbit[keep])
end

#function PlanetarySystemSingleStar(ps::PlanetarySystemSingleStar, keep::Vector{Int64})
#function PlanetarySystem{StarT<:StarAbstract}(ps::PlanetarySystem{StarT}, keep::Vector{Int64})
#   PlanetarySystem(ps.star,ps.planet[keep],ps.orbit[keep])
#end

flux(ps::PlanetarySystem{StarAbstract}) = flux(ps.star)
flux(ps::PlanetarySystem{Star}) = flux(ps.star)
flux(ps::PlanetarySystem{BinaryStar}) = flux(ps.star)
flux(ps::PlanetarySystem{MultipleStar}) = flux(ps.star)

function num_planets(s::PlanetarySystem{Star})
  @assert( length(s.planet) == length(s.orbit) )    # TODO OPT: Deactivate inner assert's like this for speed once tested
  return length(s.planet)
end

function generate_planet_mass_from_radius_powerlaw(r::Float64, s::Star, o::Orbit, sim_param::SimParam) # TODO SCI: IMPORTANT once have stability criteria, replace w/ better M-R relationship
  const mr_power_index::Float64 = get_real(sim_param,"mr_power_index")
  const mr_const::Float64 = get_real(sim_param,"mr_const")
  m = mr_const*earth_mass*(r/earth_radius)^mr_power_index
  return m
end

function generate_num_planets_poisson(lambda::Real, max_planets_in_sys::Integer; min_planets_in_sys::Integer = 0)
  if lambda < min_planets_in_sys*1e-3
      return min_planets_in_sys
  end
  d = Distributions.Truncated(Distributions.Poisson(lambda),min_planets_in_sys,max_planets_in_sys)
  n = rand(d)
  #=
  n = -1
  while !(min_planets_in_sys<=n<=max_tranets_in_sys)
     n = rand(Distributions.Poisson(lambda))
  end
  =#
 return n
end

function generate_num_planets_poisson(s::Star, sim_param::SimParam)
  const lambda::Float64 = exp(get_real(sim_param,"log_eta_pl"))
  const max_tranets_in_sys::Int64 = get_int(sim_param,"max_tranets_in_sys")
  generate_num_planets_poisson(lambda,max_tranets_in_sys)
end

function generate_period_and_sizes_log_normal(s::Star, sim_param::SimParam; num_pl::Integer = 1)  # TODO SCI:  IMPORTANT: Improve how periods and sizes are drawn
    const mu_log_r::Float64 = get_real(sim_param,"mean_log_planet_radius")
    const sigma_log_r::Float64 = get_real(sim_param,"sigma_log_planet_radius")
    const mu_log_P::Float64 = get_real(sim_param,"mean_log_planet_period")
    const sigma_log_P::Float64 = get_real(sim_param,"sigma_log_planet_period")
    const min_period::Float64 = get_real(sim_param,"min_period")
    const max_period::Float64 = get_real(sim_param,"max_period")
    const min_radius::Float64 = get_real(sim_param,"min_radius")
    const max_radius::Float64 = get_real(sim_param,"max_radius")
    const max_draws::Int64 = 100

    if   sigma_log_r <= 0. || sigma_log_P<=0.
     println("# mu_log_r= ", mu_log_r, " sigma_log_r= ", sigma_log_r, " mu_log_P= ", mu_log_P, " sigma_log_P= ", sigma_log_P)
    end
    const rdist = LogNormal(mu_log_r,sigma_log_r)
    const Pdist = LogNormal(mu_log_P,sigma_log_P)
    #Rlist = rand(rdist,num_pl)
    #Plist = rand(Pdist,num_pl)
    #idx_keep = find(i->(min_radius<=Rlist[i]<=max_radius) && (min_period<=Plist[i]<=max_period), 1:num_pl )
    #return Plist[idx_keep], Rlist[idx_keep]  # replaced because want to return exactly num_pl
    Rlist = zeros(num_pl)
    Plist = zeros(num_pl)
    for i in 1:num_pl
      j = 0
      while ! (min_radius<Rlist[i]<=max_radius) && j<max_draws
            Rlist[i] = rand(rdist)
            j+=1
      end
      if j>=max_draws
         println("# Struggled to draw size for: ",mu_log_r, " ", sigma_log_r)
      end
      j = 0
      while ! (min_period<Plist[i]<=max_period) && j<max_draws
            Plist[i] = rand(Pdist)
            j+=1
      end
      if j>=max_draws
         println("# Struggled to draw period for: ",mu_log_P, " ", sigma_log_P)
      end
    end
    return Plist, Rlist
end

function draw_power_law(n::Real, x0::Real, x1::Real, num_pl::Integer)
     ((x1^(n+1) - x0^(n+1))*rand(num_pl) + x0^(n+1)).^(1/(n+1))
end

function generate_period_and_sizes_power_law(s::Star, sim_param::SimParam; num_pl::Integer = 1)
    const power_law_r::Float64 = get_real(sim_param,"power_law_r")
    const power_law_P::Float64 = get_real(sim_param,"power_law_P")
    const min_period::Float64 = get_real(sim_param,"min_period")
    const max_period::Float64 = get_real(sim_param,"max_period")
    const min_radius::Float64 = get_real(sim_param,"min_radius")
    const max_radius::Float64 = get_real(sim_param,"max_radius")
    Rlist = power_law_r!=-1.0 ? draw_power_law(power_law_r,min_radius,max_radius, num_pl) : exp(log(min_radius).+rand()*log(max_radius/min_radius))
    Plist = power_law_P!=-1.0 ? draw_power_law(power_law_P,min_period,max_period, num_pl) : exp(log(min_period).+rand()*log(max_period/min_period))
    return Plist, Rlist
end

function generate_e_omega_rayleigh(sigma_hk::Float64)
  h = k = 1.0
  while h*h+k*k >= 1.0
    h = sigma_hk*randn()
    k = sigma_hk*randn()
  end
  ecc = sqrt(h*h+k*k)
  w = atan2(k,h)
  return ecc, w
end

function generate_e_omega_rayleigh(sim_param::SimParam)
  sigma_hk::Float64 = get_real(sim_param,"sigma_hk")
  generate_e_omega_rayleigh(sigma_hk)
end

function generate_planetary_system_hardcoded_example(star::StarAbstract, sim_param::SimParam; verbose::Bool = false)
  # in this version we specify fixed functions that are known at compile time, allowing for additional optimizations (~0.6 second faster per Kepler catalog out of ~3.6 sec on my laptop w/ 1 core)
  const  generate_planet_mass_from_radius = generate_planet_mass_from_radius_powerlaw
  const  generate_num_planets = generate_num_planets_poisson
  const  generate_period_and_sizes = generate_period_and_sizes_log_normal

  const  generate_e_omega =  generate_e_omega_rayleigh

  # const  generate_star = get_function(sim_param,"generate_star")
  # const star::StarAbstract = generate_star(sim_param)
  const num_pl::Int64 = generate_num_planets(star, sim_param)

  if( num_pl==0 )
    return PlanetarySystem(star)
  else
     pl = Array{Planet}(num_pl)
     orbit = Array{Orbit}(num_pl)
     (Plist::Vector{Float64}, Rlist::Vector{Float64}) = generate_period_and_sizes(star, sim_param, num_pl=num_pl)
     idx = sortperm(Plist)                   # TODO OPT: Check to see if sorting is significant time sink.  If so, it might could be deferred

    for i in 1:num_pl
      # if verbose   println("i=",i," idx=",idx," Plist=",Plist[idx] );     end
      P = Plist[idx[i]]
      (ecc::Float64,  omega::Float64) = generate_e_omega(sim_param)
      incl::Float64 = acos(rand())
      orbit[idx[i]] = Orbit(P,ecc,incl,omega,2pi*rand(),2pi*rand())
      # set!(orbit[idx[i]],P,ecc,incl,omega,2pi*rand(),2pi*rand())
      mass::Float64 = generate_planet_mass_from_radius(Rlist[idx[i]], star, orbit[idx[i]], sim_param)
      pl[i] = Planet( Rlist[idx[i]],  mass )
    end
  return PlanetarySystem(star,pl,orbit)
  end
end

function generate_planetary_system_uncorrelated_incl(star::StarAbstract, sim_param::SimParam; verbose::Bool = false)
  # load functions to use for drawing parameters
  const  generate_planet_mass_from_radius = get_function(sim_param,"generate_planet_mass_from_radius")
  const  generate_num_planets = get_function(sim_param,"generate_num_planets")
  const  generate_period_and_sizes = get_function(sim_param,"generate_period_and_sizes")
  const  generate_e_omega = get_function(sim_param,"generate_e_omega")

  # const  generate_star = get_function(sim_param,"generate_star")
  # const star::StarAbstract = generate_star(sim_param)
  const num_pl::Int64 = generate_num_planets(star, sim_param)::Int64
  const sigma_ecc::Float64 = haskey(sim_param,"sigma_hk") ? get_real(sim_param,"sigma_hk") : 0.0

  if( num_pl==0 )
    return PlanetarySystem(star)::PlanetarySystem
  else
     pl = Array{Planet}(num_pl)
     orbit = Array{Orbit}(num_pl)
     (Plist::Vector{Float64}, Rlist::Vector{Float64}) = generate_period_and_sizes(star, sim_param, num_pl=num_pl)
     idx = sortperm(Plist)                   # TODO OPT: Check to see if sorting is significant time sink.  If so, it might could be deferred

    for i in 1:num_pl
      # if verbose   println("i=",i," idx=",idx," Plist=",Plist[idx] );     end
      P = Plist[idx[i]]
      if haskey(sim_param,"sigma_hk_one") && haskey(sim_param,"sigma_hk_multi")
         sigma_ecc = num_pl == 1 ? get_real(sim_param,"sigma_hk_one") : get_real(sim_param,"sigma_hk_multi")
      end
      (ecc::Float64,  omega::Float64) = generate_e_omega(sim_param)
      incl::Float64 = acos(rand())
      orbit[idx[i]] = Orbit(P,ecc,incl,omega,2pi*rand(),2pi*rand())
      # set!(orbit[idx[i]],P,ecc,incl,omega,2pi*rand(),2pi*rand())
      mass::Float64 = generate_planet_mass_from_radius(Rlist[idx[i]], star, orbit[idx[i]], sim_param)
      pl[i] = Planet( Rlist[idx[i]],  mass )
    end
  return PlanetarySystem(star,pl,orbit)
  end
end

# This version generates more systems roughly near a common plane, but until incorporate CORBITS data, ABC can't match input param
function generate_planetary_system_simple(star::StarAbstract, sim_param::SimParam; verbose::Bool = false)
  # load functions to use for drawing parameters
  const  generate_planet_mass_from_radius = get_function(sim_param,"generate_planet_mass_from_radius")
  const  generate_num_planets = get_function(sim_param,"generate_num_planets")
  #const  generate_num_planets = generate_num_planets_christiansen
  #const  generate_period_and_sizes = generate_period_and_sizes_christiansen
  const  generate_period_and_sizes = get_function(sim_param,"generate_period_and_sizes")
  const  generate_e_omega = get_function(sim_param,"generate_e_omega")
  const  sigma_incl = deg2rad(get_real(sim_param,"sigma_incl"))

  # const  generate_star = get_function(sim_param,"generate_star")
  # const star::StarAbstract = generate_star(sim_param)
  const num_pl = generate_num_planets(star, sim_param)::Int64
  const sigma_ecc::Float64 = haskey(sim_param,"sigma_hk") ? get_real(sim_param,"sigma_hk") : 0.0

  if( num_pl==0 )
    return PlanetarySystem(star)
  else
     pl = Array{Planet}(num_pl)
     orbit = Array{Orbit}(num_pl)
     (Plist::Vector{Float64}, Rlist::Vector{Float64}) = generate_period_and_sizes(star, sim_param, num_pl=num_pl)
     idx = sortperm(Plist)                   # TODO OPT: Check to see if sorting is significant time sink.  If so, it might could be deferred
     incl_sys = acos(rand())

    for i in 1:num_pl
      # if verbose   println("i=",i," idx=",idx," Plist=",Plist[idx] );     end
      P = Plist[idx[i]]
      if haskey(sim_param,"sigma_hk_one") && haskey(sim_param,"sigma_hk_multi")
         sigma_ecc = num_pl == 1 ? get_real(sim_param,"sigma_hk_one") : get_real(sim_param,"sigma_hk_multi")
      end
      (ecc,  omega) = generate_e_omega(sim_param)::Tuple{Float64,Float64}
      incl_mut = sigma_incl*sqrt(randn()^2+randn()^2) # rand(Distributions.Rayleigh(sigma_incl)) # sigma_incl*randn()
      asc_node = 2pi*rand()
      mean_anom = 2pi*rand()
      #incl = incl_sys + sigma_incl*randn()
      incl =  incl_mut!=zero(incl_mut) ? acos( cos(incl_sys)*cos(incl_mut) + sin(incl_sys)*sin(incl_mut)*cos(asc_node) ) : incl_sys
      orbit[idx[i]] = Orbit(P,ecc,incl,omega,asc_node,mean_anom)
      # set!(orbit[idx[i]], P,ecc,incl,omega,asc_node,mean_anom) # if Orbit were mutable
      mass = generate_planet_mass_from_radius(Rlist[idx[i]], star, orbit[idx[i]], sim_param)::Float64
      pl[i] = Planet( Rlist[idx[i]],  mass )
    end
  return PlanetarySystem(star,pl,orbit)
  end
end

function test_planetary_system_constructors(sim_param::SimParam)
  generate_star = get_function(sim_param,"generate_star")
  star = generate_star(sim_param)
  empty_sys = PlanetarySystem(star)
  earth = Planet(earth_radius,earth_mass)
  earth_orbit = Orbit(365.2425,0.0167,0.5*pi,0.0,0.0,0.0)
  solar_sys = PlanetarySystem(star, earth,earth_orbit)
  m = generate_planet_mass_from_radius_powerlaw(0.02,star,earth_orbit,sim_param)/earth_mass
  generate_planetary_system_simple(star,sim_param,verbose=true)
end


using Plots
gr()
using DataFrames,CSV

using LsqFit
r2=0

#%% Load the data
include("/home/kevin/Documents/1 - PhD Projects/2 - Confidence Attractor Network/ConfianceAttractorNetwork/src/decision_making.jl")
#include("/home/kevin/Documents/1 - PhD Projects/2 - Confidence Attractor Network/src/reaction_time_analysis_functions.jl")
include("/home/kevin/Documents/1 - PhD Projects/2 - Confidence Attractor Network/ConfianceAttractorNetwork/src/distributions.jl")

function sim_mf_decisionmaking_pattern_fastv2(coh_temp::Array{Float64},Ir::Float64,If::Float64,
  param::Dict{String,Float64},fixed_param::Dict{String,Float64},simu_param::Dict{String,Float64})

  # FI curve parameters - adapted from Wanf fit
  a=fixed_param["a"]
  b=fixed_param["b"]
  d=fixed_param["d"]
  falset= param["false_time"]
  thr = param["Threshold"]

  # time variables of the  simulation

  dt=simu_param["dt"]
  time_wind=simu_param["time_wind"] # mean window of the variables
  slide_wind = simu_param["slide_wind"]
  false_time=param["false_time"]
  falsetIf=param["false_timeIf"]

  #---- Intialise and vectorise variables to be used in loops below ------

  # Final Variables
  res=zeros(length(coh_temp))
  reaction_time=zeros(length(coh_temp))
  diffS=zeros(length(coh_temp))
  S1=zeros(length(coh_temp))
  S2=zeros(length(coh_temp))
  R1=zeros(length(coh_temp))
  R2=zeros(length(coh_temp))

  ###### Simu variables aprameters
  noise_amp=param["noise_amp"]

  mu0=  param[  "mu0"]

  I0E1p=  param[  "I0E1"]
  I0E2p =  param[  "I0E2"]
  JN11 =  param[ "JN11"]
  JN22  = param[ "JN22"]
  JN12 =  param[ "JN12"]
  JN21 =  param[ "JN21"]
  JAext  = param[ "JAext"]
  Tnmda = param[  "Tnmda"]
  gamma = param[  "gamma"]
  Tstim = param[  "Tstim"]
  Tampa = param[  "Tampa"]
  coh=  param[ "coh"]

  false_timeIf=  param[  "false_timeIf"]

  #####

  # Temporary variables

    Isyn1=0.0
  Isyn2=0.0
  Trest=param["T_rest"]
  s1=0.1
  s2=0.1
  phi1=0.0
  phi2=0.0
  I_eta1=noise_amp * randn()
  I_eta2=noise_amp * randn()

  nu1=Float64[2.0]
  nu2=Float64[2.0]
  k=0 # variable for mod 4 in the mean
  th=false
  result=true

  for (j,coh) in enumerate(coh_temp) # loop on trials number

    th=false
    result=true
    t=0

    nu1=Float64[nu1[end]]
    nu2=Float64[nu2[end]]
    k=0
    while th==false # while we do not cross the threshold
      t=t+dt
      k=k+1

      if t<Trest && result==true
        I0E1=I0E1p -Ir*exp(-t/falset)
        I0E2=I0E2p -Ir*exp(-t/falset)

        I_stim1=0
        I_stim2=0

      elseif t<Trest
        I0E1=I0E1p -Ir*exp(-t/falset) -If*exp(-t/falsetIf)
        I0E2=I0E2p -Ir*exp(-t/falset) -If*exp(-t/falsetIf)
        I_stim1=0
        I_stim2=0

      elseif t<Trest+dt/2 && t>Trest-dt/2
        I0E1=I0E1p
        I0E2=I0E2p
        I_stim1 = JAext*mu0*(1+coh/100)
        I_stim2 = JAext*mu0*(1-coh/100)

      else
        I0E1=I0E1p
        I0E2=I0E2p
        I_stim1 = JAext*mu0*(1+coh/100)
        I_stim2 = JAext*mu0*(1-coh/100)
      end

      Isyn1=JN11*s1-JN12*s2+I_stim1 + I_eta1
      Isyn2=JN22*s2-JN21*s1+I_stim2 + I_eta2

      phi1=(a*Isyn1-b)/(1-exp(-d*(a*Isyn1-b)))
      phi2=(a*Isyn2-b)/(1-exp(-d*(a*Isyn2-b)))

      #---- Dynamical equations -------------------------------------------
      # Mean NMDA-mediated synaptic dynamics updating
      s1 = s1 + dt*(-s1/Tnmda + (1-s1)*gamma*nu1[end]/1000);

      s2 = s2 + dt*(-s2/Tnmda + (1-s2)*gamma*nu2[end]/1000);

      # Ornstein-Uhlenbeck generation of noise in pop1 and 2
      I_eta1 = I_eta1 + (dt/Tampa)*(I0E1-I_eta1) + sqrt(dt/Tampa)*noise_amp*randn() ;
      I_eta2 = I_eta2 + (dt/Tampa)*(I0E2-I_eta2) + sqrt(dt/Tampa)*noise_amp*randn() ;

      # To ensure firing rates are always positive. Large noise amplitude
      # may result in unwanted negative values
      if phi1 < 0
        push!(nu1,0)
        phi1 = 0;
      else
        push!(nu1,phi1)
      end;
      if phi2 < 0
        push!(nu2,0)
        phi2 = 0;
      else
        push!(nu2,phi2)
      end;

      if mod(k,4)==0

        # Test the crossing of a threshold
        @views m1= mean(nu1[k-3:k])
        @views m2 = mean(nu2[k-3:k])
        if  t>Trest&& m1>thr
          th=true
          if coh>=0
          @inbounds  res[j]=1
            result=true
          else
            @inbounds  res[j]=-1
            result=false
          end

        elseif t>Trest&& m2>thr
          th = true
          if coh>=0
            @inbounds  res[j]=-1
            result=false
          else
            @inbounds  res[j]=1
            result=true
          end

        end
      end
    end
  @inbounds  reaction_time[j]=t-Trest
  @inbounds    diffS[j]=s2-s1
  @inbounds    R1[j]=nu1[end]
  @inbounds  R2[j]=nu2[end]
  @inbounds    S1[j]=s1
  @inbounds  S2[j]=s2
    th=false
  end

  result=DataFrame()
  result[:RTs]=reaction_time
  result[:NotError]=res
  result[:Distri]=coh_temp
  result[:DiffS]=diffS
  result[:S1]=S1
  result[:S2]=S2
  result[:R1]=R1
  result[:R2]=R2
  result[:LastTrial]=vcat([0],res[1:end-1])
  result[:LastCoh]=vcat([0],coh_temp[1:end-1])
  return result
end

function list_elements(l)
      a = Float64[]
        i=1
        while (length(a) <20)
           if any(a.==l[i])

            else
                push!(a,l[i])
            end
            i=i+1
        end
        return sort(a)
    end

    using Distributions, StatPlots,Colors

colorsT=[:red,:blue,:darkorange,:violet,:green]
colorsT=colormap("Blues",5)
o=4

param0=zeros(40)
paramR0=zeros(40)
paramR1=zeros(40)
param1=zeros(40)
xAcc=zeros(160)
xTr=zeros(160)
f=0
u=1
fig=plot(legend=false)

for Tr in [5000]#[200,500,1500,2500,5000]
    o=o
    c_min=-51
     c_max = 51
     number = 20

       distri = make_distributions(c_min,c_max,number,10000)

     param["T_rest"] = 1000
     Ir=0.035
     If=0.0

      r2= sim_mf_decisionmaking_pattern_fastv2(distri,Ir,If,
        param,fixed_param,simu_param)
        j=0
Accrep=Float64[]
Accalt=Float64[]
for a in list_elements(r2[:Distri])
    j=j+1

for i=2:10000
    if r2[:Distri][i]==a
        if r2[:NotError][i-1]*sign(r2[:Distri][i-1])==r2[:NotError][i]*sign(r2[:Distri][i])

            push!(Accrep,r2[:RTs][i])
        else
            push!(Accalt,r2[:RTs][i])
        end
    end
end

end
histogram!(Accalt,color=:forestgreen,alpha=0.4,bins=100,normed=true)
histogram!(Accrep,color=:darkorange,alpha=0.4,bins=100,normed=true)
print(mean(Accalt)-mean(Accrep))

#savefig(fig,"./LeftRightIndecAcc$Tr.svg")

end
fig
xlims!((0,1000))
savefig(fig,"./Hist0355000.svg")

fig

#### sauvegarde RSI energy package

# /home/kevin/Documents/1 - PhD Projects/1 - Sequential Decision making/3-SequentialEffects/2-Data/testsAltRep
param = Dict(
    "mu0"=>30.0,
    "noise_amp"=>0.02,
    "I0E1"=>0.3255,
    "I0E2"=>0.3255,
    "JN11"=>0.2609,
    "JN22"=>0.2609,
    "JN12"=>0.0497,
    "JN21"=>0.0497,
    "JAext" => 0.00052,
    "Tnmda" => 100.0,
    "Tampa" => 2.0,
    "gamma" => 0.641,
    "Tstim"=>5000.0,
    "T_rest"=>700.0,
    "coh"=>5.0,
    "false_time"=>200.0,
    "false_timeIf"=>500.0,
    "Threshold"=>20.0
    )
    fixed_param=Dict("a"=>270.0,"b"=>108.0,"d"=>0.1540)
  simu_param=Dict("dt"=>0.5,"time_wind"=>4.0,"slide_wind"=>4.0) #4=2/dt
Icdlist=[0.035,0.04,0.05,0.06,0.07,0.08,0.09,0.1,0.2,0.3]
#Icdlist=[0.2,0.3]#
Trlist=[500,1000,1500,2000,2500,5000,10000]

for Icd in Icdlist
  for Tr in Trlist

    c_min=-51
     c_max = 51
     number = 20

       distri = make_distributions(c_min,c_max,number,10000)

     param["T_rest"] = Tr
     Ir=Icd
     If=0.0

      r2= sim_mf_decisionmaking_pattern_fastv2(distri,Ir,If,
        param,fixed_param,simu_param)
        CSV.write("/home/kevin/Documents/1 - PhD Projects/1 - Sequential Decision making/3-SequentialEffects/2-Data/testsAltRep/EnergyTest/10000RepAlt-T$Tr-Ir$Icd.csv",r2)

  end
end


export iauApco
"""
For a terrestrial observer, prepare star-independent astrometry
parameters for transformations between ICRS and observed
coordinates.  The caller supplies the Earth ephemeris, the Earth
rotation information and the refraction constants as well as the
site coordinates.

This function is part of the International Astronomical Union's
SOFA (Standards of Fundamental Astronomy) software collection.

Status:  support function.

Given:
   date1  double       TDB as a 2-part...
   date2  double       ...Julian Date (Note 1)
   ebpv   double[2][3] Earth barycentric PV (au, au/day, Note 2)
   ehp    double[3]    Earth heliocentric P (au, Note 2)
   x,y    double       CIP X,Y (components of unit vector)
   s      double       the CIO locator s (radians)
   theta  double       Earth rotation angle (radians)
   elong  double       longitude (radians, east +ve, Note 3)
   phi    double       latitude (geodetic, radians, Note 3)
   hm     double       height above ellipsoid (m, geodetic, Note 3)
   xp,yp  double       polar motion coordinates (radians, Note 4)
   sp     double       the TIO locator s' (radians, Note 4)
   refa   double       refraction constant A (radians, Note 5)
   refb   double       refraction constant B (radians, Note 5)

Returned:
   astrom iauASTROM*   star-independent astrometry parameters:
      pmt    double       PM time interval (SSB, Julian years)
      eb     double[3]    SSB to observer (vector, au)
      eh     double[3]    Sun to observer (unit vector)
      em     double       distance from Sun to observer (au)
      v      double[3]    barycentric observer velocity (vector, c)
      bm1    double       sqrt(1-|v|^2): reciprocal of Lorenz factor
      bpn    double[3][3] bias-precession-nutation matrix
      along  double       longitude + s' (radians)
      xpl    double       polar motion xp wrt local meridian (radians)
      ypl    double       polar motion yp wrt local meridian (radians)
      sphi   double       sine of geodetic latitude
      cphi   double       cosine of geodetic latitude
      diurab double       magnitude of diurnal aberration vector
      eral   double       "local" Earth rotation angle (radians)
      refa   double       refraction constant A (radians)
      refb   double       refraction constant B (radians)

Notes:

   1. The TDB date date1+date2 is a Julian Date, apportioned in any
   convenient way between the two arguments.  For example,
   JD(TDB)=2450123.7 could be expressed in any of these ways, among
   others:

         date1          date2

      2450123.7           0.0       (JD method)
      2451545.0       -1421.3       (J2000 method)
      2400000.5       50123.2       (MJD method)
      2450123.5           0.2       (date & time method)

   The JD method is the most natural and convenient to use in cases
   where the loss of several decimal digits of resolution is
   acceptable.  The J2000 method is best matched to the way the
   argument is handled internally and will deliver the optimum
   resolution.  The MJD method and the date & time methods are both
   good compromises between resolution and convenience.  For most
   applications of this function the choice will not be at all
   critical.

   TT can be used instead of TDB without any significant impact on
   accuracy.

   2. The vectors eb, eh, and all the astrom vectors, are with respect
   to BCRS axes.

   3. The geographical coordinates are with respect to the WGS84
   reference ellipsoid.  TAKE CARE WITH THE LONGITUDE SIGN
   CONVENTION:  the longitude required by the present function is
   right-handed, i.e. east-positive, in accordance with geographical
   convention.

   4. xp and yp are the coordinates (in radians) of the Celestial
   Intermediate Pole with respect to the International Terrestrial
   Reference System (see IERS Conventions), measured along the
   meridians 0 and 90 deg west respectively.  sp is the TIO locator
   s', in radians, which positions the Terrestrial Intermediate
   Origin on the equator.  For many applications, xp, yp and
   (especially) sp can be set to zero.

   Internally, the polar motion is stored in a form rotated onto the
   local meridian.

   5. The refraction constants refa and refb are for use in a
   dZ = A*tan(Z)+B*tan^3(Z) model, where Z is the observed
   (i.e. refracted) zenith distance and dZ is the amount of
   refraction.

   6. It is advisable to take great care with units, as even unlikely
   values of the input parameters are accepted and processed in
   accordance with the models used.

   7. In cases where the caller does not wish to provide the Earth
   Ephemeris, the Earth rotation information and refraction
   constants, the function iauApco13 can be used instead of the
   present function.  This starts from UTC and weather readings etc.
   and computes suitable values using other SOFA functions.

   8. This is one of several functions that inserts into the astrom
   structure star-independent parameters needed for the chain of
   astrometric transformations ICRS <-> GCRS <-> CIRS <-> observed.

   The various functions support different classes of observer and
   portions of the transformation chain:

         functions         observer        transformation

      iauApcg iauApcg13    geocentric      ICRS <-> GCRS
      iauApci iauApci13    terrestrial     ICRS <-> CIRS
      iauApco iauApco13    terrestrial     ICRS <-> observed
      iauApcs iauApcs13    space           ICRS <-> GCRS
      iauAper iauAper13    terrestrial     update Earth rotation
      iauApio iauApio13    terrestrial     CIRS <-> observed

   Those with names ending in "13" use contemporary SOFA models to
   compute the various ephemerides.  The others accept ephemerides
   supplied by the caller.

   The transformation from ICRS to GCRS covers space motion,
   parallax, light deflection, and aberration.  From GCRS to CIRS
   comprises frame bias and precession-nutation.  From CIRS to
   observed takes account of Earth rotation, polar motion, diurnal
   aberration and parallax (unless subsumed into the ICRS <-> GCRS
   transformation), and atmospheric refraction.

   9. The context structure astrom produced by this function is used by
   iauAtioq, iauAtoiq, iauAtciq* and iauAticq*.

Called:
   iauAper      astrometry parameters: update ERA
   iauC2ixys    celestial-to-intermediate matrix, given X,Y and s
   iauPvtob     position/velocity of terrestrial station
   iauTrxpv     product of transpose of r-matrix and pv-vector
   iauApcs      astrometry parameters, ICRS-GCRS, space observer
   iauCr        copy r-matrix

This revision:   2013 October 9

SOFA release 2018-01-30

Copyright (C) 2018 IAU SOFA Board.  See notes at end.
"""
function iauApco(date1::Real, date2::Real,
                 ebpv::AbstractMatrix{<:Real}, ehp::AbstractVector{<:Real},
                 x::Real, y::Real, s::Real, theta::Real,
                 elong::Real, phi::Real, hm::Real,
                 xp::Real, yp::Real, sp::Real,
                 refa::Real, refb::Real)

   # Allocate return value
   ref_astrom = Ref{iauASTROM}(iauASTROM())

   # Transpose to map Julia (FORTRAN) -> C style memory allocation
   ebpv = Array{Float64, 2}(ebpv') # Transpose input up front

   ccall((:iauApco, libsofa_c), Cvoid,
            (Cdouble, Cdouble, Ptr{Cdouble}, Ptr{Cdouble},
            Cdouble, Cdouble, Cdouble,
            Cdouble, Cdouble, Cdouble,
            Cdouble, Cdouble, Cdouble,
            Cdouble, Cdouble, Cdouble,
            Ref{iauASTROM}),
            convert(Float64, date1),
            convert(Float64, date2),
            pointer(ebpv),
            pointer(ehp),
            convert(Float64, x),
            convert(Float64, y),
            convert(Float64, s),
            convert(Float64, theta),
            convert(Float64, elong),
            convert(Float64, phi),
            convert(Float64, hm),
            convert(Float64, xp),
            convert(Float64, yp),
            convert(Float64, sp),
            convert(Float64, refa),
            convert(Float64, refb),
            ref_astrom)

   return ref_astrom[]
end


using DrWatson
@quickactivate("HyperspectraWithNeXL")
# Load the necessary libraries
using HyperspectraWithNeXL
using NeXLSpectrum
using Unitful
using FileIO, ImageIO

# Load the HyperSpectrum from disk
lt = 0.72*4.0*18.0*3600.0/(1024*1024) # 18.0 hours on 4 detectors
hs = NeXLSpectrum.compress(HyperSpectrum(
    LinearEnergyScale(0.0,10.0),
    Dict{Symbol,Any}(
      :TakeOffAngle => deg2rad(35.0),
      :ProbeCurrent => 1.0,
      :LiveTime => lt,
      :BeamEnergy => 20.0e3,
      :Name => "Mn Nodule"
    ),
    readrplraw(joinpath(datadep"MnNodule","map[15]")),
    fov = [ 4.096u"mm", 4.096u"mm"], offset= [ 0.0u"mm", 0.0u"mm" ]
))
outpath = joinpath(papersdir(),"Figures","Figure 5")
mkpath(outpath)
cp(joinpath(datadep"MnNodule","Image[0][[1]].png"), joinpath(outpath,"Mn_nodule_BSED.png"), force=true)

# Independent normalization (to brightest pixel)
outpath = joinpath(papersdir(),"Figures","Figure 6")
mkpath(outpath)
FileIO.save(File{format"PNG"}(joinpath(outpath,"I[C K-L2].png")), hs[n"Mn K-L3"])
FileIO.save(File{format"PNG"}(joinpath(outpath,"I[Mn K-L3].png")), hs[n"Mn K-L3"])
FileIO.save(File{format"PNG"}(joinpath(outpath,"I[Fe K-L3].png")), hs[n"Fe K-L3"])
FileIO.save(File{format"PNG"}(joinpath(outpath,"I[O K-L3].png")), hs[n"O K-L3"])
# Normalized as a set (normalize to sum of intensities)

outpath=plotsdir()
mkpath(outpath)
cxrs = [n"Mn K-L3", n"O K-L3", n"Fe K-L3" ]
imgs = hs[ cxrs ]
for (cxr, img) in zip(cxrs, imgs)
  FileIO.save(File{format"PNG"}(joinpath(outpath,"I_rel[$cxr].png")), img)
end
# RGB
outpath = joinpath(papersdir(),"Figures","Figure 7")
mkpath(outpath)
img = colorize(hs, cxrs, :All)
FileIO.save(File{format"PNG"}(joinpath(outpath,"colorized[Mn,O,Fe,All].png")), img)
img = colorize(hs, cxrs, :Each)
FileIO.save(File{format"PNG"}(joinpath(outpath,"colorized[Mn,O,Fe,Each].png")), img)


module MotifPvalue

#### Dependencies ##################
using DataStructures, DoubleFloats
####################################

#### Exported methods and types ####
export score2pvalue, pvalue2score
####################################

#### Load files ####################
include("helpers.jl")
include("score2pval.jl")
include("pval2score.jl")
####################################

end


using Test, WAVI, MAT, JLD2, NetCDF, Dates

function output_test(; dt  = 0.5,
                    end_time = 100.,
                    output_freq = 5.,
                    output_format = "jld2",
                    zip_format = "none",
                    prefix = "outfile",
                    dump_vel = false,
                    output_path = ".",
                    pchkpt_freq = Inf,
                    chkpt_freq = Inf,
                    niter0 = 0,
                    output_start = false)

    grid = Grid() #default grid with nx = 80, ny = 10
    bed = WAVI.mismip_plus_bed
    solver_params = SolverParams(maxiter_picard = 1)
    model = Model(grid = grid, bed_elevation = bed, solver_params = solver_params)
    timestepping_params = TimesteppingParams(niter0 = niter0, dt = dt, end_time = end_time, chkpt_freq = chkpt_freq, pchkpt_freq = pchkpt_freq)

    outputs = (h = model.fields.gh.h,
                u = model.fields.gh.u,
                v = model.fields.gh.v,
                b = model.fields.gh.b) #output velocities and thickness

    output_freq = output_freq
    output_params = OutputParams(outputs = outputs,
                        output_freq = output_freq,
                        output_format = output_format,
                        output_path = output_path,
                        zip_format = zip_format,
                        prefix = prefix,
                        dump_vel = dump_vel,
                        output_start = output_start)

    simulation = Simulation(model = model,
                        timestepping_params = timestepping_params,
                        output_params = output_params)

    #perform the simulation
    run_simulation!(simulation)
    return simulation
end

#########################################################
############### Test flags ##############################
test_basic_outputting = true
test_zipping_output   = true
test_output_after_restart = true
test_output_errors = true
##########################################################

@testset "Outputting" begin
    if test_basic_outputting
        @testset "Output files" begin
            @info "Testing outputting..."

            for output_format in ["mat", "jld2"] #check that both mat and jld2 work
                for folder = ["outputs/", "outputs", "./outputs"] #check that both with and without / works
                    isdir(folder) && rm(folder, force = true, recursive = true) #make a new folder
                    mkdir(folder)

                    #run the simulation
                    sim = output_test(output_path = folder,
                                end_time = 100.,
                                output_freq = 5.,
                                prefix = "testoutfile",
                                output_format = output_format)

                    foldersim = sim.output_params.output_path
                    @test foldersim[end] == '/' #test that we do have the / at end of path
                    files = [string(foldersim,f) for f in  readdir(foldersim) if endswith(f, output_format)]
                    println(files)
                    @test length(files) == 20    #check there are the correct number of output files

                    #check they have the correct suffix
                    prefices = [split(str,".")[1] for str in files]
                    suffices = [split(str,".")[end] for str in files]
                    @test all(suffices .== output_format)

                    #check that variables included and correct size
                    fname = string(foldersim,readdir(folder)[1])
                    if output_format == "mat"
                    dict = matread(fname)
                    elseif output_format == "jld2"
                    dict = load(fname)
                    end
                    @test length(dict["t"]) == 1
                    @test size(dict["x"]) == (80,10)
                    @test size(dict["y"]) == (80,10)
                    @test size(dict["b"]) == (80,10)
                    @test size(dict["u"]) == (80,10)
                    @test size(dict["h"]) == (80,10)
                    @test size(dict["v"]) == (80,10)

                    #check it has the correct prefix and number of zeros
                    fname = readdir(folder)[1]
                    @test fname[1:11]  == "testoutfile"

                    #delete the folder
                    rm(folder, force = true, recursive = true)
                end
            end #end loop over output format

        end
        @testset "Test output start" begin
            for output_start in [true, false]
                folder = "outputs/"
                isdir(folder) && rm(folder, force = true, recursive = true) #make a new folder
                mkdir(folder)
                #test that if we only get an output at the start (e.g. if the output freq > end time), then outputted solution matches the expected thing
                sim = output_test(output_path = folder,
                                end_time = 1.,
                                output_freq = 5.,
                                prefix = "testoutfile",
                                output_format = "jld2",
                                output_start = output_start)

                foldersim = sim.output_params.output_path
                if output_start
                    #check only a single file outputted
                    @test length(readdir(foldersim)) == 1

                    fname = string(foldersim,readdir(folder)[1])
                    dict = load(fname)

                    #check that thickness same as IC and time = 0
                    @test dict["h"] == sim.model.initial_conditions.initial_thickness
                    @test dict["t"] == 0.0

                else
                    #check that no file output
                    @test isempty(readdir(foldersim))
                end

                #delete the folder
                rm(folder, force = true, recursive = true)
            end
        end
    end

    if test_zipping_output
        @testset "Zipping output" begin
            @info "Testing zipping output..."
            for output_format in ["mat", "jld2"]
                for output_start in [true, false]
                    folder = "outputs/"
                    isdir(folder) && rm(folder, force = true, recursive = true) #make a new folder
                    mkdir(folder)

                    sim = output_test(zip_format = "nc",
                                        output_path = folder,
                                        output_format = "mat",
                                        output_start = output_start)

                    @test sim isa Simulation
                    fname = string(folder, sim.output_params.prefix, ".nc")
                    @test isfile(fname) #check the zipped file exists

                    #test variables read from nc file
                    x = ncread(fname, "x")
                    y = ncread(fname, "y")
                    h = ncread(fname, "h")
                    u = ncread(fname, "u")
                    v = ncread(fname, "v")
                    b = ncread(fname, "b")
                    t = ncread(fname, "TIME")
                    if output_start
                        @test ncread(fname, "TIME") == 0.:5.:100.
                    else
                        @test ncread(fname, "TIME") == 5.:5.:100.
                    end

                    @test size(x) == (80,)
                    @test size(y) == (10,)
                    @test size(b) == (length(x),length(y),length(t))
                    @test size(u) == (length(x),length(y),length(t))
                    @test size(v) == (length(x),length(y),length(t))
                    @test size(h) == (length(x),length(y),length(t))

                    #delete the folder
                    rm(folder, force = true, recursive = true)
                end
            end
        end
    end

    if test_output_after_restart
        @testset "Outputting after a restart" begin
            @info "Testing outputting after a restart"

            for output_format in ["mat", "jld2"]

                folder = "outputs/"
                isdir(folder) && rm(folder, force = true, recursive = true) #make a new folder
                mkdir(folder)

                #run simulation first time with no zipping
                output_test(; dt  = 0.5,
                            end_time = 10.,
                            output_freq = 0.5,
                            output_format = output_format,
                            output_path = folder,
                            zip_format = "nc",
                            dump_vel = true,
                            pchkpt_freq = 1.)

                #get the time that the first file was outputted (test for https://github.com/RJArthern/WAVI.jl/issues/35)
                first_file_name = joinpath("outputs",string("outfile0000000001.", output_format));
                @test isfile(first_file_name) #check the first output point exists
                dt1 = Dates.unix2datetime(mtime(first_file_name)) #time of last modification

                #run again with different niter0
                sim = output_test(; dt  = 0.5,
                            end_time = 20.,
                            output_freq = 0.5,
                            niter0 = 20,
                            output_format = output_format,
                            output_path = folder,
                            zip_format = "nc",
                            dump_vel = true,
                            pchkpt_freq = 1.)

                #check that the time that first file modified has not changed -- i.e. simulation has not touched outputs at earlier times than it should have
                dt2 = Dates.unix2datetime(mtime(first_file_name))
                @test dt1 == dt2

                #test variables read from nc file
                fname = string(folder, sim.output_params.prefix, ".nc")
                println(fname)
                x = ncread(fname, "x")
                y = ncread(fname, "y")
                h = ncread(fname, "h")
                u = ncread(fname, "u")
                v = ncread(fname, "v")
                b = ncread(fname, "b")
                t = ncread(fname, "TIME")
                @test ncread(fname, "TIME") == 0.5:0.5:20.
                @test size(x) == (80,)
                @test size(y) == (10,)
                @test size(b) == (length(x),length(y),length(t))
                @test size(u) == (length(x),length(y),length(t))
                @test size(v) == (length(x),length(y),length(t))
                @test size(h) == (length(x),length(y),length(t))

                #check we have the right number of output files
                foldersim = sim.output_params.output_path
                @test foldersim[end] == '/' #test that we do have the / at end of path
                files = [string(foldersim,f) for f in  readdir(foldersim) if endswith(f, output_format)]
                println(files)
                @test length(files) == 40    #check there are the correct number of output files

                #now repeat for just one timestep
                sim = output_test(; dt  = 0.5,
                                end_time = 20.5,
                                output_freq = 0.5,
                                niter0 = 40,
                                output_format = output_format,
                                output_path = folder,
                                zip_format = "nc",
                                dump_vel = true,
                                pchkpt_freq = 1.)

                #test variables read from nc file
                fname = string(folder, sim.output_params.prefix, ".nc")
                println(fname)
                x = ncread(fname, "x")
                y = ncread(fname, "y")
                h = ncread(fname, "h")
                u = ncread(fname, "u")
                v = ncread(fname, "v")
                b = ncread(fname, "b")
                t = ncread(fname, "TIME")
                @test ncread(fname, "TIME") == 0.5:0.5:20.5
                @test size(x) == (80,)
                @test size(y) == (10,)
                @test size(b) == (length(x),length(y),length(t))
                @test size(u) == (length(x),length(y),length(t))
                @test size(v) == (length(x),length(y),length(t))
                @test size(h) == (length(x),length(y),length(t))

                #check we have the right number of files
                foldersim = sim.output_params.output_path
                @test foldersim[end] == '/' #test that we do have the / at end of path
                files = [string(foldersim,f) for f in  readdir(foldersim) if endswith(f, output_format)]
                println(files)
                @test length(files) == 41    #check there are the correct number of output files

                #check we have dumped the velocity
                @test isfile(string(foldersim, "outfile.nc")) #check the zipped file exists

                # delete everything you just made
                rm(folder, force = true, recursive = true)
                foreach(rm, filter(endswith(".mat"), readdir()))
                foreach(rm, filter(endswith(".jld2"), readdir()))
                foreach(rm, filter(endswith(".bin"), readdir()))
            end
        end
    end

    if test_output_errors
        @testset "Outputting errors" begin
            @info "Testing outputting errors"
            #check error for weird output format
            @test_throws ArgumentError output_test(output_format = "incorrect_format")

            #check that non-standard zip_format reverts to none
            sim = output_test(end_time = 1., zip_format = "incorrect_zip_format")
            @test sim.output_params.zip_format == "none"

            @test_throws ArgumentError output_test(output_freq =  -2.)

        end
    end
end


module TemporalGPs

    using AbstractGPs
    using BlockDiagonals
    using ChainRulesCore
    using FillArrays
    using LinearAlgebra
    using KernelFunctions
    using Random
    using StaticArrays
    using StructArrays
    using Zygote
    using ZygoteRules

    using FillArrays: AbstractFill
    using Zygote: _pullback, AContext

    import AbstractGPs: mean, cov, logpdf, FiniteGP, AbstractGP, posterior, dtc, elbo

    using KernelFunctions:
        SimpleKernel,
        KernelSum,
        ScaleTransform,
        ScaledKernel,
        TransformedKernel

    export
        to_sde,
        SArrayStorage,
        ArrayStorage,
        RegularSpacing,
        checkpointed,
        posterior,
        logpdf_and_rand,
        Separable

    # Various bits-and-bobs. Often commiting some type piracy.
    include(joinpath("util", "harmonise.jl"))
    include(joinpath("util", "linear_algebra.jl"))
    include(joinpath("util", "scan.jl"))
    include(joinpath("util", "zygote_friendly_map.jl"))
    include(joinpath("util", "zygote_rules.jl"))
    include(joinpath("util", "gaussian.jl"))
    include(joinpath("util", "mul.jl"))
    include(joinpath("util", "storage_types.jl"))
    include(joinpath("util", "regular_data.jl"))

    # Linear-Gaussian State Space Models.
    include(joinpath("models", "linear_gaussian_conditionals.jl"))
    include(joinpath("models", "gauss_markov_model.jl"))
    include(joinpath("models", "lgssm.jl"))
    include(joinpath("models", "missings.jl"))

    # Converting GPs to Linear-Gaussian SSMs.
    include(joinpath("gp", "data_representations.jl"))
    include(joinpath("gp", "lti_sde.jl"))
    include(joinpath("gp", "posterior_lti_sde.jl"))

    # Converting space-time GPs to Linear-Gaussian SSMs.
    include(joinpath("space_time", "rectilinear_grid.jl"))
    include(joinpath("space_time", "regular_in_time.jl"))
    include(joinpath("space_time", "separable_kernel.jl"))
    include(joinpath("space_time", "to_gauss_markov.jl"))
    include(joinpath("space_time", "pseudo_point.jl"))
end # module


"""
Function creates writeable object and creates & writes -∞ Array to it. Array Dimmensions are not stored.
# Arguments
- `dimsize::NTuple{N,:<Int}`: Dimmensions of an array to write
- `dir_name::String = "/tmp/io.mmap": Path to write a file.
"""
function create_mmap_array(dimsize::NTuple{N,<:Integer}, dir_name::String = "/tmp/io.mmap") where N

    s = Mmap.open(dir_name, "w+")
    Mmap.write(s, repeat([-Inf::Float64], dimsize... ) )
    close(s)

end

"""
Function opens with r+ permission a readable file with an array and returns IO Array (Mmap)
# Arguments
- `dimsize::NTuple{N,:<Int}`: Ntuple of dimensions of Array
- `dir_name::String = "/tmp/io.mmap": Path to a file.
"""
function read2write_mmap_array(dimsize::NTuple{N,<:Integer}, dir_path::String = "/tmp/io.mmap") where N

    s = Mmap.open(dir_path, "r+")
    A = Mmap.mmap(s, Array{Float64,length(dimsize)}, dimsize)
    close(s)
    return(A)

end

"""
Function opens with r permission a readable file with an array and returns IO Array (Mmap)
# Arguments
- `dimsize::NTuple{N,:<Int}`: Ntuple of dimensions of Array
- `dir_name::String = "/tmp/io.mmap": Path to a file.
"""
function read_mmap_array(dimsize::NTuple{N,<:Integer}, dir_path::String = "/tmp/io.mmap") where N

    s = Mmap.open(dir_path, "r")
    A = Mmap.mmap(s, Array{Float64,length(dimsize)}, dimsize)
    close(s)
    return(A)

end

"""
Function that returns Utility for given prices and coordinates. Requires precompilation of Utility Cube. MMap (hard drive) version. Requires precompilation of utils cube on disk. Accesses values directly. Requires few GB of space.
# Arguments
- `ind_vec::NTuple{M,<:Integer}`: coordinates of Utility.
- `mdl::Model`: mutable struct Model."""

function cube_Mmap( ind_vec::Vector{<:Integer}, mdl::Model, additional_args=nothing)

    grid = mdl.GridDef
    index, status = ndim2lin(ind_vec, mdl.GridDef)
    return(mdl.Utils.qb[ status ][ index ])

end

"""
Function building cube. GPU version. Requires precompilation of utils cube in VRAM. Accesses values directly.Requires as many GPUs as there are statuses.
# Arguments
- `mdl::ModelInit`: struct ModelInit.
# Examples```julia-repl\njulia> execute_cube(my_model, "CPU")\n```
"""
function cube_Mmap_qb(mdl::Model)

    n_dim = tuple( length.(mdl.GridDef.Grids)[1:(end-1)]... )
    cube = Vector{Array{Float64}}()

    for status = mdl.GridDef.Grids[end]

        path_ = joinpath(pwd(), "/tmp/utils_" * string(status) * ".io")
        create_mmap_array(n_dim, path_)
        append!( cube, [ read2write_mmap_array( n_dim, path_ ) ] )

    end

    Threads.@threads for idx = CartesianIndices(n_dim)

        for status = mdl.GridDef.Grids[end]

            ind_vec = CartesianIndex( idx, status )
            cube[status][idx] =  mdl.HAutility( mdl.HAconstraint( Tuple(ind_vec) , mdl )... )

        end

    end

    cube_out = Vector{Array{Float64}}()

    for status = mdl.GridDef.Grids[end]

        Mmap.sync!(cube[status])
        path_ = "/tmp/utils_" * string(status) * ".io"

        append!( cube_out, [read_mmap_array(n_dim, path_)])

    end

    return(cube_out)

end


using PrettyTables
using Statistics

function hasoutliers(data)
    Qi, Qf = quantile(data, [0.01, 0.99])
    return any( data .> Qf) || any(data .< Qi)
end

function raw_average(p = :fermi, engine = "MKL")
    if p == :fermi
        timings = get_fermi(engine)
        avg = [sum(timings[s,:])/10 for s = 1:22]
    else
        timings = get_psi4()
        avg = [sum(timings[s,:])/5 for s = 1:22]
    end
    return avg
end

function get_fermi(engine = "MKL")
    timings = zeros(22,10)
    for s in 1:22
        for i in 1:10
            rex = Regex("S$s - \\(T\\) alg $(engine == "MKL" ? 1 : 2)\\s+?$i run:\\s+([0-9]*\\.?[0-9]*)")
            timings[s,i] = get_linedata(rex, "fermi/output.dat")
        end
    end
    return timings
end

function get_psi4()
    Erex = r"FNOCC:\striples\s+:.+?([0-9]*\.?[0-9]*)w\s+[0-9]+\scalls"
    timings = zeros(22, 5)
    for s in 1:22
        for i in 1:5
            path = joinpath(@__DIR__, "psi4/R$i/S$s/timer.dat")
            timings[s,i] = get_linedata(Erex, path)
        end
    end
    return timings
end

function get_linedata(rex, fname)
    fpath = joinpath(@__DIR__, fname)
    out = 0.0
    f = false
    for l = eachline(fpath)
        m = match(rex, l)
        if m !== nothing
            out = m.captures[1] |> String |> x->parse(Float64,x)
            f = true
            break
        end
    end
    if !f
        error("Data not found for $(rex) in $(fname)")
    end
    return out
end

if abspath(PROGRAM_FILE) == @__FILE__
    fermi_mkl = get_mkl()
    fermi_octavian = get_octavian()
    psi4 = get_psi4()

    @pt :header = ["Fermi - MKL", "Fermi - Octavian", "Psi4"] [fermi_mkl fermi_octavian psi4]
end


# Naive implementation

# using Simulation_Env_module

mutable struct PrioritizedReplayBuffer
    max_size::Int64
    batch_size::Int64
    rng::AbstractRNG
    α::Float64
    β::Float64
    ϵ::Float64
    _curr_size::Int64
    _idx::Int64
    _priorities::Vector{Float64}
    _experience::Vector{DQExperience}

    _s_batch::Array{Float64}
    _a_batch::Vector{Int64}
    _r_batch::Vector{Float64}
    _sp_batch::Array{Float64}
    _done_batch::Vector{Bool}
    _weights_batch::Vector{Float64}

    function PrioritizedReplayBuffer(env::AbstractEnvironment,
                                    max_size::Int64,
                                    batch_size::Int64;
                                    rng::AbstractRNG = MersenneTwister(0),
                                    α::Float64 = 0.6,
                                    β::Float64 = 0.4,
                                    ϵ::Float64 = 1e-3)
        s_dim = obs_dimensions(env)
        experience = Vector{DQExperience}(max_size)
        priorities = Vector{Float64}(max_size)
        _s_batch = zeros(batch_size, s_dim...)
        _a_batch = zeros(Int64, batch_size)
        _r_batch = zeros(batch_size)
        _sp_batch = zeros(batch_size, s_dim...)
        _done_batch = zeros(Bool, batch_size)
        _weights_batch = zeros(Float64, batch_size)
        return new(max_size, batch_size, rng, α, β, ϵ, 0, 1, priorities, experience,
                   _s_batch, _a_batch, _r_batch, _sp_batch, _done_batch, _weights_batch)
    end
end

is_full(r::PrioritizedReplayBuffer) = r._curr_size == r.max_size

max_size(r::PrioritizedReplayBuffer) = r.max_size

function add_exp!(r::PrioritizedReplayBuffer, expe::DQExperience, td_err::Float64=abs(expe.r))
    @assert td_err + r.ϵ > 0.
    priority = (td_err + r.ϵ)^r.α
    r._experience[r._idx] = expe
    r._priorities[r._idx] = priority
    r._idx = mod1((r._idx + 1),r.max_size)
    if r._curr_size < r.max_size
        r._curr_size += 1
    end
end

function update_priorities!(r::PrioritizedReplayBuffer, indices::Vector{Int64}, td_errors::Float64)
    @assert td_errors + r.ϵ .> 0.
    new_priorities = (td_err + r.ϵ)^r.α
    r._priorities[indices] = new_priorities
end

function StatsBase.sample(r::PrioritizedReplayBuffer)
    @assert r._curr_size >= r.batch_size
    @assert r.max_size >= r.batch_size # could be checked during construction
    sample_indices = sample(r.rng, 1:r._curr_size, Weights(r._priorities[1:r._curr_size]), r.batch_size, replace=false)
    @assert length(sample_indices) == size(r._s_batch)[1]
    for (i, idx) in enumerate(sample_indices)
        r._s_batch[Base.setindex(indices(r._s_batch), i, 1)...] = r._experience[idx].s
        r._a_batch[i] = r._experience[idx].a
        r._r_batch[i] = r._experience[idx].r
        r._sp_batch[Base.setindex(indices(r._sp_batch), i, 1)...] = r._experience[idx].sp
        r._done_batch[i] = r._experience[idx].done
        r._weights_batch[i] = r._priorities[idx]
    end
    r._weights_batch = (r._weights_batch.*r._curr_size).^-r.β./maximum(r._weights_batch)
    return r._s_batch, r._a_batch, r._r_batch, r._sp_batch, r._done_batch, sample_indices, r._weights_batch
end

function populate_replay_buffer!(replay::PrioritizedReplayBuffer, env::AbstractEnvironment;
                                 max_pop::Int64=replay.max_size, max_steps::Int64=2000)
    render_option = env.render
    env.render = false
    println("Start populating PRIORITIZED replay buffer...")
    # println("Using closest intruder detection...")
    if !env.random_populate_replay_buffer
        println("Using multi intruder detection ...")
    else
        println("Using random ...")
    end
    o = reset(env)
    done = false
    step = 0
    for t = 1:(max_pop - replay._curr_size)
        if !env.random_populate_replay_buffer
            q_low_fi = lowfi_values(env, o) # multi-threat
            # q_low_fi = get_qvals(env.low_fi_policy, convert_ac_state(env.ego_agent.state)) # closest
            action = actions(env)[indmax(q_low_fi)]
        else
            action = sample_action(env)
        end
        ai = action_index(env.problem, action)
        op, rew, _, done, info = step!(env, action)
        exp = DQExperience(o, ai, rew, op, done)
        add_exp!(replay, exp, abs(rew)) # assume initial td error is r
        o = op
        step += 1
        if done || step >= max_steps
            o = reset(env)
            done = false
            step = 0
        end
    end
    @assert replay._curr_size >= replay.batch_size
    println("Done populating replay buffer!")
    env.render = render_option
end


@test   Options() == Options( 1e9, :ScatteringParameters, :MagnitudeAngle, 50.0 )
@test   TouchstoneData( DataPoint[], NoiseDataPoint[], Options( 1e9, :ScatteringParameters, :MagnitudeAngle, 50.0 ) ) == TouchstoneData( DataPoint[] )

@test DataPoint( 1.0, [ 1 2; 3 4 ] ) ==  DataPoint( 1.0, [ 1 2; 3 4 ] )
@test DataPoint( 2.0, [ 1 2; 3 4 ] ) !=  DataPoint( 1.0, [ 1 2; 3 4 ] )
@test DataPoint( 1.0, [ 2 2; 3 4 ] ) !=  DataPoint( 1.0, [ 1 2; 3 4 ] )

@test TouchstoneData( [ DataPoint( 1.0, [ 1 2; 3 4] ) ] ) == TouchstoneData( [ DataPoint( 1.0, [ 1 2; 3 4]) ] )
@test TouchstoneData( [ DataPoint( 2.0, [ 1 2; 3 4] ) ] ) != TouchstoneData( [ DataPoint( 1.0, [ 1 2; 3 4]) ] )
@test TouchstoneData( [ DataPoint( 1.0, [ 2 2; 3 4] ) ] ) != TouchstoneData( [ DataPoint( 1.0, [ 1 2; 3 4]) ] )

@test TouchstoneData( [
    DataPoint( 1.0, [ 1 2; 3 4]),
    DataPoint( 2.0, [ 1 2; 3 4])
  ] ) == TouchstoneData( [
    DataPoint( 1.0, [ 1 2; 3 4]),
    DataPoint( 2.0, [ 1 2; 3 4])
  ] )

@test TouchstoneData( [
    DataPoint( 1.0, [ 1 2; 3 4]),
    DataPoint( 2.0, [ 1 2; 3 4])
  ] ) != TouchstoneData( [
    DataPoint( 1.0, [ 1 2; 3 4]),
    DataPoint( 3.0, [ 1 2; 3 4])
  ] )

@test_throws ErrorException(
    "Frequencies of data vector not in ascending order."
  ) TouchstoneData( [
    DataPoint( 2.0, [ 1 2; 3 4]),
    DataPoint( 1.0, [ 1 2; 3 4])
  ] )

@test NoiseDataPoint( 1.0, 2.0, 3.0 + 4im, 5.0 ) ==  NoiseDataPoint( 1.0, 2.0, 3.0 + 4im, 5.0 )

@test NoiseDataPoint( 1.0, 2.0, 3.0 + 4im, 5.0 ) !=  NoiseDataPoint( 2.0, 2.0, 3.0 + 4im, 5.0 )
@test NoiseDataPoint( 1.0, 2.0, 3.0 + 4im, 5.0 ) !=  NoiseDataPoint( 2.0, 2.0, 3.0 + 4im, 5.0 )
@test NoiseDataPoint( 1.0, 2.0, 3.0 + 4im, 5.0 ) !=  NoiseDataPoint( 2.0, 1.0, 3.0 + 4im, 5.0 )
@test NoiseDataPoint( 1.0, 2.0, 3.0 + 4im, 5.0 ) !=  NoiseDataPoint( 2.0, 2.0, 5.0 + 4im, 5.0 )
@test NoiseDataPoint( 1.0, 2.0, 3.0 + 4im, 5.0 ) !=  NoiseDataPoint( 2.0, 2.0, 3.0 + 4im, 4.0 )

@test TouchstoneData( [
    DataPoint( 1.0, [ 1 2; 3 4])
  ], [
    NoiseDataPoint( 1.0, 2.0, 3.0 + 4im, 5.0 ),
    NoiseDataPoint( 2.0, 2.0, 3.0 + 4im, 5.0 )
  ] ) == TouchstoneData( [
    DataPoint( 1.0, [ 1 2; 3 4])
  ], [
    NoiseDataPoint( 1.0, 2.0, 3.0 + 4im, 5.0 ),
    NoiseDataPoint( 2.0, 2.0, 3.0 + 4im, 5.0 )
  ] )

@test TouchstoneData( [
    DataPoint( 2.0, [ 1 2; 3 4])
  ], [
    NoiseDataPoint( 1.0, 2.0, 3.0 + 4im, 5.0 )
  ] ) != TouchstoneData( [
    DataPoint( 2.0, [ 1 2; 3 4])
  ], [
    NoiseDataPoint( 2.0, 2.0, 3.0 + 4im, 5.0 )
  ] )

@test_throws ErrorException(
    "Frequencies of noise data vector not in ascending order."
  ) TouchstoneData( [
    DataPoint( 2.0, [ 1 2; 3 4])
  ], [
    NoiseDataPoint( 2.0, 2.0, 3.0 + 4im, 5.0 ),
    NoiseDataPoint( 1.0, 2.0, 3.0 + 4im, 5.0 )
  ] )

@test_throws ErrorException(
    "Last frequency of data vector is smaller than first frequency of noise data vector."
  ) TouchstoneData( [
    DataPoint( 1.0, [ 1 2; 3 4])
  ], [
    NoiseDataPoint( 2.0, 2.0, 3.0 + 4im, 5.0 )
  ] )


##### Sum wrapper

export Σfluxes, Σsources

"""
    flux

An individual flux.
See [`BalanceLaw`](@ref) for more info.
"""
function flux end

"""
    source

An individual source.
See [`BalanceLaw`](@ref) for more info.
"""
function source end

"""
    ntuple_sum(nt::NTuple{N,T}) where {N, T}

sum of `NTuple`, which requires more strict
type input than `sum`. This is added to better
synchronize the success/failure between CPU/GPU
runs to help improve debugging.
"""
ntuple_sum(nt::NTuple{N, T}) where {N, T} = sum(nt)

"""
    Σfluxes(fluxes::NTuple, bl, args)

Sum of the fluxes where
 - `fluxes` is an `NTuple{N, TendencyDef{Flux{O}}} where {N, O}`
 - `bl` is the balance law
 - `args` are the arguments passed to the individual `flux` functions
"""
function Σfluxes(
    pv::PV,
    fluxes::NTuple{N, TendencyDef{Flux{O}}},
    bl,
    args,
) where {N, O, PV}
    return ntuple_sum(
        ntuple(Val(N)) do i
            projection(pv, bl, fluxes[i], args, flux(pv, fluxes[i], bl, args))
        end,
    )
end

# Emptry scalar case:
function Σfluxes(
    pv::PV,
    fluxes::NTuple{0, TendencyDef{Flux{O}}},
    args...,
) where {O, PV}
    return SVector(0, 0, 0)
end

# Emptry vector case:
function Σfluxes(
    pv::PV,
    fluxes::NTuple{0, TendencyDef{Flux{O}}},
    args...,
) where {O, PV <: AbstractMomentumVariable}
    return SArray{Tuple{3, 3}}(ntuple(i -> 0, 9))
end

# Emptry tracer case:
function Σfluxes(
    pv::PV,
    fluxes::NTuple{0, TendencyDef{Flux{O}}},
    args...,
) where {O, N, PV <: AbstractTracersVariable{N}}
    return SArray{Tuple{3, N}}(ntuple(i -> 0, 3 * N))
end

"""
    Σsources(sources::NTuple, bl, args)

Sum of the sources where
 - `sources` is an `NTuple{N, TendencyDef{Source}} where {N}`
 - `bl` is the balance law
 - `args` are the arguments passed to the individual `source` functions
"""
function Σsources(
    pv::PV,
    sources::NTuple{N, TendencyDef{Source}},
    bl,
    args,
) where {N, PV}
    return ntuple_sum(
        ntuple(Val(N)) do i
            projection(
                pv,
                bl,
                sources[i],
                args,
                source(pv, sources[i], bl, args),
            )
        end,
    )
end

# Emptry scalar case:
function Σsources(
    pv::PV,
    sources::NTuple{0, TendencyDef{Source}},
    args...,
) where {PV}
    return 0
end

# Emptry vector case:
function Σsources(
    pv::AbstractMomentumVariable,
    sources::NTuple{0, TendencyDef{Source}},
    args...,
)
    return SVector(0, 0, 0)
end

# Emptry tracer case:
function Σsources(
    pv::AbstractTracersVariable{N},
    sources::NTuple{0, TendencyDef{Source}},
    args...,
) where {N}
    return SArray{Tuple{N}}(ntuple(i -> 0, N))
end


#### indexes

export DomainIdx, subdomains, alldomains, eachdomain, allcombinations

function get_idx(gm, en, ud)
    i_gm, i_en, i_ud = 0, 0, (0,)
    ud > 0 && (i_ud = Tuple([i for i in 1:ud]))
    en > 0 && (i_en = max(i_ud...) + 1)
    gm > 0 && en > 0 && (i_gm = i_en + 1)
    gm > 0 && !(en > 0) && (i_gm = max(i_ud...) + 1)
    return i_gm, i_en, i_ud
end

""" Constructor for DomainSet """
function DomainIdx(domain_set::DomainSet)
    i_gm, i_en, i_ud = get_idx(n_subdomains(domain_set)...)
    return DomainIdx{i_gm, i_en, i_ud}()
end

gridmean(::DomainIdx{GM, EN, UD}) where {GM, EN, UD} = GM
environment(::DomainIdx{GM, EN, UD}) where {GM, EN, UD} = EN
updraft(::DomainIdx{GM, EN, UD}) where {GM, EN, UD} = UD

has_gridmean(::DomainIdx{GM, EN, UD}) where {GM, EN, UD} = !(GM == 0)
has_environment(::DomainIdx{GM, EN, UD}) where {GM, EN, UD} = !(EN == 0)
has_updraft(::DomainIdx{GM, EN, UD}) where {GM, EN, UD} = !(UD == (0,))

# TODO: write a state-vector wrapper for the exported functions
"""
    subdomains(idx::DomainIdx)

Tuple of sub-domain indexes, including environment and updrafts
"""
subdomains(idx::DomainIdx) = (environment(idx), updraft(idx)...)

"""
    alldomains(idx::DomainIdx)

Tuple of all sub-domain indexes, including grid-mean, environment and updrafts
"""
alldomains(idx::DomainIdx) = (gridmean(idx), environment(idx), updraft(idx)...)

"""
    eachdomain(idx::DomainIdx)

Tuple of indexes including
 - grid-mean
 - environment
 - Tuple of updraft indexes
"""
eachdomain(idx::DomainIdx) = (gridmean(idx), environment(idx), updraft(idx))

"""
    allcombinations(idx::DomainIdx)

Tuple of indexes including
 - grid-mean
 - environment
 - Tuple of updraft indexes
 - Tuple of all sub-domains (environment and updrafts)
 - Tuple of all domains
"""
allcombinations(idx::DomainIdx) = (
    gridmean(idx),
    environment(idx),
    updraft(idx),
    subdomains(idx),
    alldomains(idx),
)

function DomainIdx(
    domain_set::DomainSet,
    domain_subset::DomainSubSet{BGM, BEN, BUD},
) where {BGM, BEN, BUD}
    i_gm, i_en, i_ud = get_idx(n_subdomains(domain_set, domain_subset)...)
    return DomainIdx{i_gm, i_en, i_ud}()
end

@inline get_i_state_vec(vm, a_map::AbstractArray, ϕ::Symbol, i) =
    vm[ϕ][a_map[i]]
@inline get_i_state_vec(domain_decomp::DomainDecomposition, ϕ::Symbol, i) =
    get_i_state_vec(domain_decomp.var_mapper, domain_decomp.a_map[ϕ], ϕ, i)
@inline get_i_var(a_map::AbstractArray, i) = a_map[i]
@inline get_i_var(domain_decomp::DomainDecomposition, ϕ, i) =
    get_i_var(domain_decomp.a_map[ϕ], i)

function var_suffix(vm, idx::DomainIdx, idx_ss::DomainIdx, ϕ::Symbol, i)
    if i == gridmean(idx)
        return "_gm"
    elseif i == environment(idx)
        return "_en"
    elseif i in updraft(idx)
        return "_ud_" * string(i)
    else
        throw(BoundsError(vm[ϕ], i))
    end
end

var_string(vm, idx::DomainIdx, idx_ss::DomainIdx, ϕ::Symbol, i) =
    string(ϕ) * var_suffix(vm, idx, idx_ss, ϕ, i)

function var_suffix(domain_decomp::DomainDecomposition, ϕ::Symbol, i = 0)
    i == 0 && (i = gridmean(domain_decomp))
    return var_suffix(
        domain_decomp.var_mapper,
        domain_decomp.idx,
        domain_decomp.idx_ss_per_var[ϕ],
        ϕ,
        i,
    )
end

function var_string(domain_decomp::DomainDecomposition, ϕ::Symbol, i = 0)
    i == 0 && (i = gridmean(domain_decomp))
    return var_string(
        domain_decomp.var_mapper,
        domain_decomp.idx,
        domain_decomp.idx_ss_per_var[ϕ],
        ϕ,
        i,
    )
end


##
# sample test program to produce monthly aggregate count of smileys
# used in tweets between year 2006 to 2009, based on data from infochimps.

using DataFrames
using Gaston
using HDFS
using HDFS.MapReduce
using Blocks

import HDFS.MapReduce.find_rec

function find_rec(rdr, iter_status, t::Type{DataFrame}, rec_sep::Char='\n', col_sep::Char=',')
    (iter_status != nothing) && return (nothing, true, iter_status)
    ios = HDFS.MapReduce.get_stream(rdr)
    sz = nb_available(ios)
    # frequent peek done by DataFrame reader causes libhdfs to misbehave
    buff = Array(Uint8, sz)
    read(ios, buff)
    pb = PipeBuffer(buff)
    #println("bytes available: $sz")
    #df = readtable(pb, header=false, separator=col_sep, nrows=sz, buffersize=sz)
    df = as_dataframe(pb, header=false, separator=col_sep)
    return (df, false, position(ios))

    #(dlmarr, iseof, iter_status) = find_rec(rdr, iter_status, Matrix, rec_sep, col_sep)
    #df = (nothing == dlmarr) ? nothing : DataFrame(dlmarr)
    #println("hdfs_find_rec_table got $dlmarr")
    #(df, iseof, iter_status)
end

##
# find smiley records from HDFS CSV file
find_smiley_df(r, next_rec_pos) = find_rec(r, next_rec_pos, DataFrame, '\n', '\t')

##
# reduce smiley counts or array of counts
reduce_smiley_df(reduced, results...) = +((nothing == reduced) ? 0 : reduced, filter(x->(x!=nothing),results)...)

##
# for finding total counts across all years
function map_total(rec)
    #println("map got $rec")
    (nothing == rec) && return []
    [sum(rec[3])]
end

function collect_total(results, rec)
    #println("collect got $rec")
    (length(rec) == 0) && return results
    (results == nothing) ? rec : (results+rec)
end

function do_dataframe_test(furl::String, use_hdfs=true)
    println("starting dmapreduce job...")
    finp = use_hdfs ? MRHdfsFileInput([furl], find_smiley_df) : MRFsFileInput([furl], find_smiley_df, 1024*16)
    j = dmapreduce(finp, map_total, collect_total, reduce_smiley_df)
    println("waiting for jobs to finish...")
    loopstatus = true
    while(loopstatus)
        sleep(2)
        jstatus,jstatusinfo = status(j,true)
        ((jstatus == "error") || (jstatus == "complete")) && (loopstatus = false)
        (jstatus == "running") && println("$(j): $(jstatusinfo)% complete...")
    end

    wait(j)
    println("time taken (total time, wait time, run time): $(times(j))")
    println("")
    jstatus,total_count = results(j)
    println("completed with status: $jstatus")
    if(jstatus == "complete")
        println("results:")
        println("total number of smileys: $total_count")
    else
        println(status(j, true))
    end
    unload(j) # ensure file handles are closed cleanly
end


### Imports ###

import Base: size, setindex!, getindex, broadcast!, Broadcast.fill!, Broadcast.fill

### Exports ###

export ZeroArray, RingBuffer, AnyNode
export fasttanh, canthread

### Types ###

# Useful type for writing/reading inactive containers efficiently
mutable struct ZeroArray{T, N} <: DenseArray{T, N}
  size::NTuple{N, Int}
end
ZeroArray(T::Type, size::NTuple{N,Int}) where N = ZeroArray{T,N}(size)
ZeroArray(arr::DenseArray{T,N}) where {T,N} = ZeroArray{T,N}(size(arr))

mutable struct RingBuffer{T}
  buf::Matrix{T}
  pos::Int
end
RingBuffer(T, width, length) = RingBuffer(zeros(T, width, length), 1)

# TODO: LockedArray for locking individual arrays

mutable struct AnyNode{T} <: AbstractNode
  item::T
end

### Methods ###

Base.size(za::ZeroArray) = za.size
# TODO: Check for OOB situation?
Base.setindex!(za::ZeroArray{T,N}, value, idx...) where {T,N} = zero(T)
Base.getindex(za::ZeroArray{T,N}, idx...) where {T,N} = zero(T)
# TODO: More broadcast overrides
Base.broadcast!(f, za::ZeroArray{T,N}, args...) where {T,N} = za
Base.Broadcast.fill!(za::ZeroArray{T,N}, value) where {T,N} = za
Base.Broadcast.fill(za::ZeroArray{T,N}, value::T) where {T,N} =
  (value == zero(T) ? za : error("Cannot fill a ZeroArray with a non-zero element"))

function rotate!(rb::RingBuffer)
  if rb.pos < size(rb.buf, 2)
    rb.pos += 1
  else
    rb.pos = 1
  end
end
function _rbindex(rb::RingBuffer, idx)
  @assert 0 <= idx <= size(rb.buf, 2)-1 "Invalid RingBuffer index: $idx"
  npos = rb.pos - idx
  if npos < 1
    npos += size(rb.buf, 2)
  end
  npos
end
Base.getindex(rb::RingBuffer, idx::Int) = getindex(rb.buf, Colon(), _rbindex(rb, idx))
Base.getindex(rb::RingBuffer, ::Colon) = getindex(rb.buf, Colon(), rb.pos)
function Base.getindex(rb::RingBuffer{T}, idx::Vector{Int}) where T
  @assert length(idx) == size(rb.buf, 1) "Invalid RingBuffer indexing vector size: $(length(idx))"
  result = zeros(T, size(rb.buf, 1))
  for i in axes(rb.buf, 1)
    result[i] = rb.buf[i, _rbindex(rb, idx[i])]
  end
  result
end
Base.setindex!(rb::RingBuffer, value, idx::Int) = (rb.buf[:, _rbindex(rb, idx)] .= value)
Base.setindex!(rb::RingBuffer, value, ::Colon) = (rb.buf[:, rb.pos] .= value)
function Base.setindex!(rb::RingBuffer{T}, value::Vector{T}, idx::Vector{Int}) where T
  @assert length(idx) == size(rb.buf, 1) "Invalid RingBuffer indexing vector size: $(length(idx))"
  for i in axes(rb.buf, 1)
    rb.buf[i, _rbindex(rb, idx[i])] = value[i]
  end
end
Base.lastindex(rb::RingBuffer) = size(rb.buf, 2)
clear!(rb::RingBuffer{T}) where T = fill!(rb.buf, zero(T))

# Returns a resized array, with as many existing entries copied as possible
function resize_array(arr::Array{T,N} where {T,N}, new_size::NTuple{N,Int} where N, fill_value=zero(T))
  old_size = size(arr)
  new_arr = fill(fill_value, new_size)
  shared_region = CartesianRange(min.(old_size, new_size))
  copy!(new_arr, shared_region, arr, shared_region)
end

### Miscellaneous Utilities ###

fasttanh(x) = x * ( 27 + x * x ) / ( 27 + 9 * x * x )

canthread() = SAMANTHA_THREADS && !Threads.in_threaded_loop.x && Threads.nthreads() > 1


using JLD2
using BenchmarkTools
using Benchmarks

# Benchmark parameters

Nx = 256
Ny = 256
Nz = Nx

ranks = (1, 2, 4, 8, 16)

# Run and collect benchmarks

print_system_info()

for r in ranks
    @info "Benchmarking distributed incompressible model strong scaling [N=($Nx, $Ny, $Nz), ranks=$r]..."
    julia = Base.julia_cmd()
    run(`mpiexec -np $r $julia --project strong_scaling_incompressible_model_single.jl $Nx $Ny $Nz`)
end

suite = BenchmarkGroup(["size", "ranks"])
for r in ranks
    jldopen("strong_scaling_incompressible_model_$r.jld2", "r") do file
        suite[((Nx, Ny, Nz), r)] = file["trial"]
    end
end

# Summarize benchmarks

df = benchmarks_dataframe(suite)
sort!(df, :ranks)
benchmarks_pretty_table(df, title="Incompressible model strong scaling benchmark")

suite_Δ = speedups_suite(suite, base_case=((Nx, Ny, Nz), 1))
df_Δ = speedups_dataframe(suite_Δ)
sort!(df_Δ, :ranks)
benchmarks_pretty_table(df_Δ, title="Incompressible model strong scaling speedup")


"""
This is a sample input file for the CloudScat code.  To use it run julia
and type

julia> include("ozone_absorption.jl"); run()

The output file will be based on this file's name so to run different simulations
you can just copy this file, change the configuration and run it as above with
the new file name.  You can also run from the command line as e.g.

> julia ozone_absorption.jl

- - -
2020 Alejandro Luque IAA-CSIC (aluque@iaa.es)

"""

using CloudScat
using DataFrames
using DelimitedFiles

const co = CloudScat.constants

# This example depends on the ozone profiles used by MODTRAN, which cannot
# be redistributed.  If you have working MODTRAN installation, modify this
# to point to the folder with the profiles.
const MODTRAN_PROFILES = joinpath(@__DIR__, "profiles")

function run()
    profiles = ["MidLat_Summer.dat",
                "MidLat_Winter.dat",
                "Tropical.dat",
                "US_Std_1976.dat"]

    for prof in profiles
        run_profile(prof, "Hale.dat")
    end

    run_profile("Tropical.dat", "WarrenBrandt.dat")
end

function run_profile(fname, refindex)
    df = load_profile(joinpath(MODTRAN_PROFILES, fname))
    @info "Profile $(fname) loaded"

    # This is the absorption cross-section for 200 nm, where ozone absorption
    # is lowest.  Obtained from
    # L.T Molina and M.J. Molina, "Absolute absorption cross sections of
    # ozone in the 185- to 350-nm wavelength range,"
    # J. Geophys. Res. 91, 14501-14508 (1986).
    σ = 3.145E-19 * co.centi^2

    # The profiles reach only 100 km but the observer is above this altitude
    # We simply add another point with a negligible O3 density.
    z = [df.Z; 1000.0]
    dens = [df.O3; 0.0]
    T = [df.T; df.T[end]]

    absorb = StratifiedAbsorption(σ,
                                  (@. z * co.kilo),
                                  (@. dens * 1e-5 * co.atm / co.k / T))

    params = Params(
        # NUmber of simulated photons
        N = 10000000,

        # The wavelength of the scattered light
        λ = 200 * co.nano,

        # The source is defined as a straight line running from source_a
        # to source_b.  Within this line photons are distributed uniformly with
        # initially random, isotropic directions.  To use a point source set
        # source_b = source_a
        source_a = [0, 0, 10 * co.kilo],
        source_b = [0, 0, 10 * co.kilo],

        # Minimum weight before a particle is discarded
        weight_min = 1e-7)

    # World geometry
    # Describe the cloud geometry here.  It may consist in combinations of
    # geometrical figures (cylinders, spheres, cones, semi-planes).
    # See cloud_gemoetry.jl for a more detailed description
    #
    # A cylinder is defined as Cylinder(zbottom, ztop, xcenter, ycenter, radius)
    c1 = Cylinder(7 * co.kilo,  15 * co.kilo, 0, 0, 200 * co.kilo)
    cloud = c1

    # Geometry of the full domain.  Photons outside the domain are
    # immediately discarded.
    domain = Cylinder(7 * co.kilo, 60 * co.kilo, 0, 0, 200 * co.kilo)

    # Set a homogeneous cloud droplet density and radius.
    # See variable_droplet_radius.jl for how to deal with inhomogeneous clouds.
    # Radius of the scattering particles
    radius = 10e-6

    # Density of the scattering centers.
    nscat = 100 * co.centi^-3

    composition = Homogeneous(params.λ, nscat, radius,
                              refindex=refindex)

    # Define the full simulation world
    world = World(cloud, domain, composition, absorb)

    # Observers record a time curve and an image of received light.  You can
    # set up as many observers as you wish but they have a significant impact
    # on performance.

    observers = [Observer(
        # Location of the observer
        position = [0, 0, 400 * co.kilo],

        # Sampling interval
        tsample = 1e-5,

        # Collected samples
        nsamples = 10000,

        # fov of the camera (half of the diagonal fov).
        fov = 40,

        # camera pixels
        pixels = 1024)]

    basename = splitext(splitdir(@__FILE__)[2])[1]
    profbasename = splitext(fname)[1]
    refindexbasename = splitext(refindex)[1]

    # The saveto option specifies the output file.  Leave it as it is to base
    # the filename on the name of this file specifying the parameters.
    CloudScat.main(params, world, observers,
                   saveto="$(basename)_$(profbasename)_$(refindexbasename).h5")
end

function load_profile(fname)
    local d, colnames

    open(fname) do fd
        for i in 1:2
            readline(fd)
        end
        colnames = map(Symbol, (split(readline(fd))))
        readline(fd)

        d = readdlm(fd)
    end
    # From atm * cm / km to m^-3
    # but let's leave this to the code using the dataframe
    # @. d[:, 5:end] = d[:, 5:end] * 1e-5 * co.atm / co.k * T

    dict = Dict(col => data for (col, data) in zip(colnames, eachcol(d)))
    df = DataFrame(;dict...)
end

# Check if the code has been 'included' or run from the shell.  In the latter
# case, run the simulation.
isinteractive() || run()