"""
  Package: FishEBM
  File: simulate.jl

  Description: Brings together all of the functions necessary for a life cycle
    simulation

  Returns: Vector(EnviroAgent)

  Contributors: Justin Angevaare, Devin Rose

  Last update: October 2016
"""

function simulate(carrying_capacity::Vector{Float64}, effort::Vector{Float64}, initStock::Vector{Int64},
  stock_age::Vector{Int64}, e_a::EnvironmentAssumptions, adult_a::AdultAssumptions,
  age_a::AgentAssumptions; progress::Bool=true, plotPopDensity::Bool=false,
  plotPopDistribution::Bool=false, limit::Int64=1000000, simDescription::String="")

  # preconditions
  @assert(all(carrying_capacity .> 0.), "There is at least one negative carrying capacity")
  @assert(plotPopDensity == false || plotPopDistribution == false, "Only one plot can be run during
    a simulation")
  @assert(length(initStock) == length(stock_age), "Unmatching vector length for initializing the
    population, stock length = $(length(initStock)) & $(length(stock_age))")
  @assert(length(effort) == length(carrying_capacity) || (length(effort) == 1 && effort[1] == 0.0),
    "Effort must be the same length as the carrying capacity, or a length of one with a value of zero (effort[1] = 0) for no harvest")

  # initialize agent db and hash the enviro
  years = length(carrying_capacity)
  a_db = AgentDB(e_a); hashEnvironment!(a_db, e_a);
  if plotPopDensity
    popDensity = initPopulationDensity(e_a)
  end

  #initialize the stock with incoming paramaters
  for i = 1:length(initStock)
    injectAgents!(a_db, e_a.spawningHash, initStock[length(initStock)+1-i], stock_age[length(initStock)+1-i])
  end

  #Memory allocation for population data storage
  stageDataFrame = DataFrame(Week = 0, Stage1 = 0, Stage2 = 0, Stage3 = 0, Stage4 = 0, Total = 0)
  adultDataFrame = DataFrame(Week = 0, Age2 = 0, Age3 = 0, Age4 = 0, Age5 = 0, Age6 = 0, Age7 = 0, Age8Plus = 0, Total = 0)

  #Memory allocation for mortality data storage (includes harvest)
  harvestDataFrame = DataFrame(Week = 0, Age2 = 0, Age3 = 0, Age4 = 0, Age5 = 0, Age6 = 0, Age7 = 0, Age8Plus = 0, Total = 0)
  harvestZoneData = DataFrame(Week = 0, z1 = 0, z2 = 0, z3 = 0, z4 = 0, z5 = 0, z6 = 0, z7 = 0, z8 = 0, z9 = 0, z10 = 0,
                              z11 = 0, z12 = 0, z13 = 0, z14 = 0, z15 = 0, z16 = 0, z17 = 0, z18 = 0, Total = 0)
  killedDataFrame = DataFrame(Week = 0, Natural = 0, Extra = 0, Compensatory = 0, Total = 0)

  #Memory allocation for spawning data storage
  spawnDataFrame = DataFrame(Week = 0, Age2 = 0, Age3 = 0, Age4 = 0, Age5 = 0, Age6 = 0, Age7 = 0, Age8Plus = 0, Total = 0)

  harvest_effort = fill(0., years)
  harvest_effort[1:length(effort)] = effort

  totalPopulation = sum(initStock)

  #initialize the progress meter
  if progress
    progressBar = Progress(years*52, 30, " $totalPopulation total agents, Year 1 (of $years), week 1 of simulation \n", 30)
  end

  # Specify the final week before beginning harvest season
  spawnMin = 39

  for y = 1:years
    for w = 1:52
      # Find total number of weeks passed in the simulation
      totalWeek = ((y-1)*52)+w

      # age specific population
      ageSpecificPop = fill(0, 7)
      for i = 1:length(a_db)
        for age = 2:8
          ageSpecificPop[age - 1] += getAgeSpecificPop(age, totalWeek, a_db[i].alive, a_db[i].weekNum, age_a)
        end #for age
      end #for i

      totalAdults = sum(ageSpecificPop)

      push!(adultDataFrame, vcat(totalWeek, ageSpecificPop..., sum(ageSpecificPop)))

      if progress
        progressBar.desc = " $totalAdults adults, $(ageSpecificPop[7]) age 8+, $totalPopulation total, Year $y (of $years), week $w of simulation "
        next!(progressBar)
      end

      # Harvest is all year but has peak periods
      harvest!(harvest_effort[y], totalWeek, a_db, e_a, adult_a, age_a, harvestDataFrame, harvestZoneData)

      # Spawn can be set to any week(s)
      if w > spawnMin
        spawn!(a_db, adult_a, age_a, e_a, totalWeek, carrying_capacity[y], spawnDataFrame)
      else
        push!(spawnDataFrame, (totalWeek, 0, 0, 0, 0, 0, 0, 0, 0))
      end

      #Agents are killed and moved weekly
      push!(killedDataFrame, (totalWeek, 0, 0, 0, 0))
      killAgeSpecific!(a_db, adult_a, ageSpecificPop, carrying_capacity[y], totalWeek, killedDataFrame)

      kill!(a_db, e_a, age_a, totalWeek, killedDataFrame)

      # Update agent locations weekly
      move!(a_db, age_a, e_a, totalWeek)

      #update population information
      stagePopulation = [0,0,0,0]; totalPopulation = 0;
      for j = 1:4
        stagePopulation[j] = getStagePopulation(j, totalWeek, a_db, age_a)
      end
      totalPopulation = sum(stagePopulation)

      push!(stageDataFrame, vcat(totalWeek,stagePopulation..., sum(stagePopulation)))

      #show a real time plot (every 10 weeks) of agent movement
      if plotPopDensity
        if w == 1 || w%10 == 0
          updatePopulationDensity!(a_db, popDensity, totalWeek)
          popPlot = spy(popDensity, Guide.title("Year = $y, week = $w, totalPop=$totalPopulation, totalAdult=$totalAdults"))
          display(popPlot)
        end
      end

      #show a real time plot (every 10 weeks) of adult age distribution
      if plotPopDistribution
        if w ==1 || w%10 == 0
          adultAge = [2,3,4,5,6,7,8]
          ageDistPlot = Gadfly.plot(x=adultAge, y=ageSpecificPop, Geom.point,
            Guide.xlabel("Adult age in years"), Guide.ylabel("Adult population"),
            Guide.title("Age Distribution Plot of Adult fish by age,\n y = $y, w = $w, totalPopulation = $totalPopulation"))
          display(ageDistPlot)
        end
      end

      #simulation failure protocol
      if totalPopulation == 0 || totalAdults > limit
        #simply for finishing the progress meter loops
        if progress
          for year = y:years
            for week = 1:52
              progressBar.desc = " $totalAdults adults, $totalPopulation total, Year $y (of $years), week $w of simulation "
              next!(progressBar)
            end
          end
        end

        removeEmptyClass!(a_db)
        description = simDescription*"\n\nSimulation population failure in year $y, week $w (total population
          = $totalPopulation, total adults = $totalAdults, population limit = $limit).\n\n"

        simSummary(adult_a, age_a, a_db, effort, ((length(carrying_capacity))*52), initStock, carrying_capacity,
                  stageDataFrame, adultDataFrame, harvestDataFrame, harvestZoneData, spawnDataFrame, killedDataFrame, description)
        return a_db
      end #population regulation failure
    end #end for week
    #Remove empty cohorts annually
    removeEmptyClass!(a_db)
  end #end for year

  description = string(simDescription, "\n\nSimulation was successfully completed.\n\n")

  simSummary(adult_a, age_a, a_db, effort, ((length(carrying_capacity))*52), initStock,
    carrying_capacity, stageDataFrame, adultDataFrame, harvestDataFrame, harvestZoneData,
    spawnDataFrame, killedDataFrame, description)

  return a_db
end


##########################################################
"""
$(TYPEDEF)

Abstract supertype of quantities
"""
abstract type AbstractQuantity{Ti<:Integer} end

##########################################################
"""
$(TYPEDEF)

A continuous quantity is represented by exactly one species number

$(TYPEDFIELDS)
"""
struct ContinuousQuantity{Ti} <: AbstractQuantity{Ti}
    """
    Species number representing the quantity
    """
    ispec::Ti

    """
    Quantity identifier allowing to use the quantity as index
    in parameter fields
    """
    id::Ti
end

"""
     ContinuousQuantity(system,regions; ispec=0, id=0)

Add continuous quantity to the regions listed in `regions`.

Unless specified in `ispec`, the species number is generated
automtically.

Unless specified by `id`, the quantity ID is generated automatically.
"""
function ContinuousQuantity(sys::AbstractSystem{Tv,Ti,Tm},regions; ispec=0, id=0) where {Tv,Ti,Tm}
    if ispec==0
        nspec=num_species(sys)
        nspec=nspec+1
    else
        nspec=ispec
    end
    sys.num_quantities+=1
    if id==0
        id=sys.num_quantities
    end
    enable_species!(sys,nspec,regions)
    ContinuousQuantity{Ti}(nspec,id)
end

##########################################################
"""
$(TYPEDEF)

An interface quantity is represented by exactly one species number

$(TYPEDFIELDS)
"""
struct InterfaceQuantity{Ti}<: AbstractQuantity{Ti}
    """
    Species number representing the quantity
    """
    ispec::Ti

    """
    Quantity identifier allowing to use the quantity as index
    in parameter fields
    """
    id::Ti
end

"""
     InterfaceQuantity(system,regions; ispec=0, id=0)

Add interface quantity to the boundary regions listed in `regions`.

Unless specified in `ispec`, the species number is generated
automtically.

Unless specified by `id`, the quantity ID is generated automatically.
"""
function InterfaceQuantity(sys::AbstractSystem{Tv,Ti,Tm},regions;ispec=1,id=0) where {Tv,Ti,Tm}
    if ispec==0
        nspec=num_species(sys)
        nspec=nspec+1
    else
        nspec=ispec
    end
    enable_boundary_species!(sys,nspec,regions)
    sys.num_quanties+=1
    if id==0
        id=sys.num_quantities
    end
    InterfaceQuantity{Ti}(nspec,id)
end

###########################################################
"""
$(TYPEDEF)

A discontinuous quantity is represented by different species
in neigboring regions.

$(TYPEDFIELDS)
"""
struct DiscontinuousQuantity{Ti} <: AbstractQuantity{Ti}
    """
    Species numbers representing the quantity in each region
    """
    regionspec::Vector{Ti}

    """
    Quantity identifier allowing to use the quantity as index
    in parameter fields
    """
    id::Ti
end

"""
     DiscontinuousQuantity(system,regions; regionspec=nothing, id=0)

Add discontinuous quantity to the regions listed in `regions`.

Unless specified in `regionspec`, the species numbers for each region
are generated automatically.

Unless specified by `id`, the quantity ID is generated automatically.
"""
function DiscontinuousQuantity(sys::AbstractSystem{Tv,Ti,Tm},regions; regionspec=nothing, id=0) where {Tv,Ti,Tm}
    rspec=zeros(Ti,num_cellregions(sys.grid))
    if regionspec==nothing
        nspec=num_species(sys)
        for ireg ∈ regions
            nspec=nspec+1
            enable_species!(sys,nspec,[ireg])
            rspec[ireg]=nspec
        end
    else
        for ireg ∈ regions
            enable_species!(sys,regionspec[ireg],[ireg])
            rspec[ireg]=regionspec[ireg]
        end
    end
    sys.num_quantities+=1
    if id==0
        id=sys.num_quantities
    end
    quantity=DiscontinuousQuantity{Ti}(rspec,id)
    quantity
end

"""
    $(SIGNATURES)

Number of quantities defined for system
"""
num_quantities(system::AbstractSystem) = system.num_quantities

"""
    subgrids(quantity, system)

Return a vector of subgrids containing a subgrid for each
region where discontinuous quantitiy is defined.
"""
function subgrids(quantity::DiscontinuousQuantity, sys)
    grid=sys.grid
    subgrids=Vector{ExtendableGrid}(undef,0)
    for ireg=1:num_cellregions(grid)
        ispec=quantity.regionspec[ireg]
        if ispec>0
            push!(subgrids,subgrid(grid,[ireg]))
        end
    end
    subgrids
end

"""
    views(quantity, subgrids,system)

Return a vector of subgrids containing a subgrid for each
region where discontinuous quantitiy is defined.
"""
function views(U, quantity::DiscontinuousQuantity, subgrids,sys)
    grid=sys.grid
    projections=Vector[]
    j=1
    for ireg=1:num_cellregions(grid)
        ispec=quantity.regionspec[ireg]
        if ispec>0
            push!(projections,view(U[ispec,:],subgrids[j]))
            j=j+1
        end
    end
    projections
end

function views(U, quantity::ContinuousQuantity, subgrids,sys)
    grid=sys.grid
    projections=Vector[]
    j=1
    for ireg=1:num_cellregions(grid)
        push!(projections,view(U[quantity.ispec,:],subgrids[j]))
        j=j+1
    end
    projections
end

# just return the first which comes into mind.
# we need to ensure homogeneity of bregion-region structure.
# if that is true, this works.
function get_cellregion(sys,ibc)
    grid=sys.grid
    bfregions=grid[BFaceRegions]
    cregions=grid[CellRegions]
    for ibface=1:num_bfaces(sys.grid)
        if bfregions[ibface]==ibc
            bfcells=grid[BFaceCells]
            return cregions[bfcells[ibface,1]]
        end
    end
    return 0
end

"""
    boundary_dirichlet(system, quantity, ibc, value)

Set Dirichlet boundary `value` for `quantity` at boundary `ibc`.
"""
function boundary_dirichlet!(sys::AbstractSystem, q::DiscontinuousQuantity, ibc, v)
    boundary_dirichlet!(sys,
                        q.regionspec[get_cellregion(sys,ibc)],
                        ibc,
                        v)
end

function boundary_dirichlet!(sys::AbstractSystem, q::ContinuousQuantity, ibc, v)
    boundary_dirichlet!(sys,
                        q.ispec,
                        ibc,
                        v)
end

"""
    node[quantity]
    edge[quantity]
Return species number on [`AbstractNode`](@ref) or [`AbstractEdge`](@ref)
"""
Base.getindex(q::ContinuousQuantity,node::AbstractNode)=q.ispec
Base.getindex(q::AbstractQuantity,edge::AbstractEdge)=q.ispec

Base.getindex(q::DiscontinuousQuantity,edge::Edge)=@inbounds q.regionspec[edge.region]
Base.getindex(q::DiscontinuousQuantity,edge::BEdge)=nothing
Base.getindex(q::DiscontinuousQuantity,node::Node)=@inbounds q.regionspec[node.region]

"""
    bnode[quantity]
Return species number of discontinuous quantity region `ireg`  adjacent
to  [`BoundaryNode`](@ref) for outer boundary nodes.
"""
Base.getindex(q::DiscontinuousQuantity,bnode::BNode)=@inbounds q.regionspec[bnode.cellregions[1]]

"""
    bnode[quantity,ireg]
Return species number of discontinuous quantity region `ireg`  adjacent
to  [`BoundaryNode`](@ref).
"""
Base.getindex(q::DiscontinuousQuantity,bnode::BNode,j)=@inbounds q.regionspec[bnode.cellregions[j]]

"""
    u[q,j]
Return value of quantity in unknowns on edge in flux callbacks.
"""
Base.getindex(u::AbstractEdgeData,q::AbstractQuantity,j)=@inbounds u[q[u.geom],j]

"""
    u[q]
Return value of quantity in unknowns on node in  node callbacks.
"""
Base.getindex(u::AbstractNodeData,q::AbstractQuantity)=@inbounds u[q[u.geom]]

"""
    f[q]=value
Set rhs value for quantity in callbacks
"""
Base.setindex!(f::AbstractNodeData,v,q::AbstractQuantity)=@inbounds f[q[f.geom]]=v

"""
    u[q,ireg]
Return value of discontinuous quantity in unknowns adjacent to unknowns on boundary node.
"""
Base.getindex(u::BNodeUnknowns,q::DiscontinuousQuantity,j)=@inbounds u[q[u.geom,j]]
Base.getindex(f::BNodeRHS,q::DiscontinuousQuantity,j)=@inbounds f[q[f.geom,j]]

"""
    f[q,ireg]=v
Set rhs value for discontinuous quantity in adjacent regions of  boundary node.
"""
Base.setindex!(f::BNodeRHS,v,q::DiscontinuousQuantity,j)=@inbounds f[q[f.geom,j]]=v

"""
    M[q,i]

Access columns  `M` using id of quantity `q`
"""
Base.getindex(m::AbstractMatrix,q::AbstractQuantity,j)= m[q.id,j]

"""
    M[q,i]

Set element of `M` using id of quantity `q`
"""
Base.setindex!(m::AbstractMatrix,v,q::AbstractQuantity,j)= m[q.id,j]=v

"""
    A[q]

Access columns  of Array `A` using id of quantity `q`
"""
Base.getindex(A::AbstractArray,q::AbstractQuantity)= A[q.id]

"""
    A[q]

Set element of `A` using id of quantity `q`
"""
Base.setindex!(A::AbstractMatrix,v,q::AbstractQuantity)= A[q.id]=v


module AtomicData

using Unitful
@unit Ry "Ry" Rydberg 13.605_693_009u"eV" false;
@unit Ha "Ha" Hartree 27.211_386_02u"eV" false;

# This __init__ etc. is necessary for Unitful for the units
# to be fully available when this package is loaded.
# (http://ajkeller34.github.io/Unitful.jl/stable/extending/)
const _local_Unitful_basefactors = Unitful.basefactors
function __init__()
    # To use the atomic units outside this module
    merge!(Unitful.basefactors, _local_Unitful_basefactors)
    Unitful.register(AtomicData)
end
Unitful.register(AtomicData)

using DataFrames
using CSV
using HTTP

function parse_J(J::AbstractVector)
    function parse_number(j::AbstractString)
        if occursin("/", j)
            //(parse.(Ref(Int), split(j, "/"))...)
        else
            parse(Int, j)
        end
    end
    map(J) do j
        if ismissing(j) || j == "---"
            missing
        elseif occursin(",", j)
            parse_number.(split(j, ","))
        else
            parse_number(j)
        end
    end
end

function parse_eng(E::AbstractVector{<:Union{Missing,String}}, unit)
    map(E) do EE
        if !ismissing(EE)
            for c in " ()[]"
                EE = replace(EE, c=>"")
            end
            v = parse(Float64, EE)
            if unit == u"Ha"
                v*u"Ry" |> u"Ha"
            else
                v*unit
            end
        else
            EE
        end
    end |> Vector{Union{Missing,Quantity}}
end

function parse_eng(E::AbstractVector{<:Union{Missing,<:Real}}, unit)
    map(E) do EE
        if !ismissing(EE)
            if unit == u"Ha"
                EE*u"Ry" |> u"Ha"
            else
                EE*unit
            end
        else
            EE
        end
    end |> Vector{Union{Missing,Quantity}}
end

function get_nist_data(io::IO, unit)
    df = io |> f -> CSV.File(f, delim='\t') |> DataFrame

    [df[:, 1:2] DataFrame(J = parse_J(df[!, 3]), Level = parse_eng(df[!, 4], unit), Uncertainty = parse_eng(df[!, 5], unit)) df[:, 6:end]]
end

function get_nist_data(name::String, unit)
    units = Dict(u"cm"^(-1) => 0,
                 u"eV" => 1,
                 u"Ry" => 2,
                 u"Ha" => 2
                 )
    http_name = replace(name, " " => "+")
    url = "https://physics.nist.gov/cgi-bin/ASD/energy1.pl?encodedlist=XXT2&de=0&spectrum=$(http_name)&units=$(units[unit])&upper_limit=&parity_limit=both&conf_limit=All&conf_limit_begin=&conf_limit_end=&term_limit=All&term_limit_begin=&term_limit_end=&J_limit=&format=3&output=0&page_size=15&multiplet_ordered=0&conf_out=on&term_out=on&level_out=on&unc_out=on&j_out=on&lande_out=on&perc_out=on&biblio=on&temp=&submit=Retrieve+Data"

    req_body = HTTP.request("GET", url).body |> String
    occursin("Invalid element symbol", req_body) && error("Failed to retrieve atomic data for $(name)")
    req_body |> IOBuffer |> io -> get_nist_data(io, unit)
end

macro get_nist_data(varname, name, args...)
    varstr = string(varname)
    glob = Expr(:global, varname)
    quote
        if !isdefined(Main, Symbol($varstr))
            $glob
            $(esc(varname)) = get_nist_data($name, $(args...))
        end
    end
end

export get_nist_data, @get_nist_data

include("latex_tables.jl")

end # module


using DataStructures
using NCDatasets
using Dates

function SaveOutput(fname)
  Dataset(fname,"c",attrib = OrderedDict("title" => "test")) do ds
    defDim(ds,"Alon",1)
    defDim(ds,"Alat",1)
    defDim(ds,"Alev",vleva)
    defDim(ds,"Olon",1)
    defDim(ds,"Olat",1)
    defDim(ds,"Olev",vlevo)
    defDim(ds,"time",1)
    defVar(ds,"time",Dates.datetime2epochms(mtime)/1000/3600/24,("time",),
      attrib = OrderedDict("units" => "days since 0000-01-01 00:00:00" ))
    defVar(ds,"Alon",lon,("Alon",),
      attrib = OrderedDict("units" => "degree"))
    defVar(ds,"Alat",lon,("Alat",),
      attrib = OrderedDict("units" => "degree"))
    defVar(ds,"Alev",ZAC,("Alev",),
      attrib = OrderedDict("units" => "m"))
    defVar(ds,"Olon",lon,("Olon",),
      attrib = OrderedDict("units" => "degree"))
    defVar(ds,"Olat",lon,("Olat",),
      attrib = OrderedDict("units" => "degree"))
    defVar(ds,"Olev",ZOC,("Olev",),
      attrib = OrderedDict("units" => "m"))
    defVar(ds,"THETA",ΘA,("Alev",),
      attrib = OrderedDict("units" => "degree Kelvin"))
    defVar(ds,"TA",TA,("Alev",),
      attrib = OrderedDict("units" => "degree Celsius"))
    defVar(ds,"qA",qA,("Alev",),
      attrib = OrderedDict("units" => "kg/kg"))
    defVar(ds,"UA",UA,("Alev",),
      attrib = OrderedDict("units" => "m/s"))
    defVar(ds,"rhoA",ρA,("Alev",),
      attrib = OrderedDict("units" => "kg/m^3"))
    defVar(ds,"TO",TO,("Olev",),
      attrib = OrderedDict("units" => "degree Celsius"))
    defVar(ds,"UO",UO,("Olev",),
      attrib = OrderedDict("units" => "m/s"))
    defVar(ds,"rhoO",ρO,("Olev",),
      attrib = OrderedDict("units" => "kg/kg"))
  end
end

function ReadOutput(fname)
  ds = NCDataset(fname)
  global lat=ds["Alat"][1]
  global lon=ds["Alon"][1]
  global time=ds["time"][1]
  global ZAC=ds["Alev"][:]
  global ZOC=ds["Olev"][:]
  global ΘA=ds["THETA"][:]
  global TA=ds["TA"][:]
  global qA=ds["qA"][:]
  global UA=ds["UA"][:]
  global ρA=ds["rhoA"][:]
  global TO=ds["TO"][:]
  global UO=ds["UO"][:]
  global ρO=ds["rhoO"][:]
end


#--- 1d1f1v ---#
function flux_kcu!(
    fw::T1,
    ff::T2,
    fL::T3,
    fR::T3,
    u::T4,
    ω::T4,
    nu,
    inK,
    γ,
    visRef,
    visIdx,
    Pr,
    dt,
    sfL = zeros(eltype(fL), axes(fL))::T3,
    sfR = zeros(eltype(fL), axes(fL))::T3,
) where {
    T1<:AbstractArray{<:AbstractFloat,1},
    T2<:AbstractArray{<:AbstractFloat,1},
    T3<:AbstractArray{<:AbstractFloat,1},
    T4<:AbstractArray{<:AbstractFloat,1},
}
    cd(@__DIR__)
    ccall(
        (:__kinetic_MOD_flux_kcu_1f1v, "fortran/kitmod.so"),
        Nothing,
        (
            Ref{Float64},
            Ref{Float64},
            Ref{Float64},
            Ref{Float64},
            Ref{Float64},
            Ref{Float64},
            Ref{Int},
            Ref{Float64},
            Ref{Float64},
            Ref{Float64},
            Ref{Float64},
            Ref{Float64},
            Ref{Float64},
            Ref{Float64},
            Ref{Float64},
        ),
        fw,
        ff,
        fL,
        fR,
        u,
        ω,
        nu,
        inK,
        γ,
        visRef,
        visIdx,
        Pr,
        dt,
        sfL,
        sfR,
    )
end

#--- 1d2f1v ---#
function flux_kcu!(
    fw::T1,
    fh::T2,
    fb::T2,
    hL::T3,
    bL::T3,
    hR::T3,
    bR::T3,
    u::T4,
    ω::T4,
    nu,
    inK,
    γ,
    visRef,
    visIdx,
    Pr,
    dt,
    shL = zeros(eltype(hL), axes(hL))::T3,
    sbL = zeros(eltype(bL), axes(bL))::T3,
    shR = zeros(eltype(hR), axes(hR))::T3,
    sbR = zeros(eltype(bR), axes(bR))::T3,
) where {
    T1<:AbstractArray{<:AbstractFloat,1},
    T2<:AbstractArray{<:AbstractFloat,1},
    T3<:AbstractArray{<:AbstractFloat,1},
    T4<:AbstractArray{<:AbstractFloat,1},
}
    cd(@__DIR__)
    ccall(
        (:__kinetic_MOD_flux_kcu_2f1v, "fortran/kitmod.so"),
        Nothing,
        (
            Ref{Float64},
            Ref{Float64},
            Ref{Float64},
            Ref{Float64},
            Ref{Float64},
            Ref{Float64},
            Ref{Float64},
            Ref{Float64},
            Ref{Float64},
            Ref{Int},
            Ref{Float64},
            Ref{Float64},
            Ref{Float64},
            Ref{Float64},
            Ref{Float64},
            Ref{Float64},
            Ref{Float64},
            Ref{Float64},
            Ref{Float64},
            Ref{Float64},
        ),
        fw,
        fh,
        fb,
        hL,
        bL,
        hR,
        bR,
        u,
        ω,
        nu,
        inK,
        γ,
        visRef,
        visIdx,
        Pr,
        dt,
        shL,
        sbL,
        shR,
        sbR,
    )
end


# This file is a part of Julia. License is MIT: http://julialang.org/license

abstract AbstractCartesianIndex{N} # This is a hacky forward declaration for CartesianIndex
typealias NonSliceIndex Union{Colon, AbstractArray}
typealias ViewIndex Union{Real, NonSliceIndex}
typealias ScalarIndex Union{Real, AbstractCartesianIndex}

# L is true if the view itself supports fast linear indexing
immutable SubArray{T,N,P,I,L} <: AbstractArray{T,N}
    parent::P
    indexes::I
    offset1::Int       # for linear indexing and pointer, only valid when L==true
    stride1::Int       # used only for linear indexing
    function SubArray(parent, indexes, offset1, stride1)
        check_parent_index_match(parent, indexes)
        new(parent, indexes, offset1, stride1)
    end
end
# Compute the linear indexability of the indices, and combine it with the linear indexing of the parent
function SubArray(parent::AbstractArray, indexes::Tuple, dims::Tuple)
    SubArray(linearindexing(viewindexing(indexes), linearindexing(parent)), parent, indexes, dims)
end
function SubArray{P, I, N}(::LinearSlow, parent::P, indexes::I, dims::NTuple{N})
    SubArray{eltype(P), N, P, I, false}(parent, indexes, 0, 0)
end
function SubArray{P, I, N}(::LinearFast, parent::P, indexes::I, dims::NTuple{N})
    # Compute the stride and offset
    stride1 = compute_stride1(parent, indexes)
    SubArray{eltype(P), N, P, I, true}(parent, indexes, compute_offset1(parent, stride1, indexes), stride1)
end

check_parent_index_match(parent, indexes) = check_parent_index_match(parent, index_ndims(indexes...))
check_parent_index_match{T,N}(parent::AbstractArray{T,N}, ::NTuple{N, Bool}) = nothing
check_parent_index_match{N}(parent, ::NTuple{N, Bool}) =
    throw(ArgumentError("number of indices ($N) must match the parent dimensionality ($(ndims(parent)))"))

# This computes the linear indexing compatability for a given tuple of indices
viewindexing() = LinearFast()
# Leading scalar indexes simply increase the stride
viewindexing(I::Tuple{ScalarIndex, Vararg{Any}}) = (@_inline_meta; viewindexing(tail(I)))
# Colons may begin a section which may be followed by any number of Colons
viewindexing(I::Tuple{Colon, Colon, Vararg{Any}}) = (@_inline_meta; viewindexing(tail(I)))
# A UnitRange can follow Colons, but only if all other indices are scalar
viewindexing(I::Tuple{Colon, UnitRange, Vararg{ScalarIndex}}) = LinearFast()
# In general, ranges are only fast if all other indices are scalar
viewindexing(I::Tuple{Union{Range, Colon}, Vararg{ScalarIndex}}) = LinearFast()
# All other index combinations are slow
viewindexing(I::Tuple{Vararg{Any}}) = LinearSlow()
# Of course, all other array types are slow
viewindexing(I::Tuple{AbstractArray, Vararg{Any}}) = LinearSlow()

# Simple utilities
size(V::SubArray) = (@_inline_meta; map(n->Int(unsafe_length(n)), indices(V)))

similar(V::SubArray, T::Type, dims::Dims) = similar(V.parent, T, dims)

parent(V::SubArray) = V.parent
parentindexes(V::SubArray) = V.indexes

parent(a::AbstractArray) = a
"""
    parentindexes(A)

From an array view `A`, returns the corresponding indexes in the parent.
"""
parentindexes(a::AbstractArray) = ntuple(i->OneTo(size(a,i)), ndims(a))

## SubArray creation
# We always assume that the dimensionality of the parent matches the number of
# indices that end up getting passed to it, so we store the parent as a
# ReshapedArray view if necessary. The trouble is that arrays of `CartesianIndex`
# can make the number of effective indices not equal to length(I).
_maybe_reshape_parent(A::AbstractArray, ::NTuple{1, Bool}) = reshape(A, Val{1})
_maybe_reshape_parent{_,N}(A::AbstractArray{_,N}, ::NTuple{N, Bool}) = A
_maybe_reshape_parent{N}(A::AbstractArray, ::NTuple{N, Bool}) = reshape(A, Val{N}) # TODO: DEPRECATE FOR #14770
"""
    view(A, inds...)

Like [`getindex`](:func:`getindex`), but returns a view into the parent array `A` with the
given indices instead of making a copy.  Calling [`getindex`](:func:`getindex`) or
[`setindex!`](:func:`setindex!`) on the returned [`SubArray`](:obj:`SubArray`) computes the
indices to the parent array on the fly without checking bounds.
"""
function view(A::AbstractArray, I::ViewIndex...)
    @_inline_meta
    @boundscheck checkbounds(A, I...)
    unsafe_view(_maybe_reshape_parent(A, index_ndims(I...)), I...)
end
# But we can simply flatten scalar `CartesianIndex`s first to make life easier
view(A::AbstractArray, I::Union{ViewIndex, AbstractCartesianIndex}...) = view(A, IteratorsMD.flatten(I)...)

function unsafe_view(A::AbstractArray, I::ViewIndex...)
    @_inline_meta
    J = to_indexes(I...)
    SubArray(A, J, map(unsafe_length, index_shape(A, J...)))
end
# When we take the view of a view, it's often possible to "reindex" the parent
# view's indices such that we can "pop" the parent view and keep just one layer
# of indirection. But we can't always do this because arrays of `CartesianIndex`
# might span multiple parent indices, making the reindex calculation very hard.
# So we use _maybe_reindex to figure out if there are any arrays of
# `CartesianIndex`, and if so, we punt and keep two layers of indirection.
unsafe_view(V::SubArray, I::ViewIndex...) = (@_inline_meta; _maybe_reindex(V, to_indexes(I...)))
_maybe_reindex(V, I) = (@_inline_meta; _maybe_reindex(V, I, I))
_maybe_reindex{C<:AbstractCartesianIndex}(V, I, ::Tuple{AbstractArray{C}, Vararg{Any}}) =
    (@_inline_meta; SubArray(V, I, map(unsafe_length, index_shape(V, I...))))
# But allow arrays of CartesianIndex{1}; they behave just like arrays of Ints
_maybe_reindex{C<:AbstractCartesianIndex{1}}(V, I, A::Tuple{AbstractArray{C}, Vararg{Any}}) =
    (@_inline_meta; _maybe_reindex(V, I, tail(A)))
_maybe_reindex(V, I, A::Tuple{Any, Vararg{Any}}) = (@_inline_meta; _maybe_reindex(V, I, tail(A)))
function _maybe_reindex(V, I, ::Tuple{})
    @_inline_meta
    idxs = reindex(V, V.indexes, to_indexes(I...))
    SubArray(V.parent, idxs, map(unsafe_length, (index_shape(V.parent, idxs...))))
end

## Re-indexing is the heart of a view, transforming A[i, j][x, y] to A[i[x], j[y]]
#
# Recursively look through the heads of the parent- and sub-indexes, considering
# the following cases:
# * Parent index is array  -> re-index that with one or more sub-indexes (one per dimension)
# * Parent index is Colon  -> just use the sub-index as provided
# * Parent index is scalar -> that dimension was dropped, so skip the sub-index and use the index as is

typealias AbstractZeroDimArray{T} AbstractArray{T, 0}

reindex(V, ::Tuple{}, ::Tuple{}) = ()

# Skip dropped scalars, so simply peel them off the parent indices and continue
reindex(V, idxs::Tuple{ScalarIndex, Vararg{Any}}, subidxs::Tuple{Vararg{Any}}) =
    (@_propagate_inbounds_meta; (idxs[1], reindex(V, tail(idxs), subidxs)...))

# Colons simply pass their subindexes straight through
reindex(V, idxs::Tuple{Colon, Vararg{Any}}, subidxs::Tuple{Any, Vararg{Any}}) =
    (@_propagate_inbounds_meta; (subidxs[1], reindex(V, tail(idxs), tail(subidxs))...))

# Re-index into parent vectors with one subindex
reindex(V, idxs::Tuple{AbstractVector, Vararg{Any}}, subidxs::Tuple{Any, Vararg{Any}}) =
    (@_propagate_inbounds_meta; (idxs[1][subidxs[1]], reindex(V, tail(idxs), tail(subidxs))...))

# Parent matrices are re-indexed with two sub-indices
reindex(V, idxs::Tuple{AbstractMatrix, Vararg{Any}}, subidxs::Tuple{Any, Any, Vararg{Any}}) =
    (@_propagate_inbounds_meta; (idxs[1][subidxs[1], subidxs[2]], reindex(V, tail(idxs), tail(tail(subidxs)))...))

# In general, we index N-dimensional parent arrays with N indices
@generated function reindex{T,N}(V, idxs::Tuple{AbstractArray{T,N}, Vararg{Any}}, subidxs::Tuple{Vararg{Any}})
    if length(subidxs.parameters) >= N
        subs = [:(subidxs[$d]) for d in 1:N]
        tail = [:(subidxs[$d]) for d in N+1:length(subidxs.parameters)]
        :(@_propagate_inbounds_meta; (idxs[1][$(subs...)], reindex(V, tail(idxs), ($(tail...),))...))
    else
        :(throw(ArgumentError("cannot re-index $(ndims(V)) dimensional SubArray with fewer than $(ndims(V)) indices\nThis should not occur; please submit a bug report.")))
    end
end

# In general, we simply re-index the parent indices by the provided ones
typealias SlowSubArray{T,N,P,I} SubArray{T,N,P,I,false}
function getindex{T,N}(V::SlowSubArray{T,N}, I::Vararg{Real,N})
    @_inline_meta
    @boundscheck checkbounds(V, I...)
    @inbounds r = V.parent[reindex(V, V.indexes, to_indexes(I...))...]
    r
end

typealias FastSubArray{T,N,P,I} SubArray{T,N,P,I,true}
function getindex(V::FastSubArray, i::Real)
    @_inline_meta
    @boundscheck checkbounds(V, i)
    @inbounds r = V.parent[V.offset1 + V.stride1*to_index(i)]
    r
end
# We can avoid a multiplication if the first parent index is a Colon or UnitRange
typealias FastContiguousSubArray{T,N,P,I<:Tuple{Union{Colon, UnitRange}, Vararg{Any}}} SubArray{T,N,P,I,true}
function getindex(V::FastContiguousSubArray, i::Real)
    @_inline_meta
    @boundscheck checkbounds(V, i)
    @inbounds r = V.parent[V.offset1 + to_index(i)]
    r
end

function setindex!{T,N}(V::SlowSubArray{T,N}, x, I::Vararg{Real,N})
    @_inline_meta
    @boundscheck checkbounds(V, I...)
    @inbounds V.parent[reindex(V, V.indexes, to_indexes(I...))...] = x
    V
end
function setindex!(V::FastSubArray, x, i::Real)
    @_inline_meta
    @boundscheck checkbounds(V, i)
    @inbounds V.parent[V.offset1 + V.stride1*to_index(i)] = x
    V
end
function setindex!(V::FastContiguousSubArray, x, i::Real)
    @_inline_meta
    @boundscheck checkbounds(V, i)
    @inbounds V.parent[V.offset1 + to_index(i)] = x
    V
end

linearindexing{T<:FastSubArray}(::Type{T}) = LinearFast()
linearindexing{T<:SubArray}(::Type{T}) = LinearSlow()

getindex(::Colon, i) = to_index(i)
unsafe_getindex(::Colon, i) = to_index(i)

step(::Colon) = 1
isempty(::Colon) = false
in(::Integer, ::Colon) = true

# Strides are the distance between adjacent elements in a given dimension,
# so they are well-defined even for non-linear memory layouts
strides{T,N,P,I}(V::SubArray{T,N,P,I}) = substrides(V.parent, V.indexes)

substrides(parent, I::Tuple) = substrides(1, parent, 1, I)
substrides(s, parent, dim, ::Tuple{}) = ()
substrides(s, parent, dim, I::Tuple{Real, Vararg{Any}}) = (substrides(s*size(parent, dim), parent, dim+1, tail(I))...)
substrides(s, parent, dim, I::Tuple{AbstractCartesianIndex, Vararg{Any}}) = substrides(s, parent, dim, (I[1].I..., tail(I)...))
substrides(s, parent, dim, I::Tuple{Colon, Vararg{Any}}) = (s, substrides(s*size(parent, dim), parent, dim+1, tail(I))...)
substrides(s, parent, dim, I::Tuple{Range, Vararg{Any}}) = (s*step(I[1]), substrides(s*size(parent, dim), parent, dim+1, tail(I))...)
substrides(s, parent, dim, I::Tuple{Any, Vararg{Any}}) = throw(ArgumentError("strides is invalid for SubArrays with indices of type $(typeof(I[1]))"))

stride(V::SubArray, d::Integer) = d <= ndims(V) ? strides(V)[d] : strides(V)[end] * size(V)[end]

compute_stride1{N}(parent::AbstractArray, I::NTuple{N}) =
    compute_stride1(1, fill_to_length(indices(parent), OneTo(1), Val{N}), I)
compute_stride1(s, inds, I::Tuple{}) = s
compute_stride1(s, inds, I::Tuple{Real, Vararg{Any}}) =
    (@_inline_meta; compute_stride1(s*unsafe_length(inds[1]), tail(inds), tail(I)))
compute_stride1(s, inds, I::Tuple{Range, Vararg{Any}}) = s*step(I[1])
compute_stride1(s, inds, I::Tuple{Colon, Vararg{Any}}) = s
function compute_stride1{N}(s, inds, I::Tuple{AbstractCartesianIndex{N}, Vararg{Any}})
    @_inline_meta
    h, t = IteratorsMD.split(inds, Val{N})
    compute_stride1(s*prod(map(unsafe_length, h)), t, tail(I))
end
compute_stride1(s, inds, I::Tuple{Any, Vararg{Any}}) = throw(ArgumentError("invalid strided index type $(typeof(I[1]))"))

iscontiguous(A::SubArray) = iscontiguous(typeof(A))
iscontiguous{S<:SubArray}(::Type{S}) = false
iscontiguous{F<:FastContiguousSubArray}(::Type{F}) = true

first_index(V::FastSubArray) = V.offset1 + V.stride1 # cached for fast linear SubArrays
function first_index(V::SubArray)
    P, I = parent(V), V.indexes
    s1 = compute_stride1(P, I)
    s1 + compute_offset1(P, s1, I)
end

# Computing the first index simply steps through the indices, accumulating the
# sum of index each multiplied by the parent's stride.
# The running sum is `f`; the cumulative stride product is `s`.
# If the result is one-dimensional and it's a Colon, then linear
# indexing uses the indices along the given dimension. Otherwise
# linear indexing always starts with 1.
compute_offset1(parent, stride1::Integer, I::Tuple) = (@_inline_meta; compute_offset1(parent, stride1, find_extended_dims(I)..., I))
compute_offset1(parent, stride1::Integer, dims::Tuple{Int}, inds::Tuple{Colon}, I::Tuple) = compute_linindex(parent, I) - stride1*first(indices(parent, dims[1]))  # index-preserving case
compute_offset1(parent, stride1::Integer, dims, inds, I::Tuple) = compute_linindex(parent, I) - stride1  # linear indexing starts with 1

function compute_linindex{N}(parent, I::NTuple{N})
    IP = fill_to_length(indices(parent), OneTo(1), Val{N})
    compute_linindex(1, 1, IP, I)
end
function compute_linindex(f, s, IP::Tuple, I::Tuple{Real, Vararg{Any}})
    @_inline_meta
    Δi = I[1]-first(IP[1])
    compute_linindex(f + Δi*s, s*unsafe_length(IP[1]), tail(IP), tail(I))
end
# Just splat out the cartesian indices and continue
compute_linindex(f, s, IP::Tuple, I::Tuple{AbstractCartesianIndex, Vararg{Any}}) =
    (@_inline_meta; compute_linindex(f, s, IP, (I[1].I..., tail(I)...)))
compute_linindex(f, s, IP::Tuple, I::Tuple{Colon, Vararg{Any}}) =
    (@_inline_meta; compute_linindex(f, s*unsafe_length(IP[1]), tail(IP), tail(I)))
function compute_linindex(f, s, IP::Tuple, I::Tuple{Any, Vararg{Any}})
    @_inline_meta
    Δi = first(I[1])-first(IP[1])
    compute_linindex(f + Δi*s, s*unsafe_length(IP[1]), tail(IP), tail(I))
end
compute_linindex(f, s, IP::Tuple, I::Tuple{}) = f

find_extended_dims(I) = (@_inline_meta; _find_extended_dims((), (), 1, I...))
_find_extended_dims(dims, inds, dim) = dims, inds
_find_extended_dims(dims, inds, dim, ::Real, I...) = _find_extended_dims(dims, inds, dim+1, I...)
_find_extended_dims(dims, inds, dim, i1::AbstractCartesianIndex, I...) = _find_extended_dims(dims, inds, dim, i1.I..., I...)
_find_extended_dims(dims, inds, dim, i1, I...) = _find_extended_dims((dims..., dim), (inds..., i1), dim+1, I...)

unsafe_convert{T,N,P,I<:Tuple{Vararg{RangeIndex}}}(::Type{Ptr{T}}, V::SubArray{T,N,P,I}) =
    unsafe_convert(Ptr{T}, V.parent) + (first_index(V)-1)*sizeof(T)

pointer(V::FastSubArray, i::Int) = pointer(V.parent, V.offset1 + V.stride1*i)
pointer(V::FastContiguousSubArray, i::Int) = pointer(V.parent, V.offset1 + i)
pointer(V::SubArray, i::Int) = _pointer(V, i)
_pointer{T}(V::SubArray{T,1}, i::Int) = pointer(V, (i,))
_pointer(V::SubArray, i::Int) = pointer(V, ind2sub(indices(V), i))

function pointer{T,N,P<:Array,I<:Tuple{Vararg{RangeIndex}}}(V::SubArray{T,N,P,I}, is::Tuple{Vararg{Int}})
    index = first_index(V)
    strds = strides(V)
    for d = 1:length(is)
        index += (is[d]-1)*strds[d]
    end
    return pointer(V.parent, index)
end

# indices of the parent are preserved for ::Colon indices, otherwise
# they are taken from the range/vector
# Since bounds-checking is performance-critical and uses
# indices, it's worth optimizing these implementations thoroughly
indices(S::SubArray) = (@_inline_meta; _indices_sub(S, indices(S.parent), S.indexes...))
_indices_sub(S::SubArray, pinds) = ()
function _indices_sub(S::SubArray, pinds, ::Real, I...)
    @_inline_meta
    _indices_sub(S, tail(pinds), I...)
end
function _indices_sub(S::SubArray, pinds, ::Colon, I...)
    @_inline_meta
    (pinds[1], _indices_sub(S, tail(pinds), I...)...)
end
function _indices_sub(S::SubArray, pinds, i1::AbstractArray, I...)
    @_inline_meta
    (unsafe_indices(i1)..., _indices_sub(S, tail(pinds), I...)...)
end
function _indices_sub{N}(S::SubArray, pinds, ::AbstractCartesianIndex{N}, I...)
    @_inline_meta
    _indices_sub(S, IteratorsMD.split(pinds, Val{N})[2], I...)
end
# _indices_sub for arrays of CartesianIndex is defined in multidimensional.jl

## Compatability
# deprecate?
function parentdims(s::SubArray)
    nd = ndims(s)
    dimindex = Array{Int}(nd)
    sp = strides(s.parent)
    sv = strides(s)
    j = 1
    for i = 1:ndims(s.parent)
        r = s.indexes[i]
        if j <= nd && (isa(r,Union{Colon,Range}) ? sp[i]*step(r) : sp[i]) == sv[j]
            dimindex[j] = i
            j += 1
        end
    end
    dimindex
end

"""
    replace_ref_end!(ex)

Recursively replace occurrences of the symbol :end in a "ref" expression (i.e. A[...]) `ex`
with the appropriate function calls (`endof`, `size` or `trailingsize`). Replacement uses
the closest enclosing ref, so

    A[B[end]]

should transform to

    A[B[endof(B)]]

"""
function replace_ref_end!(ex,withex=nothing)
    if isa(ex,Symbol) && ex == :end
        withex === nothing && error("Invalid use of end")
        return withex
    elseif isa(ex,Expr)
        if ex.head == :ref
            S = ex.args[1] = replace_ref_end!(ex.args[1],withex)
            # new :ref, so redefine withex
            nargs = length(ex.args)-1
            if nargs == 0
                return ex
            elseif nargs == 1
                # replace with endof(S)
                ex.args[2] = replace_ref_end!(ex.args[2],:(Base.endof($S)))
            else
                n = 1
                J = endof(ex.args)
                for j = 2:J-1
                    exj = ex.args[j] = replace_ref_end!(ex.args[j],:(Base.size($S,$n)))
                    if isa(exj,Expr) && exj.head == :...
                        # splatted object
                        exjs = exj.args[1]
                        n = :($n + length($exjs))
                    elseif isa(n, Expr)
                        # previous expression splatted
                        n = :($n + 1)
                    else
                        # an integer
                        n += 1
                    end
                end
                ex.args[J] = replace_ref_end!(ex.args[J],:(Base.trailingsize($S,$n)))
            end
        else
            # recursive search
            for i = eachindex(ex.args)
                ex.args[i] = replace_ref_end!(ex.args[i],withex)
            end
        end
    end
    ex
end

"""
    @view A[inds...]

Creates a `SubArray` from an indexing expression. This can only be applied directly to a
reference expression (e.g. `@view A[1,2:end]`), and should *not* be used as the target of
an assignment (e.g. `@view(A[1,2:end]) = ...`).
"""
macro view(ex)
    if isa(ex, Expr) && ex.head == :ref
        ex = replace_ref_end!(ex)
        Expr(:&&, true, esc(Expr(:call,:(Base.view),ex.args...)))
    else
        throw(ArgumentError("Invalid use of @view macro: argument must be a reference expression A[...]."))
    end
end


function Base.size( ba::BOSSArray{T,N}) where {T,N}
    ([typemax(Int64) for i in 1:N]...)
end

function Base.eltype( ba::BOSSArray{T,N} ) where {T,N}
    T
end

function Base.ndims( ba::BOSSArray{T,N} ) where {T,N}
    N
end

function Base.reshape( ba::BOSSArray{T,N}, newShape ) where {T,N}
    warn("BOSSArray do not support reshaping")
end

"""
transform int to a UnitRange
"""
function idx2unitrange( idx::Union{UnitRange, Int} )
    if isa(idx, Int)
        return idx:idx
    else
        return idx
    end
end

function Base.getindex( ba::BOSSArray{T,3}, idxes::Union{UnitRange, Int} ... ) where T
    idxes = map(idx2unitrange, idxes)
    # construct the url
    # note that the start should -1 to match the coordinate system of numpy
    urlPath = "$(ba.urlPrefix)/cutout/$(ba.collectionName)"*
                    "/$(ba.experimentName)"*
                    "/$(ba.channelName)/$(ba.resolutionLevel)"*
                    "/$(idxes[1].start-1):$(idxes[1].stop)"*
                    "/$(idxes[2].start-1):$(idxes[2].stop)"*
                    "/$(idxes[3].start-1):$(idxes[3].stop)"
    resp = Requests.get(URI(urlPath); headers = ba.headers)
    data = Blosc.decompress(T, resp.data)
    data = reshape(data, map(length, idxes))
    return data
end

function Base.getindex( ba::BOSSArray{T,4}, idxes::Union{UnitRange, Int} ... ) where T
    idxes = map(idx2unitrange, idxes)
    # construct the url
    # note that the start should -1 to match the coordinate system of numpy
    urlPath = "$(ba.urlPrefix)/cutout/$(ba.collectionName)"*
                    "/$(ba.experimentName)"*
                    "/$(ba.channelName)/$(ba.resolutionLevel)"*
                    "/$(idxes[1].start-1):$(idxes[1].stop)"*
                    "/$(idxes[2].start-1):$(idxes[2].stop)"*
                    "/$(idxes[3].start-1):$(idxes[3].stop)"*
                    "/$(idxes[4].start-1):$(idxes[4].stop)"
    resp = Requests.get(URI(urlPath); headers = ba.headers)
    data = Blosc.decompress(T, resp.data)
    data = reshape(data, map(length, idxes))
    return data
end

function Base.setindex!(ba::BOSSArray{T,3}, buffer::AbstractArray,
                         idxes::Union{UnitRange, Int}...) where T
    @assert ndims(buffer)==3
    buffer = convert(Array{T,3}, buffer)
    idxes = map(idx2unitrange, idxes)
    # construct the url
    # note that the start should -1 to match the coordinate system of numpy
    urlPath = "$(ba.urlPrefix)/cutout/$(ba.collectionName)"*
                    "/$(ba.experimentName)"*
                    "/$(ba.channelName)/$(ba.resolutionLevel)"*
                    "/$(idxes[1].start-1):$(idxes[1].stop)"*
                    "/$(idxes[2].start-1):$(idxes[2].stop)"*
                    "/$(idxes[3].start-1):$(idxes[3].stop)"
    data = Blosc.compress( buffer )
    local resp
    for i in 1:3
        resp = Requests.post(URI(urlPath); data = data, headers = ba.headers)
        if statuscode(resp) == 201
            break
        else
            sleep(3)
        end
    end

    if statuscode(resp) != 201
        error("writting to boss have error: $resp")
    end
    return resp
end


export
    AbstractCBSModel,
    DefaultCBSModel,
    PrioritizedDFSPlanner,
    AbstractISPSModel,
    DefaultISPSModel,
    PrioritizedPlannerISPSModel,
    AbstractPathFinderModel,
    AStarPathFinderModel,
    PrioritizedAStarModel,
    DFS_PathFinder,
    PC_TAPF_Solver

abstract type AbstractCBSModel end
struct DefaultCBSModel <: AbstractCBSModel end
@with_kw struct PrioritizedDFSPlanner <: AbstractCBSModel
    max_iters::Int = 3000
end
abstract type AbstractISPSModel end
@with_kw struct DefaultISPSModel <: AbstractISPSModel
    n_repair_iters::Int = 2
end
abstract type AbstractPathFinderModel end
@with_kw struct AStarPathFinderModel <: AbstractPathFinderModel
    search_history::Vector{State}   = State[]
    replan::Bool                    = false # Flag for replanning with empty conflict table after timeout
end
@with_kw struct PrioritizedAStarModel <: AbstractPathFinderModel
    search_history::Vector{State}   = State[]
    replan::Bool                    = false # Flag for replanning with empty conflict table after timeout
end
struct DFS_PathFinder <: AbstractPathFinderModel end

@with_kw mutable struct PC_TAPF_Solver{M,C,I,A,T}
    # TODO parameterize by MILP Solver, CBS solver, ISPS solver, A_star solver
    nbs_model                   ::M = SparseAdjacencyMILP()
    cbs_model                   ::C = DefaultCBSModel()
    isps_model                  ::I = DefaultISPSModel()
    astar_model                 ::A = AStarPathFinderModel()
    LIMIT_assignment_iterations ::Int = 50
    LIMIT_CBS_iterations        ::Int = 100
    LIMIT_A_star_iterations     ::Int = 5000

    num_assignment_iterations   ::Int = 0
    num_CBS_iterations          ::Int = 0
    num_A_star_iterations       ::Int = 0
    best_cost                   ::T   = get_infeasible_cost(astar_model)

    total_assignment_iterations ::Int = 0
    total_CBS_iterations        ::Int = 0
    total_A_star_iterations     ::Int = 0

    max_CBS_iterations          ::Int = 0
    max_A_star_iterations       ::Int = 0

    start_time                  ::Float64 = time()
    time_limit                  ::Float64 = 200.0
    nbs_time_limit              ::Float64 = 190.0

    verbosity                   ::Int = 0
    l1_verbosity                ::Int = 0
    l2_verbosity                ::Int = 0
    l3_verbosity                ::Int = 0
    l4_verbosity                ::Int = 0
    DEBUG                       ::Bool = false
end

function TOML.parse(solver::S) where {S<:PC_TAPF_Solver}
    toml_dict = Dict()

    toml_dict["LIMIT_assignment_iterations"] = solver.LIMIT_assignment_iterations
    toml_dict["LIMIT_CBS_iterations"] = solver.LIMIT_CBS_iterations
    toml_dict["LIMIT_A_star_iterations"] = solver.LIMIT_A_star_iterations
    toml_dict["num_assignment_iterations"] = solver.num_assignment_iterations
    toml_dict["num_CBS_iterations"] = solver.num_CBS_iterations
    toml_dict["num_A_star_iterations"] = solver.num_A_star_iterations
    if any(i->i==Inf, solver.best_cost)
        toml_dict["best_cost"] = map(i->-1,1:length(solver.best_cost))
    else
        toml_dict["best_cost"] = collect(solver.best_cost)
    end
    toml_dict["total_assignment_iterations"] = solver.total_assignment_iterations
    toml_dict["total_CBS_iterations"] = solver.total_CBS_iterations
    toml_dict["total_A_star_iterations"] = solver.total_A_star_iterations
    toml_dict["max_CBS_iterations"] = solver.max_CBS_iterations
    toml_dict["max_A_star_iterations"] = solver.max_A_star_iterations
    toml_dict["time_limit"] = solver.time_limit
    toml_dict["verbosity"] = solver.verbosity
    toml_dict["l1_verbosity"] = solver.l1_verbosity
    toml_dict["l2_verbosity"] = solver.l2_verbosity
    toml_dict["l3_verbosity"] = solver.l3_verbosity
    toml_dict["l4_verbosity"] = solver.l4_verbosity
    toml_dict["DEBUG"]        = solver.DEBUG

    toml_dict
end
function read_solver(toml_dict::Dict)
    solver = PC_TAPF_Solver(
        LIMIT_assignment_iterations = toml_dict["LIMIT_assignment_iterations"],
        LIMIT_CBS_iterations = toml_dict["LIMIT_CBS_iterations"],
        LIMIT_A_star_iterations = toml_dict["LIMIT_A_star_iterations"],

        num_assignment_iterations = toml_dict["num_assignment_iterations"],
        num_CBS_iterations = toml_dict["num_CBS_iterations"],
        num_A_star_iterations = toml_dict["num_A_star_iterations"],
        best_cost = Tuple(toml_dict["best_cost"]),

        total_assignment_iterations = toml_dict["total_assignment_iterations"],
        total_CBS_iterations = toml_dict["total_CBS_iterations"],
        total_A_star_iterations = toml_dict["total_A_star_iterations"],

        max_CBS_iterations = toml_dict["max_CBS_iterations"],
        max_A_star_iterations = toml_dict["max_A_star_iterations"],

        time_limit = toml_dict["time_limit"],

        verbosity = toml_dict["verbosity"],
        l1_verbosity = toml_dict["l1_verbosity"],
        l2_verbosity = toml_dict["l2_verbosity"],
        l3_verbosity = toml_dict["l3_verbosity"],
        l4_verbosity = toml_dict["l4_verbosity"],
        DEBUG = toml_dict["DEBUG"]
    )
end
function read_solver(io)
    read_solver(TOML.parsefile(io))
end

function CRCBS.check_termination_criteria(solver::S,env::E,cost_so_far,s) where {S<:PC_TAPF_Solver,E<:AbstractLowLevelEnv}
    # solver.num_A_star_iterations += 1
    if solver.num_A_star_iterations > solver.LIMIT_A_star_iterations
        # throw(SolverAstarMaxOutException(string("# MAX OUT: A* limit of ",solver.LIMIT_A_star_iterations," exceeded.")))
        return true
    end
    return false
end


uri_pages = ["index", "home", "search", "main", "post", "homepage", "category", "register", "login", "faq", "about", "terms","privacy", "author"]
uri_paths = ["app", "main", "wp-content", "search", "category", "tag", "categories", "tags", "blog", "posts", "list", "explore"]
uri_extensions = [".html", ".html", ".html", ".htm", ".htm", ".php", ".php", ".jsp", ".asp"]

pixels()= rand(1:1024)

user_name()= bothify(executor(data["en"]["faker"]["internet"]["user_name_formats"]))

domain_suffix()= executor(data["en"]["faker"]["internet"]["domain_suffix"])
safe_email()= "$(user_name())@example.$(domain_suffix())"

free_email_domain() = executor(data["en"]["faker"]["internet"]["free_email"])
free_email()= "$(user_name())@$(free_email_domain())"

domain_word()=
   ( company_elements = split(company(), [',', ' ','-']);
    sal = "";
    for i in company_elements
               if length(i)>0
                   sal*="$(i[1])"
               end
          end;

    lowercase(sal) )

domain_name()="$(domain_word()).$(domain_suffix())"

company_email()= "$(user_name())@$(domain_name())"

email()= executor(data["en"]["faker"]["internet"]["email_formats"])

url()= executor(data["en"]["faker"]["internet"]["url_formats"])

ipv4()= "$(rand( 172:255)).$(rand( 0:255)).$(rand(0:255)).$(rand(0:255))"

ipv6()= (res = "";
    for i = 1:8
        res*= "$(string(rand( 4100:65535),base=16))"
        if i < 8
            res*= ":"
        end
    end;res)

mac_address()=
    (res = "";
    for i = 1:6
        res*= "$(Base.hex(unsigned(rand( 0x00:0xff)),2,false))"
        i < 6 && (res*= ":")
    end;res)

uri_page()= executor(uri_pages)

uri_path(deep="None")=
    (if deep =="None"
        deep = rand(1:3)
    end; sa="";
        for i =0:deep
           sa*= uri_paths[rand( 1:length(uri_paths))]
            i < deep && (sa*= "/")
        end; sa )

uri_extension()= executor(uri_extensions)
uri()= executor(data["en"]["faker"]["internet"]["uri_formats"])
image_url()= executor(data["en"]["faker"]["internet"]["image_placeholder_services"])


#!/usr/bin/env julia

import Celeste.ParallelRun: one_node_joint_infer, infer_init, BoundingBox, find_neighbors
import Celeste.SDSSIO: RunCamcolField, load_field_images, PlainFITSStrategy

const datadir = joinpath(dirname(@__FILE__), "..", "..", "test", "data")
wd = pwd()
cd(datadir)
run(`make`)
cd(wd)

"""
This benchmark operates on a box of the sky that contains
seven light sources. During the optimization, some pixel is visited
254,771 times (``pixel visits'').
"""
function benchmark_seven_light_sources()
    # very small patch of sky that turns out to have 4 sources.
    # We checked that this patch is in the given field.
    box = BoundingBox(164.39, 164.41, 39.11, 39.13)
    rcfs = [RunCamcolField(3900, 6, 269),]

    catalog, target_sources, neighbor_map, images =
                        infer_init(rcfs, PlainFITSStrategy(datadir); box=box)
    ctni = (catalog, target_sources, neighbor_map, images)

    # Warm up---this compiles the code
    ctni2 = (catalog, target_sources[1:1], neighbor_map[1:1], images[1:1])
    one_node_joint_infer(ctni2...)

    # clear allocations in case julia is running with --track-allocations=user
    Profile.clear_malloc_data()

    if isempty(ARGS)
        @time one_node_joint_infer(ctni...)
    elseif ARGS[1] == "--profile"
        Profile.init(delay=0.01)
        @profile one_node_joint_infer(ctni...)
        Profile.print(format=:flat, sortedby=:count)
    end
end

benchmark_seven_light_sources()


abstract type RosenbrockMutableCache <: OrdinaryDiffEqMutableCache end
################################################################################

# Shampine's Low-order Rosenbrocks

@cache mutable struct Rosenbrock23Cache{uType,rateType,uNoUnitsType,JType,WType,TabType,TFType,UFType,F,JCType,GCType} <: RosenbrockMutableCache
  u::uType
  uprev::uType
  k₁::rateType
  k₂::rateType
  k₃::rateType
  du1::rateType
  du2::rateType
  f₁::rateType
  fsalfirst::rateType
  fsallast::rateType
  dT::rateType
  J::JType
  W::WType
  tmp::rateType
  atmp::uNoUnitsType
  weight::uNoUnitsType
  tab::TabType
  tf::TFType
  uf::UFType
  linsolve_tmp::rateType
  linsolve::F
  jac_config::JCType
  grad_config::GCType
end

@cache mutable struct Rosenbrock32Cache{uType,rateType,uNoUnitsType,JType,WType,TabType,TFType,UFType,F,JCType,GCType} <: RosenbrockMutableCache
  u::uType
  uprev::uType
  k₁::rateType
  k₂::rateType
  k₃::rateType
  du1::rateType
  du2::rateType
  f₁::rateType
  fsalfirst::rateType
  fsallast::rateType
  dT::rateType
  J::JType
  W::WType
  tmp::rateType
  atmp::uNoUnitsType
  tab::TabType
  tf::TFType
  uf::UFType
  linsolve_tmp::rateType
  linsolve::F
  jac_config::JCType
  grad_config::GCType
end

function alg_cache(alg::Rosenbrock23,u,rate_prototype,uEltypeNoUnits,uBottomEltypeNoUnits,tTypeNoUnits,uprev,uprev2,f,t,dt,reltol,p,calck,::Val{true})
  k₁ = zero(rate_prototype)
  k₂ = zero(rate_prototype)
  k₃ = zero(rate_prototype)
  du1 = zero(rate_prototype)
  du2 = zero(rate_prototype)
  # f₀ = zero(u) fsalfirst
  f₁ = zero(rate_prototype)
  fsalfirst = zero(rate_prototype)
  fsallast = zero(rate_prototype)
  dT = zero(rate_prototype)
  J,W = build_J_W(alg,u,uprev,p,t,dt,f,uEltypeNoUnits,Val(true))
  tmp = zero(rate_prototype)
  atmp = similar(u, uEltypeNoUnits)
  weight = similar(u, uEltypeNoUnits)
  tab = Rosenbrock23Tableau(constvalue(uBottomEltypeNoUnits))
  tf = TimeGradientWrapper(f,uprev,p)
  uf = UJacobianWrapper(f,t,p)
  linsolve_tmp = zero(rate_prototype)
  linsolve = alg.linsolve(Val{:init},uf,u)

  grad_config = build_grad_config(alg,f,tf,du1,t)
  jac_config = build_jac_config(alg,f,uf,du1,uprev,u,tmp,du2,Val(false))

  Rosenbrock23Cache(u,uprev,k₁,k₂,k₃,du1,du2,f₁,
                    fsalfirst,fsallast,dT,J,W,tmp,atmp,weight,tab,tf,uf,linsolve_tmp,
                    linsolve,jac_config,grad_config)
end

function alg_cache(alg::Rosenbrock32,u,rate_prototype,uEltypeNoUnits,uBottomEltypeNoUnits,tTypeNoUnits,uprev,uprev2,f,t,dt,reltol,p,calck,::Val{true})
  k₁ = zero(rate_prototype)
  k₂ = zero(rate_prototype)
  k₃ = zero(rate_prototype)
  du1 = zero(rate_prototype)
  du2 = zero(rate_prototype)
  # f₀ = zero(u) fsalfirst
  f₁ = zero(rate_prototype)
  fsalfirst = zero(rate_prototype)
  fsallast = zero(rate_prototype)
  dT = zero(rate_prototype)
  J,W = build_J_W(alg,u,uprev,p,t,dt,f,uEltypeNoUnits,Val(true))
  tmp = zero(rate_prototype)
  atmp = similar(u, uEltypeNoUnits)
  tab = Rosenbrock32Tableau(constvalue(uBottomEltypeNoUnits))

  tf = TimeGradientWrapper(f,uprev,p)
  uf = UJacobianWrapper(f,t,p)
  linsolve_tmp = zero(rate_prototype)
  linsolve = alg.linsolve(Val{:init},uf,u)
  grad_config = build_grad_config(alg,f,tf,du1,t)
  jac_config = build_jac_config(alg,f,uf,du1,uprev,u,tmp,du2)
  Rosenbrock32Cache(u,uprev,k₁,k₂,k₃,du1,du2,f₁,fsalfirst,fsallast,dT,J,W,tmp,atmp,tab,tf,uf,linsolve_tmp,linsolve,jac_config,grad_config)
end

struct Rosenbrock23ConstantCache{T,TF,UF,JType,WType,F} <: OrdinaryDiffEqConstantCache
  c₃₂::T
  d::T
  tf::TF
  uf::UF
  J::JType
  W::WType
  linsolve::F
end

function Rosenbrock23ConstantCache(T::Type,tf,uf,J,W,linsolve)
  tab = Rosenbrock23Tableau(T)
  Rosenbrock23ConstantCache(tab.c₃₂,tab.d,tf,uf,J,W,linsolve)
end

function alg_cache(alg::Rosenbrock23,u,rate_prototype,uEltypeNoUnits,uBottomEltypeNoUnits,tTypeNoUnits,uprev,uprev2,f,t,dt,reltol,p,calck,::Val{false})
  tf = TimeDerivativeWrapper(f,u,p)
  uf = UDerivativeWrapper(f,t,p)
  J,W = build_J_W(alg,u,uprev,p,t,dt,f,uEltypeNoUnits,Val(false))
  linsolve = alg.linsolve(Val{:init},uf,u)
  Rosenbrock23ConstantCache(constvalue(uBottomEltypeNoUnits),tf,uf,J,W,linsolve)
end

struct Rosenbrock32ConstantCache{T,TF,UF,JType,WType,F} <: OrdinaryDiffEqConstantCache
  c₃₂::T
  d::T
  tf::TF
  uf::UF
  J::JType
  W::WType
  linsolve::F
end

function Rosenbrock32ConstantCache(T::Type,tf,uf,J,W,linsolve)
  tab = Rosenbrock32Tableau(T)
  Rosenbrock32ConstantCache(tab.c₃₂,tab.d,tf,uf,J,W,linsolve)
end

function alg_cache(alg::Rosenbrock32,u,rate_prototype,uEltypeNoUnits,uBottomEltypeNoUnits,tTypeNoUnits,uprev,uprev2,f,t,dt,reltol,p,calck,::Val{false})
  tf = TimeDerivativeWrapper(f,u,p)
  uf = UDerivativeWrapper(f,t,p)
  J,W = build_J_W(alg,u,uprev,p,t,dt,f,uEltypeNoUnits,Val(false))
  linsolve = alg.linsolve(Val{:init},uf,u)
  Rosenbrock32ConstantCache(constvalue(uBottomEltypeNoUnits),tf,uf,J,W,linsolve)
end

################################################################################

### 3rd order specialized Rosenbrocks

struct Rosenbrock33ConstantCache{TF,UF,Tab,JType,WType,F} <: OrdinaryDiffEqConstantCache
  tf::TF
  uf::UF
  tab::Tab
  J::JType
  W::WType
  linsolve::F
end

@cache mutable struct Rosenbrock33Cache{uType,rateType,uNoUnitsType,JType,WType,TabType,TFType,UFType,F,JCType,GCType} <: RosenbrockMutableCache
  u::uType
  uprev::uType
  du::rateType
  du1::rateType
  du2::rateType
  k1::rateType
  k2::rateType
  k3::rateType
  k4::rateType
  fsalfirst::rateType
  fsallast::rateType
  dT::rateType
  J::JType
  W::WType
  tmp::rateType
  atmp::uNoUnitsType
  tab::TabType
  tf::TFType
  uf::UFType
  linsolve_tmp::rateType
  linsolve::F
  jac_config::JCType
  grad_config::GCType
end

function alg_cache(alg::ROS3P,u,rate_prototype,uEltypeNoUnits,uBottomEltypeNoUnits,tTypeNoUnits,uprev,uprev2,f,t,dt,reltol,p,calck,::Val{true})
  du = zero(rate_prototype)
  du1 = zero(rate_prototype)
  du2 = zero(rate_prototype)
  k1 = zero(rate_prototype)
  k2 = zero(rate_prototype)
  k3 = zero(rate_prototype)
  k4 = zero(rate_prototype)
  fsalfirst = zero(rate_prototype)
  fsallast = zero(rate_prototype)
  dT = zero(rate_prototype)
  J,W = build_J_W(alg,u,uprev,p,t,dt,f,uEltypeNoUnits,Val(true))
  tmp = zero(rate_prototype)
  atmp = similar(u, uEltypeNoUnits)
  tab = ROS3PTableau(constvalue(uBottomEltypeNoUnits),constvalue(tTypeNoUnits))
  tf = TimeGradientWrapper(f,uprev,p)
  uf = UJacobianWrapper(f,t,p)
  linsolve_tmp = zero(rate_prototype)
  linsolve = alg.linsolve(Val{:init},uf,u)
  grad_config = build_grad_config(alg,f,tf,du1,t)
  jac_config = build_jac_config(alg,f,uf,du1,uprev,u,tmp,du2)
  Rosenbrock33Cache(u,uprev,du,du1,du2,k1,k2,k3,k4,
                    fsalfirst,fsallast,dT,J,W,tmp,atmp,tab,tf,uf,linsolve_tmp,
                    linsolve,jac_config,grad_config)
end

function alg_cache(alg::ROS3P,u,rate_prototype,uEltypeNoUnits,uBottomEltypeNoUnits,tTypeNoUnits,uprev,uprev2,f,t,dt,reltol,p,calck,::Val{false})
  tf = TimeDerivativeWrapper(f,u,p)
  uf = UDerivativeWrapper(f,t,p)
  J,W = build_J_W(alg,u,uprev,p,t,dt,f,uEltypeNoUnits,Val(false))
  linsolve = alg.linsolve(Val{:init},uf,u)
  Rosenbrock33ConstantCache(tf,uf,ROS3PTableau(constvalue(uBottomEltypeNoUnits),constvalue(tTypeNoUnits)),J,W,linsolve)
end

@cache mutable struct Rosenbrock34Cache{uType,rateType,uNoUnitsType,JType,WType,TabType,TFType,UFType,F,JCType,GCType} <: RosenbrockMutableCache
  u::uType
  uprev::uType
  du::rateType
  du1::rateType
  du2::rateType
  k1::rateType
  k2::rateType
  k3::rateType
  k4::rateType
  fsalfirst::rateType
  fsallast::rateType
  dT::rateType
  J::JType
  W::WType
  tmp::rateType
  atmp::uNoUnitsType
  tab::TabType
  tf::TFType
  uf::UFType
  linsolve_tmp::rateType
  linsolve::F
  jac_config::JCType
  grad_config::GCType
end

function alg_cache(alg::Rodas3,u,rate_prototype,uEltypeNoUnits,uBottomEltypeNoUnits,tTypeNoUnits,uprev,uprev2,f,t,dt,reltol,p,calck,::Val{true})
  du = zero(rate_prototype)
  du1 = zero(rate_prototype)
  du2 = zero(rate_prototype)
  k1 = zero(rate_prototype)
  k2 = zero(rate_prototype)
  k3 = zero(rate_prototype)
  k4 = zero(rate_prototype)
  fsalfirst = zero(rate_prototype)
  fsallast = zero(rate_prototype)
  dT = zero(rate_prototype)
  J,W = build_J_W(alg,u,uprev,p,t,dt,f,uEltypeNoUnits,Val(true))
  tmp = zero(rate_prototype)
  atmp = similar(u, uEltypeNoUnits)
  tab = Rodas3Tableau(constvalue(uBottomEltypeNoUnits),constvalue(tTypeNoUnits))

  tf = TimeGradientWrapper(f,uprev,p)
  uf = UJacobianWrapper(f,t,p)
  linsolve_tmp = zero(rate_prototype)
  linsolve = alg.linsolve(Val{:init},uf,u)
  grad_config = build_grad_config(alg,f,tf,du1,t)
  jac_config = build_jac_config(alg,f,uf,du1,uprev,u,tmp,du2)
  Rosenbrock34Cache(u,uprev,du,du1,du2,k1,k2,k3,k4,
                    fsalfirst,fsallast,dT,J,W,tmp,atmp,tab,tf,uf,linsolve_tmp,
                    linsolve,jac_config,grad_config)
end

struct Rosenbrock34ConstantCache{TF,UF,Tab,JType,WType,F} <: OrdinaryDiffEqConstantCache
  tf::TF
  uf::UF
  tab::Tab
  J::JType
  W::WType
  linsolve::F
end

function alg_cache(alg::Rodas3,u,rate_prototype,uEltypeNoUnits,uBottomEltypeNoUnits,tTypeNoUnits,uprev,uprev2,f,t,dt,reltol,p,calck,::Val{false})
  tf = TimeDerivativeWrapper(f,u,p)
  uf = UDerivativeWrapper(f,t,p)
  J,W = build_J_W(alg,u,uprev,p,t,dt,f,uEltypeNoUnits,Val(false))
  linsolve = alg.linsolve(Val{:init},uf,u)
  Rosenbrock34ConstantCache(tf,uf,Rodas3Tableau(constvalue(uBottomEltypeNoUnits),constvalue(tTypeNoUnits)),J,W,linsolve)
end

################################################################################

### ROS34PW methods

@ROS34PW(:cache)

################################################################################

### ROS4 methods

@Rosenbrock4(:cache)
jac_cache(c::Rosenbrock4Cache) = (c.J,c.W)

###############################################################################

### Rodas methods

struct Rodas4ConstantCache{TF,UF,Tab,JType,WType,F} <: OrdinaryDiffEqConstantCache
  tf::TF
  uf::UF
  tab::Tab
  J::JType
  W::WType
  linsolve::F
end

@cache mutable struct Rodas4Cache{uType,rateType,uNoUnitsType,JType,WType,TabType,TFType,UFType,F,JCType,GCType} <: RosenbrockMutableCache
  u::uType
  uprev::uType
  dense1::rateType
  dense2::rateType
  du::rateType
  du1::rateType
  du2::rateType
  k1::rateType
  k2::rateType
  k3::rateType
  k4::rateType
  k5::rateType
  k6::rateType
  fsalfirst::rateType
  fsallast::rateType
  dT::rateType
  J::JType
  W::WType
  tmp::rateType
  atmp::uNoUnitsType
  tab::TabType
  tf::TFType
  uf::UFType
  linsolve_tmp::rateType
  linsolve::F
  jac_config::JCType
  grad_config::GCType
end

function alg_cache(alg::Rodas4,u,rate_prototype,uEltypeNoUnits,uBottomEltypeNoUnits,tTypeNoUnits,uprev,uprev2,f,t,dt,reltol,p,calck,::Val{true})
  dense1 = zero(rate_prototype)
  dense2 = zero(rate_prototype)
  du = zero(rate_prototype)
  du1 = zero(rate_prototype)
  du2 = zero(rate_prototype)
  k1 = zero(rate_prototype)
  k2 = zero(rate_prototype)
  k3 = zero(rate_prototype)
  k4 = zero(rate_prototype)
  k5 = zero(rate_prototype)
  k6 = zero(rate_prototype)
  fsalfirst = zero(rate_prototype)
  fsallast = zero(rate_prototype)
  dT = zero(rate_prototype)
  J,W = build_J_W(alg,u,uprev,p,t,dt,f,uEltypeNoUnits,Val(true))
  tmp = zero(rate_prototype)
  atmp = similar(u, uEltypeNoUnits)
  tab = Rodas4Tableau(constvalue(uBottomEltypeNoUnits),constvalue(tTypeNoUnits))

  tf = TimeGradientWrapper(f,uprev,p)
  uf = UJacobianWrapper(f,t,p)
  linsolve_tmp = zero(rate_prototype)
  linsolve = alg.linsolve(Val{:init},uf,u)
  grad_config = build_grad_config(alg,f,tf,du1,t)
  jac_config = build_jac_config(alg,f,uf,du1,uprev,u,tmp,du2)
  Rodas4Cache(u,uprev,dense1,dense2,du,du1,du2,k1,k2,k3,k4,
                    k5,k6,
                    fsalfirst,fsallast,dT,J,W,tmp,atmp,tab,tf,uf,linsolve_tmp,
                    linsolve,jac_config,grad_config)
end

function alg_cache(alg::Rodas4,u,rate_prototype,uEltypeNoUnits,uBottomEltypeNoUnits,tTypeNoUnits,uprev,uprev2,f,t,dt,reltol,p,calck,::Val{false})
  tf = TimeDerivativeWrapper(f,u,p)
  uf = UDerivativeWrapper(f,t,p)
  J,W = build_J_W(alg,u,uprev,p,t,dt,f,uEltypeNoUnits,Val(false))
  linsolve = alg.linsolve(Val{:init},uf,u)
  Rodas4ConstantCache(tf,uf,Rodas4Tableau(constvalue(uBottomEltypeNoUnits),constvalue(tTypeNoUnits)),J,W,linsolve)
end

function alg_cache(alg::Rodas42,u,rate_prototype,uEltypeNoUnits,uBottomEltypeNoUnits,tTypeNoUnits,uprev,uprev2,f,t,dt,reltol,p,calck,::Val{true})
  dense1 = zero(rate_prototype)
  dense2 = zero(rate_prototype)
  du = zero(rate_prototype)
  du1 = zero(rate_prototype)
  du2 = zero(rate_prototype)
  k1 = zero(rate_prototype)
  k2 = zero(rate_prototype)
  k3 = zero(rate_prototype)
  k4 = zero(rate_prototype)
  k5 = zero(rate_prototype)
  k6 = zero(rate_prototype)
  fsalfirst = zero(rate_prototype)
  fsallast = zero(rate_prototype)
  dT = zero(rate_prototype)
  J,W = build_J_W(alg,u,uprev,p,t,dt,f,uEltypeNoUnits,Val(true))
  tmp = zero(rate_prototype)
  atmp = similar(u, uEltypeNoUnits)
  tab = Rodas42Tableau(constvalue(uBottomEltypeNoUnits),constvalue(tTypeNoUnits))

  tf = TimeGradientWrapper(f,uprev,p)
  uf = UJacobianWrapper(f,t,p)
  linsolve_tmp = zero(rate_prototype)
  linsolve = alg.linsolve(Val{:init},uf,u)
  grad_config = build_grad_config(alg,f,tf,du1,t)
  jac_config = build_jac_config(alg,f,uf,du1,uprev,u,tmp,du2)
  Rodas4Cache(u,uprev,dense1,dense2,du,du1,du2,k1,k2,k3,k4,
                    k5,k6,
                    fsalfirst,fsallast,dT,J,W,tmp,atmp,tab,tf,uf,linsolve_tmp,
                    linsolve,jac_config,grad_config)
end

function alg_cache(alg::Rodas42,u,rate_prototype,uEltypeNoUnits,uBottomEltypeNoUnits,tTypeNoUnits,uprev,uprev2,f,t,dt,reltol,p,calck,::Val{false})
  tf = TimeDerivativeWrapper(f,u,p)
  uf = UDerivativeWrapper(f,t,p)
  J,W = build_J_W(alg,u,uprev,p,t,dt,f,uEltypeNoUnits,Val(false))
  linsolve = alg.linsolve(Val{:init},uf,u)
  Rodas4ConstantCache(tf,uf,Rodas42Tableau(constvalue(uBottomEltypeNoUnits),constvalue(tTypeNoUnits)),J,W,linsolve)
end

function alg_cache(alg::Rodas4P,u,rate_prototype,uEltypeNoUnits,uBottomEltypeNoUnits,tTypeNoUnits,uprev,uprev2,f,t,dt,reltol,p,calck,::Val{true})
  dense1 = zero(rate_prototype)
  dense2 = zero(rate_prototype)
  du = zero(rate_prototype)
  du1 = zero(rate_prototype)
  du2 = zero(rate_prototype)
  k1 = zero(rate_prototype)
  k2 = zero(rate_prototype)
  k3 = zero(rate_prototype)
  k4 = zero(rate_prototype)
  k5 = zero(rate_prototype)
  k6 = zero(rate_prototype)
  fsalfirst = zero(rate_prototype)
  fsallast = zero(rate_prototype)
  dT = zero(rate_prototype)
  J,W = build_J_W(alg,u,uprev,p,t,dt,f,uEltypeNoUnits,Val(true))
  tmp = zero(rate_prototype)
  atmp = similar(u, uEltypeNoUnits)
  tab = Rodas4PTableau(constvalue(uBottomEltypeNoUnits),constvalue(tTypeNoUnits))

  tf = TimeGradientWrapper(f,uprev,p)
  uf = UJacobianWrapper(f,t,p)
  linsolve_tmp = zero(rate_prototype)
  linsolve = alg.linsolve(Val{:init},uf,u)
  grad_config = build_grad_config(alg,f,tf,du1,t)
  jac_config = build_jac_config(alg,f,uf,du1,uprev,u,tmp,du2)
  Rodas4Cache(u,uprev,dense1,dense2,du,du1,du2,k1,k2,k3,k4,
                    k5,k6,
                    fsalfirst,fsallast,dT,J,W,tmp,atmp,tab,tf,uf,linsolve_tmp,
                    linsolve,jac_config,grad_config)
end

function alg_cache(alg::Rodas4P,u,rate_prototype,uEltypeNoUnits,uBottomEltypeNoUnits,tTypeNoUnits,uprev,uprev2,f,t,dt,reltol,p,calck,::Val{false})
  tf = TimeDerivativeWrapper(f,u,p)
  uf = UDerivativeWrapper(f,t,p)
  J,W = build_J_W(alg,u,uprev,p,t,dt,f,uEltypeNoUnits,Val(false))
  linsolve = alg.linsolve(Val{:init},uf,u)
  Rodas4ConstantCache(tf,uf,Rodas4PTableau(constvalue(uBottomEltypeNoUnits),constvalue(tTypeNoUnits)),J,W,linsolve)
end

function alg_cache(alg::Rodas4P2,u,rate_prototype,uEltypeNoUnits,uBottomEltypeNoUnits,tTypeNoUnits,uprev,uprev2,f,t,dt,reltol,p,calck,::Val{true})
  dense1 = zero(rate_prototype)
  dense2 = zero(rate_prototype)
  du = zero(rate_prototype)
  du1 = zero(rate_prototype)
  du2 = zero(rate_prototype)
  k1 = zero(rate_prototype)
  k2 = zero(rate_prototype)
  k3 = zero(rate_prototype)
  k4 = zero(rate_prototype)
  k5 = zero(rate_prototype)
  k6 = zero(rate_prototype)
  fsalfirst = zero(rate_prototype)
  fsallast = zero(rate_prototype)
  dT = zero(rate_prototype)
  J,W = build_J_W(alg,u,uprev,p,t,dt,f,uEltypeNoUnits,Val(true))
  tmp = zero(rate_prototype)
  atmp = similar(u, uEltypeNoUnits)
  tab = Rodas4P2Tableau(constvalue(uBottomEltypeNoUnits),constvalue(tTypeNoUnits))

  tf = TimeGradientWrapper(f,uprev,p)
  uf = UJacobianWrapper(f,t,p)
  linsolve_tmp = zero(rate_prototype)
  linsolve = alg.linsolve(Val{:init},uf,u)
  grad_config = build_grad_config(alg,f,tf,du1,t)
  jac_config = build_jac_config(alg,f,uf,du1,uprev,u,tmp,du2)
  Rodas4Cache(u,uprev,dense1,dense2,du,du1,du2,k1,k2,k3,k4,
                    k5,k6,
                    fsalfirst,fsallast,dT,J,W,tmp,atmp,tab,tf,uf,linsolve_tmp,
                    linsolve,jac_config,grad_config)
end

function alg_cache(alg::Rodas4P2,u,rate_prototype,uEltypeNoUnits,uBottomEltypeNoUnits,tTypeNoUnits,uprev,uprev2,f,t,dt,reltol,p,calck,::Val{false})
  tf = TimeDerivativeWrapper(f,u,p)
  uf = UDerivativeWrapper(f,t,p)
  J,W = build_J_W(alg,u,uprev,p,t,dt,f,uEltypeNoUnits,Val(false))
  linsolve = alg.linsolve(Val{:init},uf,u)
  Rodas4ConstantCache(tf,uf,Rodas4P2Tableau(constvalue(uBottomEltypeNoUnits),constvalue(tTypeNoUnits)),J,W,linsolve)
end

################################################################################

### Rosenbrock5

struct Rosenbrock5ConstantCache{TF,UF,Tab,JType,WType,F} <: OrdinaryDiffEqConstantCache
  tf::TF
  uf::UF
  tab::Tab
  J::JType
  W::WType
  linsolve::F
end

@cache mutable struct Rosenbrock5Cache{uType,rateType,uNoUnitsType,JType,WType,TabType,TFType,UFType,F,JCType,GCType} <: RosenbrockMutableCache
  u::uType
  uprev::uType
  dense1::rateType
  dense2::rateType
  du::rateType
  du1::rateType
  du2::rateType
  k1::rateType
  k2::rateType
  k3::rateType
  k4::rateType
  k5::rateType
  k6::rateType
  k7::rateType
  k8::rateType
  fsalfirst::rateType
  fsallast::rateType
  dT::rateType
  J::JType
  W::WType
  tmp::rateType
  atmp::uNoUnitsType
  tab::TabType
  tf::TFType
  uf::UFType
  linsolve_tmp::rateType
  linsolve::F
  jac_config::JCType
  grad_config::GCType
end

function alg_cache(alg::Rodas5,u,rate_prototype,uEltypeNoUnits,uBottomEltypeNoUnits,tTypeNoUnits,uprev,uprev2,f,t,dt,reltol,p,calck,::Val{true})
  dense1 = zero(rate_prototype)
  dense2 = zero(rate_prototype)
  du = zero(rate_prototype)
  du1 = zero(rate_prototype)
  du2 = zero(rate_prototype)
  k1 = zero(rate_prototype)
  k2 = zero(rate_prototype)
  k3 = zero(rate_prototype)
  k4 = zero(rate_prototype)
  k5 = zero(rate_prototype)
  k6 = zero(rate_prototype)
  k7 = zero(rate_prototype)
  k8 = zero(rate_prototype)
  fsalfirst = zero(rate_prototype)
  fsallast = zero(rate_prototype)
  dT = zero(rate_prototype)
  J,W = build_J_W(alg,u,uprev,p,t,dt,f,uEltypeNoUnits,Val(true))
  tmp = zero(rate_prototype)
  atmp = similar(u, uEltypeNoUnits)
  tab = Rodas5Tableau(constvalue(uBottomEltypeNoUnits),constvalue(tTypeNoUnits))

  tf = TimeGradientWrapper(f,uprev,p)
  uf = UJacobianWrapper(f,t,p)
  linsolve_tmp = zero(rate_prototype)
  linsolve = alg.linsolve(Val{:init},uf,u)
  grad_config = build_grad_config(alg,f,tf,du1,t)
  jac_config = build_jac_config(alg,f,uf,du1,uprev,u,tmp,du2)
  Rosenbrock5Cache(u,uprev,dense1,dense2,du,du1,du2,k1,k2,k3,k4,
                    k5,k6,k7,k8,
                    fsalfirst,fsallast,dT,J,W,tmp,atmp,tab,tf,uf,linsolve_tmp,
                    linsolve,jac_config,grad_config)
end

function alg_cache(alg::Rodas5,u,rate_prototype,uEltypeNoUnits,uBottomEltypeNoUnits,tTypeNoUnits,uprev,uprev2,f,t,dt,reltol,p,calck,::Val{false})
  tf = TimeDerivativeWrapper(f,u,p)
  uf = UDerivativeWrapper(f,t,p)
  J,W = build_J_W(alg,u,uprev,p,t,dt,f,uEltypeNoUnits,Val(false))
  linsolve = alg.linsolve(Val{:init},uf,u)
  Rosenbrock5ConstantCache(tf,uf,Rodas5Tableau(constvalue(uBottomEltypeNoUnits),constvalue(tTypeNoUnits)),J,W,linsolve)
end

################################################################################

### RosenbrockW6S4O

@RosenbrockW6S4OS(:cache)


# This script loads the raw data from all experiments, runs the protocols and
# stores some values that will be useful to run the tests and the extraction of
# the results from a simulated run in fields of type Base.RefValue.
# The results are saved and are loaded by FoodCachingExperiments at init.
using Pkg
Pkg.activate(joinpath(@__DIR__, ".."))
using TOML, CSV, FoodCachingExperiments, DataFramesMeta, Distributions, BSON, CodecZstd
import FoodCachingExperiments: EXPERIMENTS, Experiment, TestSummary, Food,
       default_keycols, bsave
for foodtype in instances(Food)
    eval(:(import FoodCachingExperiments: $(Symbol(foodtype))))
end

include(joinpath(@__DIR__, "..", "examples", "dummy_model.jl"))

const DATADIR = joinpath(@__DIR__, "..", "data", "raw")

function load_experiment(name; datadir = DATADIR, run = true)
    meta = TOML.parsefile(joinpath(datadir, "$name.toml"))
    data = CSV.File(joinpath(datadir, "$name.csv"), pool = false) |> DataFrame
    tests = CSV.File(joinpath(datadir, "$(name)_tests.csv")) |> DataFrame |>
            processtests
    ex = Experiment{name}(Base.RefValue{Any}(),
                          Base.RefValue{Any}(),
                          haskey(meta, "major_finding") ?
                          meta["major_finding"] : (@warn("$name misses major_finding"); ""),
                          haskey(meta, "comments") ?
                          meta["comments"] : (@warn("$name misses comments"); ""),
                          sort!(data, default_keycols(data)),
                          tests, meta["nbirds"],
                          eval.(Meta.parse.(meta["foodtypes"])))
    if run
        data = run!(ex, fill(DummyModel(), ex.nbirds))
        res = results(ex, data)
        ex = dropref(ex)
        length(res) == length(target(ex)) || @warn "$name: result and target have different lengths"
        ex
    else
        ex
    end
end

function processtests(tests)
    result = Dict{String,TestSummary}()
    for g in groupby(tests, :id)
        result[g.id[1]] = TestSummary(
                                @with(g, pvalue.(:statistic, :df, :value)),
                                string.(g.key),
                                string.(g.df),
                                string.(g.label),
                                string.(g.statistic),
                                string.(g.value),
                                Base.RefValue{Any}(),
                                Base.RefValue{Any}()
                               )
    end
    result
end
pvalue(statistic, df::String, value) = pvalue(statistic, eval(Meta.parse(df)), value)
function pvalue(statistic, df, value)
    if typeof(value) == String
        if value[1] == '<'
            value = eval(Meta.parse(value[2:end])) - .01
        elseif value[1] == '>'
            value = eval(Meta.parse(value[2:end])) + .01
        else
            value = eval(Meta.parse(value))
        end
    end
    if statistic == "F"
        ccdf(FDist(df[1], df[2]), value)
    elseif statistic == "t"
        ccdf(FDist(1, df[1]), value^2)
    elseif statistic == "U" || statistic == "FisherExactTest" || statistic == "Binomial"
        value
    elseif statistic == "Chisq"
        ccdf(Chisq(df), value)
    else
        error("Don't know how to treat statistic $statistic.")
    end
end

function dropref(x::TestSummary)
    TestSummary(x.pvalues, x.keys, x.df, x.labels, x.statistic, x.value,
                isassigned(x.locals) ? x.locals[] : nothing,
                isassigned(x.accessor) ? x.accessor[] : nothing)
end
function dropref(x::Dict{String, TestSummary})
    Dict(id => dropref(test) for (id, test) in x)
end
function dropref(e::Experiment{name}) where name
    Experiment{name}(e.data_accessor[],
                     e.target[],
                     e.major_finding,
                     e.comments,
                     e.data,
                     dropref(e.tests),
                     e.nbirds,
                     e.foodtypes)
end

println("Processing experiments. This will take some time!")
for filename in readdir(DATADIR)
    name, ending = splitext(filename)
    name = Symbol(name)
    run = name ∉ FoodCachingExperiments.CLAYTON0103_EXPERIMENTS
    if ending == ".toml"
        println("$name ...")
        EXPERIMENTS[name] = load_experiment(name; run)
    end
end

# This is a dummy experiment to aggregate all the sequentially performed experiments
println("Clayton0103 ...")
EXPERIMENTS[:Clayton0103] = Experiment{:Clayton0103}(Base.RefValue{Any}(),
                                                     Base.RefValue{Any}(),
                                                     "",
                                                     "Running 01/03 experiments in one go.",
                                                     DataFrame(),
                                                     Dict{String, TestSummary}(),
                                                     16,
                                                     [Peanut, Mealworm, Cricket])
data = run!(EXPERIMENTS[:Clayton0103], fill(DummyModel(), 16))
res = results(EXPERIMENTS[:Clayton0103], data)
EXPERIMENTS[:Clayton0103] = dropref(EXPERIMENTS[:Clayton0103])
length(res) == length(target(EXPERIMENTS[:Clayton0103])) || @warn "Clayton0103: result and target have different lengths"
for e in FoodCachingExperiments.CLAYTON0103_EXPERIMENTS
    EXPERIMENTS[e] = dropref(EXPERIMENTS[e])
end

bsave(joinpath(DATADIR, "..", "processed", "experiments"), EXPERIMENTS)


#=##############################################################################
# DESCRIPTION
    Simulation interface connecting UVLMVehicle vehicles and maneuvers.

# AUTHORSHIP
  * Author    : Eduardo J. Alvarez
  * Email     : Edo.AlvarezR@gmail.com
  * Created   : Oct 2019
  * License   : MIT
=###############################################################################

function solve(self::Simulation{V, M, R}, Vinf::Function,
                pfield::vpm.ParticleField, wake_coupled::Bool,
                dt::Real, rlx::Real, sigma::Real, rho::Real,
                speedofsound::Real, staticpfield::vpm.ParticleField;
                init_sol::Bool=false, vlm_fsgm=1
                ) where {V<:UVLMVehicle, M<:AbstractManeuver, R}

    # Bring the rotors back to their initial positions at beginning of this
    # time step before `precalculations`. This is needed to avoid double
    # accounting for the RPM when the solver is run
    rotate_rotors(self, -dt)

    vhcl = self.vehicle
    mnvr = self.maneuver
    t = self.t

    # TIME-STEPPING PROCEDURE:
    # -1) Solve one particle field time step
    # 0) Translate and rotate systems
    # 1) Recalculate horseshoes with kinematic velocity
    # 2) Paste previous Gamma solution to new system position after translation
    # 3) Shed semi-infinite wake after translation
    # 4) Calculate wake-induced velocity on VLM and Rotor system
    # 5) Solve VLM and Rotor system
    # 6) Shed unsteady-loading wake after new solution
    # 7) Save new solution as prev solution
    # Iterate
    #
    # On the first time step (pfield.nt==0), it only does steps (5) and (7),
    # meaning that the unsteady wake of the first time step is never shed.
    if self.nt==0
        # NOTE: VLMs and Rotors solutions are losely coupled

        # Set Vinf on system to get horseshoes
        vlm.setVinf(vhcl.system, Vinf)

        # Solve VLMs
        # TODO: Add Rotor-on-VLM induced velocity
        if init_sol
            # NOTE: Here I use the semi-infinite wake for the first step, which
            # may lead to some unphysical results when shedding unsteady loading wake
            vlm.solve(vhcl.vlm_system, Vinf; t=t, keep_sol=true)
        else
            vlm.solve(vhcl.vlm_system, Vinf; t=t, keep_sol=true,
                                                    vortexsheet=(X,t)->zeros(3))
        end

        # Calculate induced velocities to use in rotor solver
        ## Points where to calculate induced velocities
        Xs = _get_midXs(vhcl.rotor_systems)

        ## Particles for VLM-on-Rotor induced velocity
        static_particles_fun(pfield, args...) = _static_particles(pfield, vhcl.vlm_system, sigma)

        ## Evaluate VPM-on-Rotor induced velocity + static particles
        Vinds = Vvpm_on_Xs(pfield, Xs; static_particles_fun=static_particles_fun, dt=dt, fsgm=vlm_fsgm)

        # Solve Rotors
        for (si, rotors) in enumerate(vhcl.rotor_systems)

            # RPM of this rotor system
            RPM = self.RPMref*get_RPM(mnvr, si, t/self.ttot)

            for (ri, rotor) in enumerate(rotors)

                # Get velocities induced at every blade of this rotor
                this_Vinds = _parse_midXs(vhcl.rotor_systems,  Vinds, si, ri)

                VindVkin = _format_blades(this_Vinds, vhcl.rotor_systems, si, ri)

                if wake_coupled
                    vlm.solvefromV(rotor, VindVkin, Vinf, RPM, rho; t=t,
                                    sound_spd=speedofsound,
                                    tiploss_correction=vlm_fsgm==1)
                else
                    vlm.solvefromCCBlade(rotor, Vinf, RPM, rho;
                                              t=t, sound_spd=speedofsound)
                end
            end
        end

    else
        # ---------- 4) Calculate VPM velocity on VLM and Rotor system -----
        m = vlm.get_m(vhcl.vlm_system)

        # Control points of VLM
        Xs_cp_vlm = _get_Xs(vhcl.vlm_system, "CP"; t=t)

        # Mid-points along VLM filaments for calc_aerodynamicforce()
        ## Nodes of every horseshoe
        Ap = _get_Xs(vhcl.vlm_system, "Ap")
        A = _get_Xs(vhcl.vlm_system, "A")
        B = _get_Xs(vhcl.vlm_system, "B")
        Bp = _get_Xs(vhcl.vlm_system, "Bp")
        ## Midpoints of bound vortices
        ApA = (Ap .+ A)/2
        AB = (A .+ B)/2
        BBp = (B .+ Bp)/2
        ## Organize them
        Xs_ApA_AB_BBp_vlm = vcat(ApA, AB, BBp)

        # Points at rotors where to calculate induced velocities
        Xs_rotors = _get_midXs(vhcl.rotor_systems)

        # Calculate VPM velocity on all points (VLM and rotors)
        Vvpm = Vvpm_on_Xs(pfield, vcat(Xs_cp_vlm, Xs_ApA_AB_BBp_vlm, Xs_rotors); dt=dt, fsgm=vlm_fsgm)

        Vvpm_cp_vlm = Vvpm[1:m]
        Vvpm_ApA_AB_BBp_vlm = Vvpm[m+1:4*m]
        Vvpm_rotors = Vvpm[4*m+1:end]

        # Stitches previous rotor solutions for blade induced velocity
        prev_rotor_systems = _get_prev_rotor_systems(vhcl)
        allrotors = vlm.WingSystem()

        for (si, rotors) in enumerate(vhcl.rotor_systems)
            for (ri, rotor) in enumerate(rotors)

                vlm.addwing(allrotors, "S$(si)R$(ri)", rotor; reset=false)
                vlm.getHorseshoe(rotor, 1)          # Force HS calculation
                vlm._addsolution(rotor, "Gamma",    # Give previous solution
                                prev_rotor_systems[si][ri]._wingsystem.sol["Gamma"])
            end
        end

        # Particles for Rotor-on-VLM induced velocity
        static_particles_fun2(pfield, args...) = _static_particles(pfield, allrotors, sigma)

        # Evaluate Rotor-on-VLM induced velocity
        Vrotor_on_wing = Vvpm_on_Xs(staticpfield, vcat(Xs_cp_vlm, Xs_ApA_AB_BBp_vlm);
                                    static_particles_fun=static_particles_fun2, dt=dt, fsgm=vlm_fsgm)

        # Add and save VPM-on-VLM and Rotor-on-VLM induced velocity
        vlm._addsolution(vhcl.vlm_system, "Vvpm",
                                         Vvpm_cp_vlm + Vrotor_on_wing[1:m]; t=t)
        vlm._addsolution(vhcl.vlm_system, "Vvpm_ApA",
                        Vvpm_ApA_AB_BBp_vlm[1:m] + Vrotor_on_wing[m+1:2*m]; t=t)
        vlm._addsolution(vhcl.vlm_system, "Vvpm_AB",
                        Vvpm_ApA_AB_BBp_vlm[m+1:2*m] + Vrotor_on_wing[2*m+1:3*m]; t=t)
        vlm._addsolution(vhcl.vlm_system, "Vvpm_BBp",
                        Vvpm_ApA_AB_BBp_vlm[2*m+1:3*m] + Vrotor_on_wing[3*m+1:4*m]; t=t)

        # Calculate induced velocities to use in rotor solver
        ## Particles for VLM-on-Rotor induced velocity
        # NOTE: If I keep the rotor particles wouldn't I be double-counting
        # the blade induced velocity and make the solution unstable?
        # ANSWER: We are never double-accounting for the blade induced velocity
        function static_particles_fun3(pfield, args...)
            # Rotor static particles
            static_particles_fun2(pfield, args...)
            # VLM static particles
            _static_particles(pfield, vhcl.vlm_system, sigma)
        end

        ## Evaluate VLM-on-Rotor and Rotor-on-Rotor induced velocity
        Vvlmrotor_on_rotor = Vvpm_on_Xs(staticpfield, Xs_rotors;
                              static_particles_fun=static_particles_fun3, dt=dt, fsgm=vlm_fsgm)

        # Add VPM-on-Rotor, VLM-on-Rotor, and Rotor-on-Rotor induced velocity
        Vinds = Vvpm_rotors + Vvlmrotor_on_rotor

        # ---------- 5) Solve VLM system -----------------------------------
        # Wake-coupled solution
        if wake_coupled
            vlm.solve(vhcl.vlm_system, Vinf; t=t, extraVinf=_extraVinf2,
                                    keep_sol=true, vortexsheet=(X,t)->zeros(3))
        # Wake-decoupled solution
        else
            vlm.solve(vhcl.vlm_system, Vinf; t=t, extraVinf=_extraVinf1,
                                                                keep_sol=true)
        end

        # Relaxes (rlx->1) or stiffens (rlx->0) the VLM solution
        if rlx > 0
            rlxd_Gamma = rlx*vhcl.vlm_system.sol["Gamma"] +
                                (1-rlx)*_get_prev_vlm_system(vhcl).sol["Gamma"]
            vlm._addsolution(vhcl.vlm_system, "Gamma", rlxd_Gamma)
        end

        # ---------- 5) Solve Rotor system ---------------------------------

        # Solve Rotors
        for (si, rotors) in enumerate(vhcl.rotor_systems)

            # RPM of this rotor system
            RPM = self.RPMref*get_RPM(mnvr, si, t/self.ttot)

            for (ri, rotor) in enumerate(rotors)

                # Calculate kinematic velocities
                Vkin = _Vkinematic_rotor(vhcl.rotor_systems,
                                         prev_rotor_systems, si, ri, dt)

                # Get velocities induced at every blade of this rotor
                this_Vinds = _parse_midXs(vhcl.rotor_systems, Vinds, si, ri)

                VindVkin = _format_blades(this_Vinds + Vkin, vhcl.rotor_systems,
                                                                        si, ri)
                if wake_coupled
                    vlm.solvefromV(rotor, VindVkin, Vinf, RPM, rho; t=t,
                                    sound_spd=speedofsound,
                                    tiploss_correction=vlm_fsgm==1)
                else
                    vlm.solvefromCCBlade(rotor, Vinf, RPM, rho;
                                                t=t, sound_spd=speedofsound)
                end
            end
        end

        # Rotate rotors
        # NOTE: this means that the rotational velocity hadn't been included in
        # the kinematic velocities up to this point
        rotate_rotors(self, dt)
    end
end


using DataStructures

"""
Hybrid Actor-critic
-------------------

Hybrid model : Actor-critic & Forward learner

See learner/forwardlearner.jl, learner/actorcriticlearner.jl
and learner/hybridlearner.jl for the core implementation.

Note 1
------
get_all_puzzles(), get_all_episodes_in_puzzle(), get_data_by_puzzle_episode()
are in rl_data_toolsforalgos.jl

Note 2
------
Temperature of softmax is removed, because this learner in NOT convex and weights
can be in (0, +Inf). The temperature is absorbed in the weights. Otherwise
it is overparametrized.
"""
function getLL_Hybrid_AC_FWD(data::Array{Int,2},
                            alpha::Float64, eta_critic::Float64,
                            gamma::Float64, lambda_actor::Float64,
                            lambda_critic::Float64,
                            eta_fwd::Float64,
                            w_MB::Float64,
                            w_MF::Float64;
                            init_R = 0.,
                            init_policy=nothing,
                            init_Vs = nothing,
                            use_replacing_trace=true,
                            do_debug_print = false)

    data_stats, nr_states, nr_actions, nr_samples = get_data_stats_and_dimensions(data)
    if data_stats.nr_subjects != 1
        error("Invalid argument: getLL_Hybrid_AC_FWD supports only one subject")
    end
    all_puzzles = get_all_puzzles(data)

    LL = 0.
    action_selection_probs = zeros(Float64, nr_samples)
    # signals:
    TD_record = zeros(Float64, nr_samples)
    SPE_record = zeros(Float64, nr_samples)
    V_state = zeros(Float64, nr_samples + data_stats.nr_rewards)

    PolicyParametersDiffChosenVsUnchosen_record = zeros(Float64, nr_samples)
    PolicyDiffChosenVsUnchosen_record = zeros(Float64, nr_samples)

    learnerMB = []
    learnerMF = []
    learnerHybrid = []
    modulatingfactor = 0.
    action_counter = 0
    for i_puzzle in all_puzzles
        # Puzzle = independent block of the experiment.
        # The code was made for general purposes, in this experiment we
        # have only ONE puzzle.
        learnerMB = ForwardLearner(nr_states = nr_states,
                            nr_actions = nr_actions,
                            η = eta_fwd, γ = gamma,
                            init_R = 0.
                            )
        if init_policy == nothing # providing init_policy w/o init_Vs is considered a bug
            learnerMF = ActorCritic(nr_states = nr_states,
                                nr_actions = nr_actions,
                                α_critic = eta_critic,
                                α_actor = alpha,
                                γ = gamma,
                                λ_critic = lambda_critic,
                                λ_actor = lambda_actor,
                                use_replacing_trace = use_replacing_trace,
                                do_debug_print = do_debug_print)
        else
            learnerMF = ActorCritic(nr_states = nr_states,
                                nr_actions = nr_actions,
                                α_critic = eta_critic,
                                α_actor = alpha,
                                γ = gamma,
                                λ_critic = lambda_critic,
                                λ_actor = lambda_actor,
                                policyparameters = copy(init_policy[current_puzzle_id]),
                                V = copy(init_Vs[current_puzzle_id]),
                                use_replacing_trace = use_replacing_trace,
                                do_debug_print = do_debug_print)
        end
        learnerHybrid = Hybrid(nr_states = nr_states,
                            nr_actions = nr_actions,
                            w_MB = w_MB,
                            w_MF = w_MF
                            )
        all_epi_in_puzzle = get_all_episodes_in_puzzle(data, i_puzzle)
        j = 1
        for i_episode in all_epi_in_puzzle
            episode_data = get_data_by_puzzle_episode(data, i_puzzle, i_episode)
            path_length = size(episode_data)[1]  # = T
            if do_debug_print
                println("i_puzzle:$i_puzzle | i_episode:$i_episode | path_length:$path_length")
            end
            isendofexperiment = false

            for t in 1:path_length
                action_counter += 1
                state = episode_data[t, Int(c_state)]
                action = episode_data[t, Int(c_action)]
                reward = episode_data[t, Int(c_reward)]
                next_state = episode_data[t, Int(c_nextState)]
                done = reward>0 ? true : false
                # @show state, action, reward, next_state
                if do_debug_print
                    # println("t:$t | s:$state | a:$action | R:$reward | s':$next_state | p(s,a):$(round(p_action,2)) | p(s,1;+-bias): $(round(p1_pos,2))/$(round(p1_neg,2))")
                    println("t:$t | s:$state | a:$action | R:$reward | s':$next_state | done: $done")
                end

                # Hybrid learner
                update!(learnerHybrid, state, action, reward, learnerMB.QSA, learnerMF.policyparameters) #Q_hybrid =  w_MB*learnerMB.QSA + w_MF*learnerMF.QSA

                p_actions = getSoftmaxProbs(learnerHybrid.QSA[state, :], 1., [0., 0.])
                action_selection_probs[action_counter] = copy(p_actions[action])
                LL += log(p_actions[action])

                PolicyParametersDiffChosenVsUnchosen_record[action_counter] = learnerHybrid.QSA[state, action] - learnerHybrid.QSA[state, mod(action,2)+1]
                PolicyDiffChosenVsUnchosen_record[action_counter] = p_actions[action] - p_actions[mod(action,2)+1]

                # ForwardLearner
                update!(learnerMB, state, action, next_state, reward)
                SPE_record[action_counter] = copy(learnerMB.spe[1])

                # Track V values (used as fmri regressor)
                V_state[j] = learnerMF.V[state]
                if done
                    j+=1
                    if j < (nr_samples + data_stats.nr_rewards)
                        V_state[j] = learnerMF.V[next_state] # Track V value of landing state in case it is the Goal
                    end
                end
                j+=1

                # Actor-Critic
                update!(learnerMF, state, action, next_state, reward, p_actions[action], modulatingfactor)
                TD_record[action_counter] = learnerMF.td_error[1]
            end # end t
        end # end i_episode
    end # end i_puzzle
    #---- Without bias
    trial_by_trial_p_sa = (action_selection_probs)
    if do_debug_print
        println(learnerHybrid.QSA)
        println("action_selection_probs: $action_selection_probs")
        println("LL_max: $LL")
        println("trial_by_trial_p_sa: $trial_by_trial_p_sa")
    end
    algo = "hybrid_actorcritic_FWD"

    param_v = [alpha, eta_critic, gamma, lambda_actor, lambda_critic, eta_fwd, w_MB, w_MF]
    param_str = "alpha=$(round(alpha,digits=4)),
                eta_critic=$(round(eta_critic,digits=4)),
                gamma=$(round(gamma,digits=4)),
                lambda_actor=$(round(lambda_actor,digits=4)),
                lambda_critic=$(round(lambda_critic,digits=4)),
                eta_fwd=$(round(eta_fwd,digits=4)),
                w_MB=$(round(w_MB,digits=4)),
                w_MF=$(round(w_MF,digits=4))"

    results_dict = Dict{String,Any}("finalQSA_Hybrid_record" => learnerHybrid.QSA,
                                    "final_PolicyParameters_MF" => learnerMF.policyparameters,
                                    "finalQSA_MB_record" => learnerMB.QSA,
                                    "final_Tsas" => learnerMB.TSAS,
                                    "trialByTrial_Vs" => V_state,
                                    "PolicyParametersDiffChosenVsUnchosen" => PolicyParametersDiffChosenVsUnchosen_record,
                                    "PolicyDiffChosenVsUnchosen" => PolicyDiffChosenVsUnchosen_record,
                                )
    signals = Step_by_step_signals(data_stats.all_subject_ids[1],
    TD_record, #RPE
    SPE_record, #SPE
    trial_by_trial_p_sa,
    LL,
    data_stats,
    algo,
    param_v,
    param_str,
    99, #biased_action_idx, dummy value
    results_dict)
    return LL, signals
end