const WCS_HEADERS_TEMPLATES = [
    "DATE",
    "MJD",

    "WCSAXESa",
    "WCAXna",
    "WCSTna",
    "WCSXna",
    "CRPIXja",
    "jCRPna",
    "jCRPXn",
    "TCRPna",
    "TCRPXn",
    "PCi_ja",
    "ijPCna",
    "TPn_ka",
    "TPCn_ka",
    "CDi_ja",
    "ijCDna",
    "TCn_ka",
    "TCDn_ka",
    "CDELTia",
    "iCDEna",
    "iCDLTn",
    "TCDEna",
    "TCDLTn",
    "CROTAi",
    "iCROTn",
    "TCROTn",
    "CUNITia",
    "iCUNna",
    "iCUNIn",
    "TCUNna",
    "TCUNIn",
    "CTYPEia",
    "iCTYna",
    "iCTYPn",
    "TCTYna",
    "TCTYPn",
    "CRVALia",
    "iCRVna",
    "iCRVLn",
    "TCRVna",
    "TCRVLn",
    "LONPOLEa",
    "LONPna",
    "LATPOLEa",
    "LATPna",
    "RESTFREQ",
    "RESTFRQa",
    "RFRQna",
    "RESTWAVa",
    "RWAVna",
    "PVi_ma",
    "iVn_ma",
    "iPVn_ma",
    "TVn_ma",
    "TPVn_ma",
    "PROJPm",
    "PSi_ma",
    "iSn_ma",
    "iPSn_ma",
    "TSn_ma",
    "TPSn_ma",
    "VELREF",
    "CNAMEia",
    "iCNAna",
    "iCNAMn",
    "TCNAna",
    "TCNAMn",
    "CRDERia",
    "iCRDna",
    "iCRDEn",
    "TCRDna",
    "TCRDEn",
    "CSYERia",
    "iCSYna",
    "iCSYEn",
    "TCSYna",
    "TCSYEn",
    "CZPHSia",
    "iCZPna",
    "iCZPHn",
    "TCZPna",
    "TCZPHn",
    "CPERIia",
    "iCPRna",
    "iCPERn",
    "TCPRna",
    "TCPERn",
    "WCSNAMEa",
    "WCSNna",
    "TWCSna",
    "TIMESYS",
    "TREFPOS",
    "TRPOSn",
    "TREFDIR",
    "TRDIRn",
    "PLEPHEM",
    "TIMEUNIT",
    "DATEREF",
    "MJDREF",
    "MJDREFI",
    "MJDREFF",
    "JDREF",
    "JDREFI",
    "JDREFF",
    "TIMEOFFS",
    "DATE-OBS",
    "DOBSn",
    "DATE-BEG",
    "DATE-AVG",
    "DAVGn",
    "DATE-END",
    "MJD-OBS",
    "MJDOBn",
    "MJD-BEG",
    "MJD-AVG",
    "MJDAn",
    "MJD-END",
    "JEPOCH",
    "BEPOCH",
    "TSTART",
    "TSTOP",
    "XPOSURE",
    "TELAPSE",
    "TIMSYER",
    "TIMRDER",
    "TIMEDEL",
    "TIMEPIXR",
    "OBSGEO-X",
    "OBSGXn",
    "OBSGEO-Y",
    "OBSGYn",
    "OBSGEO-Z",
    "OBSGZn",
    "OBSGEO-L",
    "OBSGLn",
    "OBSGEO-B",
    "OBSGBn",
    "OBSGEO-H",
    "OBSGHn",
    "OBSORBIT",
    "RADESYSa",
    "RADEna",
    "RADECSYS",
    "EPOCH",
    "EQUINOXa",
    "EQUIna",
    "SPECSYSa",
    "SPECna",
    "SSYSOBSa",
    "SOBSna",
    "VELOSYSa",
    "VSYSna",
    "VSOURCEa",
    "VSOUna",
    "ZSOURCEa",
    "ZSOUna",
    "SSYSSRCa",
    "SSRCna",
    "VELANGLa",
    "VANGna",
    "RSUN_REF",
    "DSUN_OBS",
    "CRLN_OBS",
    "HGLN_OBS",
    "HGLT_OBS",
    "NAXISn",
    "CROTAn",
    "PROJPn",
    "CPDISja",
    "CQDISia",
    "DPja",
    "DQia",
    "CPERRja",
    "CQERRia",
    "DVERRa",
    "A_ORDER",
    "B_ORDER",
    "AP_ORDER",
    "BP_ORDER",
    "A_DMAX",
    "B_DMAX",
    "A_p_q",
    "B_p_q",
    "AP_p_q",
    "BP_p_q",
    "CNPIX1",
    "PPO3",
    "PPO6",
    "XPIXELSZ",
    "YPIXELSZ",
    "PLTRAH",
    "PLTRAM",
    "PLTRAS",
    "PLTDECSN",
    "PLTDECD",
    "PLTDECM",
    "PLTDECS",
    "PLATEID",
    "AMDXm",
    "AMDYm",
    "WATi_m"
]

# Expand the headers containing lower case specifers into N copies
# Find all lower case templates
const WCS_HEADERS = Set(mapreduce(vcat, WCS_HEADERS_TEMPLATES) do template
    if any(islowercase, template)
        template_chars = Vector{Char}(template)
        chars = template_chars[islowercase.(template_chars)]
        out = String[template]
        for replace_target in chars
            newout = String[]
            for template in out
                for i in [""; string.(1:4); string.('a':'d')]
                    push!(newout, replace(template, replace_target=>i))
                end
            end
            append!(out, newout)
        end
        out
    else
        template
    end
end)

"""
    emptywcs()

Given an AbstractArray, return a blank WCSTransform of the appropriate
dimensionality.
"""
emptywcs(data::AbstractArray) = WCSTransform(ndims(data))
emptywcs(img::AstroImage) = WCSTransform(length(dims(img))+length(refdims(img)))

# """
#     filterwcsheader(hdrs::FITSHeader)

# Return a new FITSHeader containing WCS header from `hdrs`.
# This is useful for creating a new image with the same coordinates
# as another.
# """
# function filterwcsheader(hdrs::FITSHeader)
#     include_keys = intersect(keys(hdrs), WCS_HEADERS)
#     return FITSHeader(
#         include_keys,
#         map(key -> hdrs[key], include_keys),
#         map(key -> get_comment(hdrs, key), include_keys),
#     )
# end

"""
    wcsfromheader(img::AstroImage; relax=WCS.HDR_ALL, ignore_rejected=true)

Helper function to create a WCSTransform from an array and
FITSHeaders.
"""
function wcsfromheader(img::AstroImage; relax=WCS.HDR_ALL)
    # We only need to stringify WCS header. This might just be 4-10 header keywords
    # out of thousands.
    local wcsout

    io = IOBuffer()
    serializeheader(io, header(img))
    hdrstr = String(take!(io))

    # Load the headers without ignoring rejected to get error messages
    try
        wcsout = WCS.from_header(
            hdrstr;
            ignore_rejected=false,
            relax
        )
    catch err
        # Load them again ignoring error messages
        wcsout = WCS.from_header(
            hdrstr;
            ignore_rejected=true,
            relax
        )
        # If that still fails, the use gets the stack trace here
        # If not, print a warning about rejected header
        @warn "WCSTransform was generated by ignoring rejected header. It may not be valid." exception=err
    end

    if length(wcsout) == 0
        return [emptywcs(img)]
    else
        return wcsout
    end
end

# Map FITS stokes numbers to a symbol
function _stokes_symbol(i)
    return if i == 1
        :I
    elseif i == 2
        :Q
    elseif i == 3
        :U
    elseif i == 4
        :V
    elseif i == -1
        :RR
    elseif i == -2
        :LL
    elseif i == -3
        :RL
    elseif i == -4
        :LR
    elseif i == -5
        :XX
    elseif i == -6
        :YY
    elseif i == -7
        :XY
    elseif i == -8
        :YX
    else
        @warn "unknown FITS stokes number $i. See \"Representations of world coordinates in FITS\", Table 7."
        nothing
    end
end
function _stokes_name(symb)
    return if symb == :I
        "Stokes Unpolarized"
    elseif symb == :Q
        "Stokes Linear Q"
    elseif symb == :U
        "Stokes Linear U"
    elseif symb == :V
        "Stokes Circular"
    elseif symb == :RR
        "Right-right cicular"
    elseif symb == :LL
        "Left-left cicular"
    elseif symb == :RL
        "Right-left cross-cicular"
    elseif symb == :LR
        "Left-right cross-cicular"
    elseif symb == :XX
        "X parallel linear"
    elseif symb == :YY
        "Y parallel linear"
    elseif symb == :XY
        "XY cross linear"
    elseif symb == :YX
        "YX cross linear"
    else
        @warn "unknown FITS stokes key $symb. See \"Representations of world coordinates in FITS\", Table 7."
        ""
    end
end

# Smart versions of pix_to_world and world_to_pix
"""
    pix_to_world(img::AstroImage, pixcoords; all=false)

Given an astro image, look up the world coordinates of the pixels given
by `pixcoords`. World coordinates are resolved using WCS.jl and a
WCSTransform calculated from any FITS header present in `img`. If
no WCS information is in the header, or the axes are all linear, this will
just return pixel coordinates.

`pixcoords` should be the coordinates in your current selection
of the image. For example, if you select a slice like this:
```julia
julia> cube = load("some-3d-cube.fits")
julia> slice = cube[10:20, 30:40, 5]
```

Then to look up the coordinates of the pixel in the bottom left corner of
`slice`, run:
```julia
julia> world_coords = pix_to_world(img, [1, 1])
[10, 30]
```

If WCS information was present in the header of `cube`, then those coordinates
would be resolved using axis 1, 2, and 3 respectively.

To include world coordinates in all axes, pass `all=true`
```julia
julia> world_coords = pix_to_world(img, [1, 1], all=true)
[10, 30, 5]
```

!! Coordinates must be provided in the order of `dims(img)`. If you transpose
an image, the order you pass the coordinates should not change.
"""
function WCS.pix_to_world(img::AstroImage, pixcoords; wcsn=1, all=false, parent=false)
    if pixcoords isa Array{Float64}
        pixcoords_prepared = pixcoords
    else
        pixcoords_prepared = [Float64(c) for c in pixcoords]
    end
    D_out = wcs(img,wcsn).naxis
    if ndims(pixcoords_prepared) > 1
        worldcoords_out = similar(pixcoords_prepared, Float64, D_out, size(pixcoords_prepared,2))
    else
        worldcoords_out = similar(pixcoords_prepared, Float64, D_out)
    end

    # Find the coordinates in the parent array.
    # Dimensional data
    # pixcoords_floored = floor.(Int, pixcoords)
    # pixcoords_frac = (pixcoords .- pixcoords_floored) .* step.(dims(img))
    # parentcoords = getindex.(dims(img), pixcoords_floored) .+ pixcoords_frac
    if parent
        parentcoords = pixcoords
    else
        parentcoords = pixcoords .* step.(dims(img)) .+ first.(dims(img))
    end
    # WCS.jl is very restrictive. We need to supply a Vector{Float64}
    # as input, not any other kind of collection.
    # TODO: avoid allocation in case where refdims=() and pixcoords isa Array{Float64}
    if ndims(pixcoords_prepared) > 1
        parentcoords_prepared = zeros(wcs(img,wcsn).naxis, size(pixcoords_prepared,2))
    else
        parentcoords_prepared = zeros(wcs(img,wcsn).naxis)
    end
    # out = zeros(Float64, wcs(img,wcsn).naxis, size(pixcoords,2))
    for (i, dim) in enumerate(dims(img))
        j = wcsax(img, dim)
        parentcoords_prepared[j,:] .= parentcoords[i,:] .- 1
    end
    for dim in refdims(img)
        j = wcsax(img, dim)
        # Non numeric reference dims can be used, e.g. a polarization axis of symbols I, Q, U, etc.
        if eltype(dim) <: Number
            z = dim[1] - 1
        else
            # Find the index of the symbol into the parent cube
            parentrefdim = img.wcsdims[findfirst(d->name(d)==name(dim), img.wcsdims)]
            z = findfirst(==(first(dim)), collect(parentrefdim)) - 1
        end
        parentcoords_prepared[j,:] .= z
    end

    # Get world coordinates along all slices
    WCS.pix_to_world!(wcs(img, wcsn), parentcoords_prepared, worldcoords_out)

    # If user requested world coordinates in all dims, not just selected
    # dims of img
    if all
        return worldcoords_out
    end

    # Otherwise filter to only return coordinates along selected dims.
    if ndims(pixcoords_prepared) > 1
        world_coords_of_these_axes = zeros(length(dims(img)), size(pixcoords_prepared,2))
    else
        world_coords_of_these_axes = zeros(length(dims(img)))
    end
    for (i, dim) in enumerate(dims(img))
        j = wcsax(img, dim)
        world_coords_of_these_axes[i,:] .= worldcoords_out[j,:]
    end

    return world_coords_of_these_axes
end

##
function WCS.world_to_pix(img::AstroImage, worldcoords; parent=false, wcsn=1)
    if worldcoords isa Array{Float64}
        worldcoords_prepared = worldcoords
    else
        worldcoords_prepared = [Float64(c) for c in worldcoords]
    end
    D_out = wcs(img,wcsn).naxis
    if ndims(worldcoords_prepared) > 1
        out = similar(worldcoords_prepared, Float64, D_out, size(worldcoords_prepared,2))
    else
        out = similar(worldcoords_prepared, Float64, D_out)
    end
    return WCS.world_to_pix!(out, img, worldcoords_prepared; parent)
end
function WCS.world_to_pix!(pixcoords_out, img::AstroImage, worldcoords; wcsn=1, parent=false)
    # # Find the coordinates in the parent array.
    # # Dimensional data
    # worldcoords_floored = floor.(Int, worldcoords)
    # worldcoords_frac = (worldcoords .- worldcoords_floored) .* step.(dims(img))
    # parentcoords = getindex.(dims(img), worldcoords_floored) .+ worldcoords_frac
    # WCS.jl is very restrictive. We need to supply a Vector{Float64}
    # as input, not any other kind of collection.
    # TODO: avoid allocation in case where refdims=() and worldcoords isa Array{Float64}
    if ndims(worldcoords) > 1
        worldcoords_prepared = zeros(wcs(img,wcsn).naxis,size(worldcoords,2))
    else
        worldcoords_prepared = zeros(wcs(img,wcsn).naxis)
    end
    # TODO: we need to pass in ref dims locations as well, and then filter the
    # output to only include the dims of the current slice?
    # out = zeros(Float64, wcs(img,wcsn).naxis, size(worldcoords,2))
    for (i, dim) in enumerate(dims(img))
        j = wcsax(img, dim)
        worldcoords_prepared[j,:] = worldcoords[i,:]
    end
    for dim in refdims(img)
        j = wcsax(img, dim)
        # Non numeric reference dims can be used, e.g. a polarization axis of symbols I, Q, U, etc.
        if eltype(dim) <: Number
            z = dim[1]
        else
            # Find the index of the symbol into the parent cube
            parentrefdim = img.wcsdims[findfirst(d->name(d)==name(dim), img.wcsdims)]
            z = findfirst(==(first(dim)),collect(parentrefdim)) -1
        end
        worldcoords_prepared[j,:] .= z
    end

    # This returns the parent pixel coordinates.
    # TODO: switch to non-allocating version.
    pixcoords_out .= WCS.world_to_pix(wcs(img, wcsn), worldcoords_prepared)

    if !parent
        coordoffsets = zeros(wcs(img,wcsn).naxis)
        coordsteps = zeros(wcs(img,wcsn).naxis)
        for (i, dim) in enumerate(dims(img))
            j = wcsax(img, dim)
            coordoffsets[j] = first(dims(img)[i])
            coordsteps[j] = step(dims(img)[i])
        end
        for dim in refdims(img)
            j = wcsax(img, dim)
            # Non numeric reference dims can be used, e.g. a polarization axis of symbols I, Q, U, etc.
            if eltype(dim) <: Number
                coordoffsets[j] = first(dim)
                coordsteps[j] = step(dim)
            else
                # Find the index of the symbol into the parent cube
                parentrefdim = img.wcsdims[findfirst(d->name(d)==name(dim), img.wcsdims)]
                z = findfirst(==(first(dim)),collect(parentrefdim))
                coordoffsets[j] = z - 1
                coordsteps[j] = 1
            end
        end

        pixcoords_out .-= coordoffsets
        pixcoords_out .= (pixcoords_out .+ 1) ./ coordsteps
    end
    return pixcoords_out
end

## For now, we use a copied version of FITSIO's show method for FITSHeader.
# We have to be careful to format things in a way WCSLib will like.
# In particular, we can't put newlines after each 80 characters.
# FITSIO has to do this so users can see the header.

# functions for displaying header values in show(io, header)
hdrval_repr(v::Bool) = v ? "T" : "F"
hdrval_repr(v::String) = @sprintf "'%-8s'" v
hdrval_repr(v::Union{AbstractFloat, Integer}) = string(v)

function serializeheader(io, hdr::FITSHeader)
    n = length(hdr)
    for i=1:n
        if hdr.keys[i] == "COMMENT" || hdr.keys[i] == "HISTORY"
                lastc = min(72, lastindex(hdr.comments[i]))
                @printf io "%s %s" hdr.keys[i] hdr.comments[i][1:lastc]
                print(io, " "^(72-lastc))
        else
            @printf io "%-8s" hdr.keys[i]
            if hdr.values[i] === nothing
                print(io, "                      ")
                rc = 50  # remaining characters on line
            elseif hdr.values[i] isa String
                val = hdrval_repr(hdr.values[i])
                @printf io "= %-20s" val
                rc = length(val) <= 20 ? 50 : 70 - length(val)
            else
                val = hdrval_repr(hdr.values[i])
                @printf io "= %20s" val
                rc = length(val) <= 20 ? 50 : 70 - length(val)
            end
            if length(hdr.comments[i]) > 0
                lastc = min(rc-3, lastindex(hdr.comments[i]))
                @printf io " / %s" hdr.comments[i][1:lastc]
                rc -= lastc + 3
            end
            print(io, " "^rc)
        end
        if i == n
            print(io, "\nEND"*(" "^77))
        else
            print(io)
        end
    end
end


module TestEAGO

import EAGO
using MathOptInterface
using Test

const MOI = MathOptInterface
const OPTIMIZER = MOI.instantiate(MOI.OptimizerWithAttributes(EAGO.Optimizer, MOI.Silent() => true))
const BRIDGED = MOI.instantiate(MOI.OptimizerWithAttributes(EAGO.Optimizer, MOI.Silent() => true), with_bridge_type = Float64)
const CONFIG = MOI.Test.Config(atol = 1e-3, rtol = 1e-3, optimal_status = MOI.OPTIMAL,
                               exclude = Any[MOI.DualObjectiveValue, MOI.ConstraintBasisStatus, MOI.VariableName, MOI.ConstraintName, MOI.delete,
                                             MOI.ConstraintDual, MOI.ListOfModelAttributesSet, MOI.add_constrained_variables])

"""
    runtests()

This function runs all functions in the this Module starting with `test_`.
"""
function runtests()
    for name in names(@__MODULE__; all = true)
        if startswith("$(name)", "test_")
            @testset "$(name)" begin
                getfield(@__MODULE__, name)()
            end
        end
    end
end

"""
    test_runtests()

This function runs all the tests in MathOptInterface.Test.

Pass arguments to `exclude` to skip tests for functionality that is not
implemented or that your solver doesn't support.
"""
function test_runtests()
    MOI.Test.runtests(BRIDGED, CONFIG,
                      exclude = [# IPOPT Inherited test exclusions
                                "test_model_ScalarFunctionConstantNotZero",
                                "test_solve_TerminationStatus_DUAL_INFEASIBLE",
                                "test_linear_VectorAffineFunction_empty_row",
                                "test_solve_DualStatus_INFEASIBILITY_CERTIFICATE_",
                                "test_model_LowerBoundAlreadySet",
                                "test_model_UpperBoundAlreadySet",
                                "test_model_copy_to_UnsupportedAttribute",
                                "test_model_copy_to_UnsupportedConstraint",
                                "test_objective_set_via_modify",
                                "test_conic_linear_VectorOfVariables_2",

                                # Cbc default test exclusions
                                "test_linear_Indicator_",
                                "test_linear_SOS1_integration",
                                "test_linear_SOS2_integration",
                                "test_solve_SOS2_add_and_delete",
                                "test_conic_NormInfinityCone_INFEASIBLE",
                                "test_conic_NormOneCone_INFEASIBLE",
                                "test_solve_TerminationStatus_DUAL_INFEASIBLE",

                                # EAGO test exclusions
                                "test_attribute_NumberOfThreads",
                                "test_modification_",
                                "test_linear_integration_delete_variables",

                                # EAGO Exclusions to Resolve (by adding conic support later and fixing twice solve issues)
                                "conic_NormOneCone_VectorAffineFunction",
                                "conic_NormOneCone_VectorOfVariables",
                                "conic_NormInfinityCone_VectorOfVariables",
                                "conic_NormInfinityCone_VectorAffineFunction",
                                "test_conic_NormInfinityCone_3",
                                "test_conic_NormOneCone",
                                "test_conic_linear_VectorOfVariables",
                                "conic_linear_VectorAffineFunction",
                                "conic_linear_VectorAffineFunction_2",

                                "linear_integer_solve_twice",
                                "linear_integration",

                                "test_quadratic_SecondOrderCone_basic",
                                "test_quadratic_constraint_GreaterThan",
                                "test_quadratic_constraint_LessThan",
                                "test_quadratic_constraint_minimize",
                                "test_quadratic_duplicate_terms",
                                "test_quadratic_integration",
                                "test_quadratic_nonconvex_constraint_integration",
                                "test_quadratic_homogeneous",
                                "test_quadratic_nonhomogeneous",
                                "test_quadratic_constraint_integration",
                                "test_quadratic_constraint_basic",

                                "test_constraint_qcp_duplicate_diagonal",
                                "test_constraint_qcp_duplicate_off_diagonal",
                                "test_objective_get_ObjectiveFunction_ScalarAffineFunction",
                                "test_objective_qp_ObjectiveFunction_edge_cases",
                                "test_objective_qp_ObjectiveFunction_zero_ofdiag",

                                "test_model_ModelFilter_ListOfConstraintIndices",
                                "test_model_ModelFilter_ListOfConstraintTypesPresent"

                                ],
                      exclude_tests_after = v"0.10.5")
end

end

@testset "MOI" begin
    TestEAGO.runtests()
end


#graphs of assets

function formpreliminaryfunds()

  local funds::DataFrame

  #form the data to grph
  funds = readprocessfundsdata()
  series = aggregatefunds(funds)

  PRELIM_STYLE_IDX::Dict = Dict(:textsize => 12.0,
      :linesize => 0.5,
      :shapesize => 0.25,
      #:xdomain => [1926, 2018],
      :width => 400,
      :height => 300,
      :widthscatter => 1600,
      :heightscatter => 1200,
      :xaxisname=>"Year",
      :legendtextsize => 9.0,
      :xaxisscattername=>"10ˣ_assets",
      :pointsize=>8.0,
      #:ydomain => [-1.0,2.0]#,
      #=:seriescolor => :black=#)

  #println(describe(series))
  graphfundassets(series,
    styleidx=PRELIM_STYLE_IDX,
    graphname="fundassets",
    yaxisname = "Assets")

  graphscatterfunds(funds,
    styleidx=PRELIM_STYLE_IDX,
    graphname="scattercash",
    yaxisname = "% cash",
    Fy = :wpercash)

    graphscatterfunds(funds,
      styleidx=PRELIM_STYLE_IDX,
      graphname="scattereq",
      yaxisname = "% eq",
      Fy = :wpercom)

end

function graphfundassets(
  series::AbstractDataFrame;
  outputfolder::String = PARAM[:figurepath],
  styleidx::Dict = error("styleidx is required"),
  Fx=:date,
  Fy=:value,
  Fgroup=:variable,
  graphname::String = error("graph name is required"),
  yaxisname::String = error("y-axis name is required"))

 #  println(series)

  panel = @vlplot() +
    @vlplot( #first part contains the line and circle dots
      data = series,
      width=styleidx[:width],
      height=styleidx[:height],
      title=nothing,
      mark={:line,
        clip=true,
        size=styleidx[:linesize]},
      x={Fx,
        axis={title=styleidx[:xaxisname],
          grid=false,
          titleFontSize=styleidx[:textsize],
          tickCount=11,
          titleFontWeight=:normal,
          labels=true},
        type=:temporal,
        },
      y={Fy,
        axis={title=yaxisname,
            grid=false,
            tickCount=4,
            titleFontSize=styleidx[:textsize],
            titleFontWeight=:normal},
          type=:quantitative,
          scale={type=:log},
        },
      color={Fgroup,
        type=:nominal,
        legend = {
          title=nothing,
          orient="bottom-right"
          }
        }
      )

  save("$outputfolder\\$graphname.pdf", panel)
end

#read in the appropriate file
function readprocessfundsdata(;prelimfundspath::String = "$(PARAM[:preliminarypath])\\$(PARAM[:prelimfunddata])",
  prelimfundretspath::String = "$(PARAM[:preliminarypath])\\$(PARAM[:prelimfundretsdata])")

  local altwords::Vector{String} = PARAM[:prelimaltwords]
  local funds::DataFrame
  local fundrets::DataFrame

  #scrubs the names and dates
  function initialclean(df::DataFrame)
    rename!(df, (cleanname).(names(df)))
    df.date = (i::Int->Date("$i", PARAM[:wrdsdateformat])).(df.caldt)
    df.year = (d->year(d)).(df.date)

    return df
  end

  if PARAM[:refreshpreliminaryfunds]
    #first get the summery data
    funds = CSV.read("$(prelimfundspath).csv") |> DataFrame
    funds = initialclean(funds)
    sort!(funds, [:year, :crspfundno, :date])
    funds.tokeep = trues(size(funds,1))
    #dedup funds

    for sfunds ∈ groupby(funds, [:year, :crspfundno])
      if size(sfunds,1) > 1
        sfunds.tokeep .= false
        sfunds.tokeep[end] = true
      end
    end
    funds = funds[funds.tokeep,:]
    #select!(funds, [:year, :crspfundno, :lipperclassname, :lipperobjname])
    size(unique(funds[!,[:year,:crspfundno]]),1) == size(funds,1) || error(
      "non-unique fund-years found! $(size(unique(funds[!,[:year,:crspfundno]]),1) - size(funds,1))")

    #put an indicator for if the fun is an alt
    funds.alt = falses(size(funds,1))
    funds.lipperclassname .= (s::MString->ismissing(s) ? missing : lowercase(s)).(funds.lipperclassname)
    funds.lipperobjname .= (s::MString->ismissing(s) ? missing : lowercase(s)).(funds.lipperobjname)

    for r ∈ eachrow(funds)
      r.alt = (!ismissing(r.lipperclassname)) && sum((s->occursin(s,r.lipperclassname)).(altwords)) > 0
      r.alt && continue
      r.alt = (!ismissing(r.lipperobjname)) && sum((s->occursin(s,r.lipperobjname)).(altwords)) > 0
    end

    #get the assets data
    fundrets = CSV.read("$(prelimfundretspath).csv") |> DataFrame
    fundrets = initialclean(fundrets)
    #println(describe(fundrets))
    fundrets.assets = (s::MString->
      ismissing(s) ? missing : parseormissing(Float64, s)).(fundrets.mtna)
    fundrets = fundrets[(!ismissing).(fundrets.assets),:]
    fundrets = fundrets[fundrets.assets .> 0,:]
    select!(fundrets, [:year, :assets, :crspfundno])

    #take the average of assets for each fund-year
    aggfunc(v::AbstractVector{<:MFloat64}) = mean(skipmissing(v))
    fundretss = groupby(fundrets, [:year, :crspfundno])
    fundrets = combine(fundretss, valuecols(fundretss) .=> aggfunc)
    rename!(fundrets, (s::String-> s=>nofunction(s, "_aggfunc")).(names(fundrets)))
    fundrets.date = (y->Date(y,12,31)).(fundrets.year)

    #merge the assets with the descriptive data
    rename!(funds, :date=>:rawdate)
    funds = innerjoin(funds, fundrets, on=[:year, :crspfundno])

    #winsorize percentage data
    @inline winper(::Missing, any...) = missing
    @inline winper(per::Real, lo::Real = -100., hi::Real = 100.) = min(hi, max(per, lo))

    funds.l10assets = (log10).(funds.assets .* 1_000_000.)
    funds.wpercash = (winper).(funds.percash)
    funds.wpercom = (x->winper(x,-100,200)).(funds.percom)

    serialize("$(prelimfundspath).jls", funds)
  else
    funds = deserialize("$(prelimfundspath).jls")
  end

    #display(funds[funds.crspfundno .== 25619,:])
  return funds
end

#adds the assets across funds
function aggregatefunds(funds::AbstractDataFrame)

  local series::DataFrame

  aggfunc(v::AbstractVector{<:MFloat64}) = sum(skipmissing(v))
  seriess::GroupedDataFrame = groupby(view(funds, :, [:date, :assets]), :date)
  series = combine(seriess, valuecols(seriess) .=> aggfunc)

  #fix the names
  rename!(series, (s::String-> s=>nofunction(s, "_aggfunc")).(names(series)))
  rename!(series, :assets=>:total_assets)

  #repeat, but this time only for alt funds
  sfunds = view(funds, funds.alt, [:date, :assets, :alt])
  sfundss::GroupedDataFrame = groupby(view(sfunds, :, [:date, :assets]), :date)
  seriesalts = combine(sfundss, valuecols(sfundss) .=> aggfunc)
  rename!(seriesalts, (s::String-> s=>nofunction(s, "_aggfunc")).(names(seriesalts)))
  rename!(seriesalts, :assets=>:alt_assets)

  #join and melt the two aggregatations
  series = leftjoin(series, seriesalts, on=:date)
  series = stack(series, Not(:date))
  #println(describe(series))

  return series
end

#makes a scatterplot of fund assets
function graphscatterfunds(
  funds::AbstractDataFrame;
  outputfolder::String = PARAM[:figurepath],
  styleidx::Dict = error("styleidx is required"),
  Fx=:l10assets,
  Fy=:error("Fy is required"),
  Fgroup=:variable,
  graphname::String = error("graph name is required"),
  yaxisname::String = error("y-axis name is required"))

  #println(series)

  panel = @vlplot() +
    @vlplot( #first part contains the line and circle dots
      data = funds,
      width=styleidx[:widthscatter],
      height=styleidx[:heightscatter],
      title=nothing,
      mark={:circle,
      size=styleidx[:pointsize]},
      x={Fx,
        axis={title=styleidx[:xaxisscattername],
          grid=false,
          titleFontSize=styleidx[:textsize],
          #tickCount=11,
          titleFontWeight=:normal,
          labels=true},
        type=:quantitative,
        scale={zero=false}
        },
      y={Fy,
        axis={title=yaxisname,
            grid=false,
#            tickCount=4,
            titleFontSize=styleidx[:textsize],
            titleFontWeight=:normal},
          type=:quantitative,
        },
      )

  save("$outputfolder\\$graphname.png", panel)
end


using ElasticArrays

Base.push!(a::ElasticArray, x) = append!(a, x)
Base.push!(a::ElasticArray{T,1}, x) where {T} = append!(a, [x])
Base.empty!(a::ElasticArray) = ElasticArrays.resize_lastdim!(a, 0)

function Base.pop!(a::ElasticArray)
    if length(a) > 0
        last_frame_inds = length(a.data)-a.kernel_length.divisor+1:length(a.data)
        d = reshape(view(a.data, last_frame_inds), a.kernel_size)
        ElasticArrays.resize!(a.data, length(a.data) - a.kernel_length.divisor)
        d
    else
        @error "can not pop! from an empty ElasticArray"
    end
end


# This file is a part of BAT.jl, licensed under the MIT License (MIT).

"""
    abstract type DifferentiationAlgorithm

*Experimental feature, not part of stable public API.*

Abstract type for integrated autocorrelation length estimation algorithms.
"""
abstract type DifferentiationAlgorithm end
export DifferentiationAlgorithm

function vjp_algorithm end
vjp_algorithm(f::Function) = ZygoteAD()

function jvp_algorithm end
jvp_algorithm(f::Function) = ForwardDiffAD()

"""
    valgradof(f::Function, algorithm::DifferentiationAlgorithm  = vjp_algorithm(f))

*Experimental feature, not part of stable public API.*

Generates a function that calculates both value and gradient of `f` at given
points. The differentiation algorithm used depends on `f`.

The function `f` must support `ValueShapes.varshape(f)` and
`ValueShapes.unshaped(f)`.

Returns function `valgrad_f` with

```julia
f_at_x, grad_of_f_at_x = valgrad_f(x)

grad_of_f_at_x = zero(x)
f_at_x = valgrad_f(!, grad_of_f_at_x, x)
```
"""
function valgradof end
export valgradof

abstract type GradientFunction <: Function end


module OceanTurbPyPlotUtils

export
    plot,
    removespine,
    removespines,
    cornerspines,
    bottomspine,

    defaultcolors,
    fontmanager,
    usecmbright

using OceanTurb, PyPlot, PyCall

# A few nice things for plotting
import PyPlot: plot

fontmanager = pyimport("matplotlib.font_manager")
defaultcolors = plt.rcParams["axes.prop_cycle"].by_key()["color"]

plot(f::AbstractField, args...; kwargs...) = plot(data(f), nodes(f), args...; kwargs...)
plot(op::Function, f::AbstractField, args...; kwargs...) = plot(op.(data(f)), nodes(f), args...; kwargs...)

latexpreamble = """
\\usepackage{cmbright}
\\renewcommand{\\b}[1]    {\\boldsymbol{#1}}
\\renewcommand{\\r}[1]    {\\mathrm{#1}}
\\renewcommand{\\d}       {\\partial}
"""

function usecmbright()
  rc("text.latex", preamble=latexpreamble)
  rc("font", family="sans-serif")
  nothing
end

"Remove `spine` from `ax`."
function removespine(side, ax=gca())
    ax.spines[side].set_visible(false)
    keywords = Dict(Symbol(side)=>false, Symbol(:label, side)=>false)
    ax.tick_params(keywords)
    nothing
end

removespines(sides...; ax=gca()) = for side in sides; removespine(side, ax); end
cornerspines(ax=gca()) = removespines("top", "right"; ax=ax)

function bottomspine(ax=gca())
    removespines("top", "right", "left"; ax=ax)
    ax.tick_params(left=false, labelleft=false)
end

end # module


function graft!(a, n, clr::Int, colors::Vararg{Int})
	graft!(a, n, clr)
	for c in colors
		graft!(a, n, c)
	end

	return nothing
end
function graft!(a, n, colors::Vector{Bool})
	for (i,c) in enumerate(colors)
		if c
			graft!(a, n, i)
		end
	end

	return nothing
end
function graft!(a::ARGNode, n::ARGNode, c::Int)
	# Check that `n` does not have an ancestor
	@assert !hasancestor(n, c)
	# Proceed: set `a` as ancestor of `n`
	set_ancestor!(n, a, c)
	# If `n` is not already in children of `a`, put it there
	if !haschild(a, n)
		add_child!(a, n)
	end

	return nothing
end

function graft!(a::RootNode, n::ARGNode, c::Int)
	@assert !hasancestor(n, c)
	set_ancestor!(n, a, c)

	return nothing
end

function set_root!(n::ARGNode, c::Int)
	graft!(RootNode(), n, c)
end

function unset_child!(a::ARGNode, n, clr)
	@assert ancestor(n, clr) == a
	@assert haschild(a, n)
	# Is a the ancestor of n for another color?
	# If yes, we're not doing anything
	for (i,c) in color(n)
		if c && (i != clr)
			return nothing
		end
	end
	# Else, remove n from the children of a
	deleteat!(findfirst(==(n), a.children))
end
unset_child!(a, n, clr) = nothing

function prune!(n::ARGNode, c::Int)
	a = ancestor(n, c)
	unset_child!(a, n, c)
	set_ancestor!(n, nothing, c)
end

# function set_global_root!(arg::ARG)
# 	gr = RootNode([])
# 	for (clr, r) in arg.roots
# 		@assert isroot(r, clr)
# 		prune!(r, clr)
# 		graft!(gr, r, clr)
# 	end
# end


#export makeAxisArray

# Support for handling complex datatypes in HDF5 files
function writeComplexArray(file, dataset, A::AbstractArray{Complex{T},D}) where {T,D}
  d_type_compound = HDF5.h5t_create(HDF5.H5T_COMPOUND,2*sizeof(T))
  HDF5.h5t_insert(d_type_compound, "r", 0 , HDF5.hdf5_type_id(T))
  HDF5.h5t_insert(d_type_compound, "i", sizeof(T) , HDF5.hdf5_type_id(T))

  shape = collect(reverse(size(A)))
  space = HDF5.h5s_create_simple(D, shape, shape)

  dset_compound = HDF5.h5d_create(file, dataset, d_type_compound, space,
                                  HDF5.H5P_DEFAULT,HDF5.H5P_DEFAULT,HDF5.H5P_DEFAULT)
  HDF5.h5s_close(space)

  HDF5.h5d_write(dset_compound, d_type_compound, HDF5.H5S_ALL, HDF5.H5S_ALL, HDF5.H5P_DEFAULT, A)

  HDF5.h5d_close(dset_compound)
  HDF5.h5t_close(d_type_compound)
end

function isComplexArray(file, dataset)
  if eltype(file[dataset]) <: Complex
    return true
  end

  # If complex number support in HDF5 is disabled (or version < 0.12.2)
  if eltype(file[dataset]) <: HDF5.HDF5Compound{2}
    if HDF5.h5t_get_member_name(datatype(file[dataset]).id,0) == "r" &&
      HDF5.h5t_get_member_name(datatype(file[dataset]).id,1) == "i"
      return true
    end
  end
  return false
end

function getComplexType(file, dataset)
  T = HDF5.hdf5_to_julia_eltype(
            HDF5Datatype(
              HDF5.h5t_get_member_type( datatype(file[dataset]).id, 0 )
          )
        )
    return Complex{T}
end

function readComplexArray(file::HDF5File, dataset)
  T = getComplexType(file, dataset)
  A = copy(readmmap(file[dataset],Array{getComplexType(file,dataset)}))
  return A
end

function readComplexArray(filename::String, dataset)
  h5open(filename, "r") do file
    return readComplexArray(file, dataset)
  end
end

function makeAxisArray(array::Array{T,5}, pixelspacing, offset, dt) where T
  N = size(array)
  im = AxisArray(array, Axis{:color}(1:N[1]),
		 Axis{:x}(range(offset[1],step=pixelspacing[1],length=N[2])),
		 Axis{:y}(range(offset[2],step=pixelspacing[2],length=N[3])),
		 Axis{:z}(range(offset[3],step=pixelspacing[3],length=N[4])),
		 Axis{:time}(range(0*unit(dt),step=dt,length=N[5])))
  return im
end

# RawFile removed see commit https://github.com/MagneticParticleImaging/MPIFiles.jl/commit/c2b49833cc00127770fb2e1f5342883f213ff002


export AbstractPDDL,
        PDDL, goal_test, execute_action,
        AbstractPlanningAction, PlanningAction,
        substitute, check_precondition,
        air_cargo_pddl, air_cargo_pddl_goal_test,
        spare_tire_pddl, spare_tire_pddl_goal_test,
        three_block_tower_pddl, three_block_tower_pddl_goal_test,
        have_cake_and_eat_cake_too_pddl, have_cake_and_eat_cake_too_pddl_goal_test,
        PlanningLevel,
        find_mutex_links, build_level_links, build_level_links_permute_arguments, perform_actions,
        planning_combinations,
        PlanningGraph, expand_graph, non_mutex_goals,
        GraphPlanProblem, check_level_off, extract_solution,
        graphplan,
        doubles_tennis_pddl, doubles_tennis_pddl_goal_test,
        PlanningHighLevelAction, check_resource, check_job_order,
        HighLevelPDDL, refinements, hierarchical_search,
        job_shop_scheduling_pddl, job_shop_scheduling_pddl_goal_test;

abstract type AbstractPDDL end;

abstract type AbstractPlanningAction end;

#=

    PlanningAction is an action schema defined by the action's name, preconditions, and effects.

    Preconditions and effects consists of either positive and negated literals.

=#
struct PlanningAction <: AbstractPlanningAction
    name::String
    arguments::Tuple
    precondition_positive::Array{Expression, 1}
    precondition_negated::Array{Expression, 1}
    effect_add_list::Array{Expression, 1}
    effect_delete_list::Array{Expression, 1}

    function PlanningAction(action::Expression, precondition::Tuple{Vararg{Array, 2}}, effect::Tuple{Vararg{Array, 2}})
        return new(action.operator, action.arguments, precondition[1], precondition[2], effect[1], effect[2]);
    end
end

function substitute(action::T, e::Expression, arguments::Tuple{Vararg{Expression}}) where {T <: AbstractPlanningAction}
    local new_arguments::AbstractVector = collect(e.arguments);
    for (index_1, argument) in enumerate(e.arguments)
        for index_2 in 1:length(action.arguments)
            if (action.arguments[index_2] == argument)
                new_arguments[index_1] = arguments[index_2];
            end
        end
    end
    return Expression(e.operator, Tuple((new_arguments...,)));
end

function check_precondition(action::T1, kb::T2, arguments::Tuple) where {T1 <: AbstractPlanningAction, T2 <: AbstractKnowledgeBase}
    # Check for positive clauses.
    for clause in action.precondition_positive
        if (!(substitute(action, clause, arguments) in kb.clauses))
            return false;
        end
    end
    # Check for negated clauses.
    for clause in action.precondition_negated
        if (substitute(action, clause, arguments) in kb.clauses)
            return false;
        end
    end
    return true;
end

"""
    execute_action(action, kb, arguments)

Execute the given action 'action' on the knowledge base of the planning problem.
"""
function execute_action(action::T1, kb::T2, arguments::Tuple) where {T1 <: AbstractPlanningAction, T2 <: AbstractKnowledgeBase}
    if (!(check_precondition(action, kb, arguments)))
        error(@sprintf("execute_action(): Action \"%s\" preconditions are not satisfied!", action.name));
    end
    # Retract negated literals to knowledge base 'kb'.
    for clause in action.effect_delete_list
        retract(kb, substitute(action, clause, arguments));
    end
    # Add positive literals to knowledge base 'kb'.
    for clause in action.effect_add_list
        tell(kb, substitute(action, clause, arguments));
    end
    nothing;
end

#=

    The Planning Domain Definition Language (PDDL) is used to define a search problem.

    The states (starting from the initial state) are represented as the conjunction of

    the statements in 'kb' (a FirstOrderLogicKnowledgeBase). The actions are described

    by 'actions' (an array of action schemas). The 'goal_test' is a function that checks

    if the current state of the problem is at the goal state.

=#
struct PDDL <: AbstractPDDL
    kb::FirstOrderLogicKnowledgeBase
    actions::Array{PlanningAction, 1}
    goal_test::Function

    function PDDL(initial_state::Array{Expression, 1}, actions::Array{PlanningAction, 1}, goal_test::Function)
        return new(FirstOrderLogicKnowledgeBase(initial_state), actions, goal_test);
    end
end

function goal_test(plan::T) where {T <: AbstractPDDL}
    return plan.goal_test(plan.kb);
end

"""
    execute_action(plan, action)

Execute the first relevant PlanningAction associated with action 'action'
on the given planning problem's knowledgebase.
"""
function execute_action(plan::T, action::Expression) where {T <: AbstractPDDL}
    local action_name::String = action.operator;
    local arguments::Tuple = action.arguments;
    local relevant_actions::AbstractVector = collect(a for a in plan.actions if (a.name == action_name));
    if (length(relevant_actions) == 0)
        error(@sprintf("execute_action(): Action \"%s\" not found!", action_name));
    else
        local first_relevant_action::PlanningAction = relevant_actions[1];
        if (!check_precondition(first_relevant_action, plan.kb, arguments))
            error(@sprintf("execute_action(): Action \"%s\" preconditions are not satisfied!", repr(action)));
        else
            execute_action(first_relevant_action, plan.kb, arguments);
        end
    end
    nothing;
end

function air_cargo_pddl_goal_test(kb::FirstOrderLogicKnowledgeBase)
    return all((function(ans)
                    if (typeof(ans) <: Bool)
                        return ans;
                    else
                        if (length(ans) == 0)   # length of Tuple
                            return false;
                        else
                            return true;
                        end
                    end
                end),
                collect(ask(kb, q) for q in (expr("At(C1, JFK)"), expr("At(C2, SFO)"))));
end

"""
    air_cargo_pddl()

Return a PDDL representing the air cargo transportation planning problem (Fig. 10.1).
"""
function air_cargo_pddl()
    local initial::Array{Expression, 1} = map(expr, ["At(C1, SFO)",
                                                "At(C2, JFK)",
                                                "At(P1, SFO)",
                                                "At(P2, JFK)",
                                                "Cargo(C1)",
                                                "Cargo(C2)",
                                                "Plane(P1)",
                                                "Plane(P2)",
                                                "Airport(JFK)",
                                                "Airport(SFO)"]);
    # Load Action Schema
    local precondition_positive::Array{Expression, 1} = map(expr, ["At(c, a)",
                                                            "At(p, a)",
                                                            "Cargo(c)",
                                                            "Plane(p)",
                                                            "Airport(a)"]);
    local precondition_negated::Array{Expression, 1} = [];
    local effect_add_list::Array{Expression, 1} = [expr("In(c, p)")];
    local effect_delete_list::Array{Expression, 1} = [expr("At(c, a)")];
    local load::PlanningAction = PlanningAction(expr("Load(c, p, a)"),
                                                (precondition_positive, precondition_negated),
                                                (effect_add_list, effect_delete_list));
    # Unload Action Schema
    precondition_positive = map(expr, ["In(c, p)", "At(p, a)", "Cargo(c)", "Plane(p)", "Airport(a)"]);
    precondition_negated = [];
    effect_add_list = [expr("At(c, a)")];
    effect_delete_list = [expr("In(c, p)")];
    local unload::PlanningAction = PlanningAction(expr("Unload(c, p, a)"),
                                                (precondition_positive, precondition_negated),
                                                (effect_add_list, effect_delete_list));
    # Fly Action Schema
    precondition_positive = map(expr, ["At(p, f)", "Plane(p)", "Airport(f)", "Airport(to)"]);
    precondition_negated = [];
    effect_add_list = [expr("At(p, to)")];
    effect_delete_list = [expr("At(p, f)")];
    local fly::PlanningAction = PlanningAction(expr("Fly(p, f, to)"),
                                                (precondition_positive, precondition_negated),
                                                (effect_add_list, effect_delete_list));
    return PDDL(initial, [load, unload, fly], air_cargo_pddl_goal_test);
end

function spare_tire_pddl_goal_test(kb::FirstOrderLogicKnowledgeBase)
    return all((function(ans)
                    if (typeof(ans) <: Bool)
                        return ans;
                    else
                        if (length(ans) == 0)   # length of Tuple
                            return false;
                        else
                            return true;
                        end
                    end
                end),
                collect(ask(kb, q) for q in (expr("At(Spare, Axle)"),)));
end

"""
    spare_tire_pddl()

Return a PDDL representing the spare tire planning problem (Fig. 10.2).
"""
function spare_tire_pddl()
    local initial::Array{Expression, 1} = map(expr, ["Tire(Flat)",
                                                    "Tire(Spare)",
                                                    "At(Flat, Axle)",
                                                    "At(Spare, Trunk)"]);
    # Remove Action Schema
    local precondition_positive::Array{Expression, 1} = [expr("At(obj, loc)")];
    local precondition_negated::Array{Expression, 1} = [];
    local effect_add_list::Array{Expression, 1} = [expr("At(obj, Ground)")];
    local effect_delete_list::Array{Expression, 1} = [expr("At(obj, loc)")];
    local remove::PlanningAction = PlanningAction(expr("Remove(obj, loc)"),
                                                (precondition_positive, precondition_negated),
                                                (effect_add_list, effect_delete_list));
    # PutOn Action Schema
    precondition_positive = map(expr, ["Tire(t)", "At(t, Ground)"]);
    precondition_negated = [expr("At(Flat, Axle)")];
    effect_add_list = [expr("At(t, Axle)")];
    effect_delete_list = [expr("At(t, Ground)")];
    local put_on::PlanningAction = PlanningAction(expr("PutOn(t, Axle)"),
                                                    (precondition_positive, precondition_negated),
                                                    (effect_add_list, effect_delete_list));
    # LeaveOvernight Action Schema
    precondition_positive = [];
    precondition_negated = [];
    effect_add_list = [];
    effect_delete_list = map(expr, ["At(Spare, Ground)", "At(Spare, Axle)", "At(Spare, Trunk)",
                                    "At(Flat, Ground)", "At(Flat, Axle)", "At(Flat, Trunk)"]);
    local leave_overnight::PlanningAction = PlanningAction(expr("LeaveOvernight"),
                                                            (precondition_positive, precondition_negated),
                                                            (effect_add_list, effect_delete_list));
    return PDDL(initial, [remove, put_on, leave_overnight], spare_tire_pddl_goal_test);
end

function three_block_tower_pddl_goal_test(kb::FirstOrderLogicKnowledgeBase)
    return all((function(ans)
                    if (typeof(ans) <: Bool)
                        return ans;
                    else
                        if (length(ans) == 0)   #length of Tuple
                            return false;
                        else
                            return true;
                        end
                    end
                end),
                collect(ask(kb, q) for q in (expr("On(A, B)"), expr("On(B, C)"))));
end

"""
    three_block_tower_pddl()

Return a PDDL representing the building of a three-block tower planning problem (Fig. 10.3).
"""
function three_block_tower_pddl()
    local initial::Array{Expression, 1} = map(expr, ["On(A, Table)",
                                                    "On(B, Table)",
                                                    "On(C, A)",
                                                    "Block(A)",
                                                    "Block(B)",
                                                    "Block(C)",
                                                    "Clear(B)",
                                                    "Clear(C)"]);
    # Move Action Schema
    local precondition_positive::Array{Expression, 1} = map(expr, ["On(b, x)", "Clear(b)", "Clear(y)", "Block(b)", "Block(y)"]);
    local precondition_negated::Array{Expression, 1} = [];
    local effect_add_list::Array{Expression, 1} = [expr("On(b, y)"), expr("Clear(x)")];
    local effect_delete_list::Array{Expression, 1} = [expr("On(b, x)"), expr("Clear(y)")];
    local move::PlanningAction = PlanningAction(expr("Move(b, x, y)"),
                                                (precondition_positive, precondition_negated),
                                                (effect_add_list, effect_delete_list));
    # MoveToTable Action Schema
    precondition_positive = map(expr, ["On(b, x)", "Clear(b)", "Block(b)"]);
    precondition_negated = [];
    effect_add_list = [expr("On(b, Table)"), expr("Clear(x)")];
    effect_delete_list = [expr("On(b, x)")];
    local move_to_table::PlanningAction = PlanningAction(expr("MoveToTable(b, x)"),
                                                        (precondition_positive, precondition_negated),
                                                        (effect_add_list, effect_delete_list));
    return PDDL(initial, [move, move_to_table], three_block_tower_pddl_goal_test);
end

function have_cake_and_eat_cake_too_pddl_goal_test(kb::FirstOrderLogicKnowledgeBase)
    return all((function(ans)
                    if (typeof(ans) <: Bool)
                        return ans;
                    else
                        if (length(ans) == 0)   # length of Tuple
                            return false;
                        else
                            return true;
                        end
                    end
                end),
                collect(ask(kb, q) for q in (expr("Have(Cake)"), expr("Eaten(Cake)"))));
end

"""
    have_cake_and_eat_cake_too_pddl()

Return a PDDL representing the 'have cake and eat cake too' planning problem (Fig. 10.7).
"""
function have_cake_and_eat_cake_too_pddl()
    local initial::Array{Expression, 1} = [expr("Have(Cake)")];
    # Eat Cake Action Schema
    local precondition_positive::Array{Expression, 1} = [expr("Have(Cake)")];
    local precondition_negated::Array{Expression, 1} = [];
    local effect_add_list::Array{Expression, 1} = [expr("Eaten(Cake)")];
    local effect_delete_list::Array{Expression, 1} = [expr("Have(Cake)")];
    local eat_cake::PlanningAction = PlanningAction(expr("Eat(Cake)"),
                                                    (precondition_positive, precondition_negated),
                                                    (effect_add_list, effect_delete_list));
    # Bake Cake Action Schema
    precondition_positive = [];
    precondition_negated = [expr("Have(Cake)")];
    effect_add_list = [expr("Have(Cake)")];
    effect_delete_list = [];
    local bake_cake::PlanningAction = PlanningAction(expr("Bake(Cake)"),
                                                    (precondition_positive, precondition_negated),
                                                    (effect_add_list, effect_delete_list));
    return PDDL(initial, [eat_cake, bake_cake], have_cake_and_eat_cake_too_pddl_goal_test);
end

mutable struct PlanningLevel
    positive_kb::FirstOrderLogicKnowledgeBase
    current_state_positive::Array{Expression, 1}    #current state of the planning problem
    current_state_negated::Array{Expression, 1}     #current state of the planning problem
    current_action_links_positive::Dict             #current actions to current state link
    current_action_links_negated::Dict              #current actions to current state link
    current_state_links_positive::Dict              #current state to action link
    current_state_links_negated::Dict               #current state to action link
    next_action_links::Dict                         #current action to next state link
    next_state_links_positive::Dict                 #next state to current action link
    next_state_links_negated::Dict                  #next state to current action link
    mutex_links::Array{Set, 1}                      #each mutex relation is a Set of 2 actions/literals

    function PlanningLevel(p_kb::FirstOrderLogicKnowledgeBase, n_kb::FirstOrderLogicKnowledgeBase)
        return new(p_kb, p_kb.clauses, n_kb.clauses, Dict(), Dict(), Dict(), Dict(), Dict(), Dict(), Dict(), []);
    end
end

function find_mutex_links(level::PlanningLevel)
    # Inconsistent effects condition between 2 action schemas at a given level
    for positive_effect in level.next_state_links_positive
        negated_effect = positive_effect;
        if (haskey(level.next_state_links_negated, negated_effect))
            for a in level.next_state_links_positive[positive_effect]
                for b in level.next_state_links_negated[negated_effect]
                    if (!(Set([a, b]) in level.mutex_links))
                        push!(level.mutex_links, Set([a, b]));
                    end
                end
            end
        end
    end
    # Inference condition between 2 action schemas at a given level
    for positive_precondition in level.current_state_links_positive
        negated_effect = positive_precondition;
        if (haskey(level.next_state_links_negated, negated_effect))
            for a in level.current_state_links_positive[positive_precondition]
                for b in level.next_state_links_negated[negated_effect]
                    if (!(Set([a, b]) in level.mutex_links))
                        push!(level.mutex_links, Set([a, b]));
                    end
                end
            end
        end
    end
    for negated_precondition in level.current_state_links_negated
        positive_effect = negated_precondition;
        if (haskey(level.next_state_links_positive, positive_effect))
            for a in level.next_state_links_positive[positive_effect]
                for b in level.current_state_links_negated[negated_precondition]
                    if (!(Set([a, b]) in level.mutex_links))
                        push!(level.mutex_links, Set([a, b]));
                    end
                end
            end
        end
    end
    # Competing needs condition between 2 action schemas
    for positive_precondition in level. current_state_links_positive
        negated_precondition = positive_precondition;
        if (haskey(level.current_state_links_negated, negated_precondition))
            for a in level.current_state_links_positive[positive_precondition]
                for b in level.current_state_links_negated[negated_precondition]
                    if (!(Set([a, b]) in level.mutex_links))
                        push!(level.mutex_links, Set([a, b]));
                    end
                end
            end
        end
    end
    # Inconsistent support condition
    local state_mutex_links::AbstractVector = [];
    for pair in level.mutex_links
        collected_pair::AbstractVector = collect(pair);
        next_state_1 = level.next_action_links[collected_pair[1]];
        if (length(sorted_pair) == 2)
            next_state_2 = level.next_action_links[collected_pair[2]];
        else
            next_state_2 = level.next_action_links[collected_pair[1]];
        end
        if ((length(next_state_1) == 1) && (length(next_state_2) == 1))
            push!(state_mutex_links, Set([next_state_1[1], next_state_2[1]]));
        end
    end
    level.mutex_links = vcat(level.mutex_links, state_mutex_links);
    nothing;
end

function build_level_links_permute_arguments(depth::Int64, objects::AbstractVector, current_permutation::Tuple, permutations_array::AbstractVector)
    if (depth == 0)
        push!(permutations_array, current_permutation);
    elseif (depth < 0)
        error("build_level_links_permute_arguments(): Found negative depth!");
    else
        for (i, item) in enumerate(objects)
            build_level_links_permute_arguments((depth - 1),
                                                Tuple((objects[1:(i - 1)]..., objects[(i + 1):end]...,)),
                                                Tuple((current_permutation..., item)),
                                                permutations_array)
        end
    end
end

function build_level_links_permute_arguments(depth::Int64, objects::Tuple, current_permutation::Tuple, permutations_array::AbstractVector)
    if (depth == 0)
        push!(permutations_array, current_permutation);
    elseif (depth < 0)
        error("build_level_links_permute_arguments(): Found negative depth!");
    else
        for (i, item) in enumerate(objects)
            build_level_links_permute_arguments((depth - 1),
                                                Tuple((objects[1:(i - 1)]..., objects[(i + 1):end]...,)),
                                                Tuple((current_permutation..., item)),
                                                permutations_array)
        end
    end
end

function build_level_links(level::PlanningLevel, actions::AbstractVector, objects::Set)
    # Create persistence actions for positive states
    for clause in level.current_state_positive
        level.current_action_links_positive[Expression("Persistence", clause)] = [clause];
        level.next_action_links[Expression("Persistence", clause)] = [clause];
        level.current_state_links_positive[clause] = [Expression("Persistence", clause)];
        level.next_state_links_positive[clause] = [Expression("Persistence", clause)];
    end
    # Create persistence actions for negated states
    for clause in level.current_state_negated
        not_expression = Expression("not"*clause.operator, clause.arguments);
        level.current_action_links_negated[Expression("Persistence", not_expression)] = [clause];
        level.next_action_links[Expression("Persistence", not_expression)] = [clause];
        level.current_state_links_negated[clause] = [Expression("Persistence", not_expression)];
        level.next_state_links_negated[clause] = [Expression("Persistence", not_expression)];
    end
    # Recursively collect num_arg depth, collecting a Tuple of Tuples
    for action in actions
        local num_arguments::Int64 = length(action.arguments);
        local possible_arguments::AbstractVector = [];
        build_level_links_permute_arguments(num_arguments, collect(objects), (), possible_arguments);
        for argument in possible_arguments
            if (check_precondition(action, level.positive_kb, argument))
                for (number, symbol) in enumerate(action.arguments)
                    if (!all(islowercase, symbol.operator))
                        argument = Tuple((argument[1:(number - 1)]..., symbol, argument[(number + 1):end]...,));
                    end
                end
                local new_action::Expression = substitute(action, Expression(action.name, action.arguments), argument);
                level.current_action_links_positive[new_action] = [];
                level.current_action_links_negated[new_action] = [];
                local new_clause::Expression;
                for clause in action.precondition_positive
                    new_clause = substitute(action, clause, argument);
                    push!(level.current_action_links_positive[new_action], new_clause);
                    if (haskey(level.current_state_links_positive, new_clause))
                        push!(level.current_state_links_positive[new_clause], new_action);
                    else
                        level.current_state_links_positive[new_clause] = [new_action];
                    end
                end
                for clause in action.precondition_negated
                    new_clause = substitute(action, clause, argument);
                    push!(level.current_action_links_negated[new_action], new_clause);
                    if (haskey(level.current_state_links_negated, new_clause))
                        push!(level.current_state_links_negated[new_clause], new_action);
                    else
                        level.current_state_links_negated[new_clause] = [new_action];
                    end
                end
                level.next_action_links[new_action] = [];
                for clause in action.effect_add_list
                    new_clause = substitute(action, clause, argument);
                    push!(level.next_action_links[new_action], new_clause);
                    if (haskey(level.next_state_links_positive, new_clause))
                        push!(level.next_state_links_positive[new_clause], new_action);
                    else
                        level.next_state_links_positive[new_clause] = [new_action];
                    end
                end
                for clause in action.effect_delete_list
                    new_clause = substitute(action, clause, argument);
                    push!(level.next_action_links[new_action], new_clause);
                    if (haskey(level.next_state_links_negated, new_clause))
                        push!(level.next_state_links_negated[new_clause], new_action);
                    else
                        level.next_state_links_negated[new_clause] = [new_action];
                    end
                end
            end
        end
    end
    nothing;
end

function perform_actions(level::PlanningLevel)
    local new_kb_positive::FirstOrderLogicKnowledgeBase = FirstOrderLogicKnowledgeBase(Array{Expression, 1}(collect(Set(collect(keys(level.next_state_links_positive))))));
    local new_kb_negated::FirstOrderLogicKnowledgeBase = FirstOrderLogicKnowledgeBase(Array{Expression, 1}(collect(Set(collect(keys(level.next_state_links_negated))))));
    return PlanningLevel(new_kb_positive, new_kb_negated);
end

function planning_combinations(items::AbstractVector) #ordered permutations of length 2
    local combinations::AbstractVector = [];
    for (i, a) in enumerate(items)
        for b in items[(i + 1):end]
            push!(combinations, (a, b));
        end
    end
    return combinations;
end

#=

    PlanningGraph is an implementation of a planning graph data structure.

    The planning graph is organized into levels based on the given PDDL

    and negated knowledge base.

=#
struct PlanningGraph
    pddl::AbstractPDDL
    levels::Array{PlanningLevel, 1}
    objects::Set{Expression}

    function PlanningGraph(pddl::T, n_kb::FirstOrderLogicKnowledgeBase) where {T <: AbstractPDDL}
        return new(pddl, [PlanningLevel(pddl.kb, n_kb)], Set(collect(arg for clause in vcat(pddl.kb.clauses, n_kb.clauses) for arg in clause.arguments)));
    end
end

function expand_graph(pg::PlanningGraph)
    local last_level = pg.levels[length(pg.levels)];
    build_level_links(last_level, pg.pddl.actions, pg.objects);
    find_mutex_links(last_level);
    push!(pg.levels, perform_actions(last_level));
    nothing;
end

function non_mutex_goals(pg::PlanningGraph, goals::AbstractVector, index::Int64)
    local goal_combinations::AbstractVector = planning_combinations(goals);
    for goal in goal_combinations
        if (index < 0)
            if (Set(collect(goal)) in reverse(pg.levels)[abs(index)].mutex_links)
                return false;
            end
        else
            error("non_mutex_goals(): Expected negative index, got ", index, "!");
        end
    end
    return true;
end

#=

    GraphPlanProblem is a data structure that stores the 'graph', 'nogoods',

    and 'solution' variables of a given Graphplan planning problem.

=#
struct GraphPlanProblem
    graph::PlanningGraph
    nogoods::AbstractVector
    solution::AbstractVector

    function GraphPlanProblem(pddl::AbstractPDDL, n_kb::FirstOrderLogicKnowledgeBase)
        return new(PlanningGraph(pddl, n_kb), [], []);
    end
end

function check_level_off(gpp::GraphPlanProblem)
    local first_check::Bool = (Set(reverse(gpp.graph.levels)[1].current_state_positive) == Set(reverse(gpp.graph.levels)[2].current_state_positive));
    local second_check::Bool = (Set(reverse(gpp.graph.levels)[1].current_state_negated) == Set(reverse(gpp.graph.levels)[2].current_state_negated));
    return (first_check && second_check);
end

function extract_solution(gpp::GraphPlanProblem, goals_positive::AbstractVector, goals_negated::AbstractVector, index::Int64)
    local level::PlanningLevel;
    if (index < 0)
        level = reverse(gpp.graph.levels)[abs(index)];
    else
        error("extract_solution(): Expected negative index, got ", index, "!");
    end
    if (!non_mutex_goals(gpp.graph, vcat(goals_positive, goals_negated), index))
        push!(gpp.nogoods, (level, goals_positive, goals_negated));
        return nothing;
    end
    level = reverse(gpp.graph.levels)[abs(index - 1)];
    local actions::AbstractVector = [];
    for goal in goals_positive
        push!(actions, level.next_state_links_positive[goal]);
    end
    for goal in goals_negated
        push!(actions, level.next_state_links_negated[goal]);
    end

    # Create all possible combinations of actions by using finding the cartesian product.
    local action_combinations::AbstractVector = iterable_cartesian_product(actions);
    # Remove action combinations that contain mutexes.
    local non_mutex_actions::AbstractVector = [];
    for action_list in action_combinations
        action_pairs = planning_combinations(collect(Set(action_list)));
        push!(non_mutex_actions, collect(Set(action_list)));
        for pair in action_pairs
            if (Set(collect(pair)) in level.mutex_links)
                pop!(non_mutex_actions);
                break;
            end
        end
    end

    for action_list in non_mutex_actions
        if (!([action_list, index] in gpp.solution))
            push!(gpp.solution, [action_list, index]);
            local new_goals_positive::AbstractVector = [];
            local new_goals_negated::AbstractVector = [];
            for action in Set(action_list)
                if (haskey(level.current_action_links_positive, action))
                    new_goals_positive = vcat(new_goals_positive, level.current_action_links_positive[action]);
                end
                if (haskey(level.current_action_links_negated, action))
                    new_goals_negated = vcat(new_goals_negated, level.current_action_links_negated[action]);
                end
            end
            if ((abs(index) + 1) == length(gpp.graph.levels))
                return nothing;
            elseif ((level, new_goals_positive, new_goals_negated) in gpp.nogoods)
                return nothing;
            else
                extract_solution(gpp, new_goals_positive, new_goals_negated, (index - 1))
            end
        end
    end
    local solution::AbstractVector = [];
    for item in gpp.solution
        if (item[2] == -1)
            push!(solution, [item[1]]);
        else
            push!(reverse(solution)[1], item[1]);
        end
    end
    for (i, item) in enumerate(solution)
        solution[i] = reverse(item);
    end
    return solution;
end

function goal_test(gpp::GraphPlanProblem, goals::AbstractVector)    #goal_test() for graphplan()
    local positive_kb::FirstOrderLogicKnowledgeBase = reverse(gpp.graph.levels)[1].positive_kb;
    return all((function(ans)
                    if (typeof(ans) <: Bool)
                        return ans;
                    else
                        if (length(ans) == 0)   # length of Tuple
                            return false;
                        else
                            return true;
                        end
                    end
                end),
                collect(ask(positive_kb, goal) for goal in goals));
end

"""
    graphplan(gpp::GraphPlanProblem, goals::Tuple)

Apply the Graphplan algorithm (Fig. 10.9) to the given planning problem 'gpp' and goal state 'goals'.
Return the solution or 'nothing' on failure.
"""
function graphplan(gpp::GraphPlanProblem, goals::Tuple)
    local goals_positive::AbstractVector = goals[1];
    local goals_negated::AbstractVector = goals[2];
    while (true)
        if (goal_test(gpp, goals_positive) && non_mutex_goals(gpp.graph, vcat(goals_positive, goals_negated), -1))
            solution = extract_solution(gpp, goals_positive, goals_negated, -1);
            if (!(typeof(solution) <: Nothing))
                return solution;
            end
        end
        expand_graph(gpp.graph);
        if ((length(gpp.graph.levels) > 1) && check_level_off(gpp))
            return nothing;
        end
    end
    return nothing;
end

function doubles_tennis_pddl_goal_test(kb::FirstOrderLogicKnowledgeBase)
    return all((function(ans)
                    if (typeof(ans) <: Bool)
                        return ans;
                    else
                        if (length(ans) == 0)   #length of Tuple
                            return false;
                        else
                            return true;
                        end
                    end
                end),
                collect(ask(kb, q) for q in map(expr, ["Returned(Ball)", "At(a, RightNet)", "At(a, LeftNet)"])));
end

"""
    doubles_tennis_pddl()

Return a PDDL representing the doubles tennis planning problem (Fig. 11.10).

The PlanningActions 'hit' and 'go' both take 3 arguments because their
preconditions will not be satisfied without the 'loc' variable.
This occurs when substituting propositional logic variables within
the check_precondition() function.
"""
function doubles_tennis_pddl()
    local initial::Array{Expression, 1} = map(expr, ["At(A, LeftBaseLine)",
                                                    "At(B, RightNet)",
                                                    "Approaching(Ball, RightBaseLine)",
                                                    "Partner(A, B)",
                                                    "Partner(B, A)"]);
    # Hit Action Schema
    local precondition_positive::Array{Expression, 1} = map(expr, ["Approaching(Ball, loc)", "At(actor, loc)"]);
    local precondition_negated::Array{Expression, 1} = [];
    local effect_add_list::Array{Expression, 1} = [expr("Returned(Ball)")];
    local effect_delete_list::Array{Expression, 1} = [];
    local hit::PlanningAction = PlanningAction(expr("Hit(actor, loc, Ball)"),
                                                (precondition_positive, precondition_negated),
                                                (effect_add_list, effect_delete_list));
    # Go Action Schema
    precondition_positive = [expr("At(actor, loc)")];
    precondition_negated = [];
    effect_add_list = [expr("At(actor, to)")];
    effect_delete_list = [expr("At(actor, loc)")];
    local go::PlanningAction = PlanningAction(expr("Go(actor, loc, to)"),
                                            (precondition_positive, precondition_negated),
                                            (effect_add_list, effect_delete_list));
    return PDDL(initial, [hit, go], doubles_tennis_pddl_goal_test);
end

#=

    PlanningHighLevelAction (HLA) is an action schema with resource constraints and refinements.

=#
mutable struct PlanningHighLevelAction <: AbstractPlanningAction
    name::String
    arguments::Tuple
    precondition_positive::Array{Expression, 1}
    precondition_negated::Array{Expression, 1}
    effect_add_list::Array{Expression, 1}
    effect_delete_list::Array{Expression, 1}
    duration::Int64
    consumes::Dict
    uses::Dict
    completed::Bool
    priority::Int64     #undefined field used to schedule multiple HLAs
    job_group::Int64    #undefined field used to schedule multiple HLAs

    function PlanningHighLevelAction(action::Expression,
                                    precondition::Tuple{Vararg{Array{Expression, 1}, 2}},
                                    effect::Tuple{Vararg{Array{Expression, 1}, 2}};
                                    duration::Int64=0,
                                    consumes::Dict=Dict(),
                                    uses::Dict=Dict())
        return new(action.operator, action.arguments, precondition[1], precondition[2], effect[1], effect[2], duration, consumes, uses, false);
    end
end

"""
    execute_action(hla, job_order, available_resources, kb, arguments)

Execute the given action 'hla' on the knowledge base of the planning problem.
This function checks for resources and job order execution required for the action to be executed.
"""
function execute_action(hla::PlanningHighLevelAction, job_order::Array{Array{PlanningHighLevelAction, 1}, 1}, available_resources::Dict, kb::AbstractKnowledgeBase, arguments::Tuple)
    if (!check_consumes_resource(hla, available_resources))
        error(@sprintf("execute_action(): High-level action \"%s\"'s consumable constraint was not satisfied!", hla.name));
    elseif (!check_uses_resource(hla, available_resources))
        error(@sprintf("execute_action(): High-level action \"%s\"'s reusable constraint was not satisfied!", hla.name));
    elseif (!check_job_order(hla, job_order))
        error(@sprintf("execute_action(): High-level action \"%s\" is not scheduled to execute! All scheduled unexecuted actions should run first!", hla.name));
    end

    # Execute 'hla' like a primitive action within this method to avoid confusion of
    # calling execute_action() within execute_action().

    if (!(check_precondition(hla, kb, arguments)))
        error(@sprintf("execute_action(): High-level action \"%s\" preconditions are not satisfied!", hla.name));
    end
    # Retract negated literals to knowledge base 'kb'.
    for clause in hla.effect_delete_list
        retract(kb, substitute(hla, clause, arguments));
    end
    # Add positive literals to knowledge base 'kb'.
    for clause in hla.effect_add_list
        tell(kb, substitute(hla, clause, arguments));
    end

    for resource in keys(hla.consumes)
        available_resources[resource] = available_resources[resource] - hla.consumes[resource];
    end

    # Update current high-level action task as completed.
    hla.completed = true;
    nothing;
end

function check_consumes_resource(hla::PlanningHighLevelAction, available_resources::Dict)
    for resource in keys(hla.consumes)
        if (!haskey(available_resources, resource))
            return false;
        elseif (available_resources[resource] < (hla.consumes)[resource])
            return false;
        end
    end
    return true;
end

function check_uses_resource(hla::PlanningHighLevelAction, available_resources::Dict)
    for resource in keys(hla.uses)
        if (!haskey(available_resources, resource))
            return false;
        elseif (available_resources[resource] < (hla.uses)[resource])
            return false;
        end
    end
    return true;
end

function check_job_order(hla::PlanningHighLevelAction, job_order::Array{Array{PlanningHighLevelAction, 1}, 1})
    for jobs in job_order
        if (hla in jobs)
            for job in jobs
                if (job == hla)
                    return true;
                elseif (!job.completed)
                    return false;
                end
            end
        end
    end
    return true;
end

#=

    HighLevelPDDL is a high-level Planning Domain Definition Language (PDDL) and is used to

    define a search problem. The HighLevelPDDL includes job scheduling and resource constraints

    in addition to the states, action schemas, and the goal test found in the PDDL datatype.

=#
struct HighLevelPDDL <: AbstractPDDL
    kb::FirstOrderLogicKnowledgeBase
    actions::Array{PlanningHighLevelAction, 1}
    goal_test::Function
    jobs::Array{Array{PlanningHighLevelAction, 1}, 1}
    resources::Dict

    function HighLevelPDDL(initial_state::Array{Expression, 1}, actions::Array{PlanningHighLevelAction, 1}, goal_test::Function; jobs::Array{Array{PlanningHighLevelAction, 1}, 1}=Array{Array{PlanningHighLevelAction, 1}, 1}(), resources::Dict=Dict())
        return new(FirstOrderLogicKnowledgeBase(initial_state), actions, goal_test, jobs, resources);
    end
end

"""
    execute_action(plan, action)

Execute the first relevant PlanningHighLevelAction associated with action 'action'
on the given planning problem's knowledge base.
"""
function execute_action(plan::HighLevelPDDL, action::PlanningHighLevelAction)
    local arguments::Tuple = action.arguments;
    local relevant_actions::AbstractVector = collect(a for a in plan.actions if (a.name == action.name));
    if (length(relevant_actions) == 0)
        error(@sprintf("execute_action(): High-level action \"%s\" not found!", action.name));
    else
        local first_relevant_action::PlanningHighLevelAction = relevant_actions[1];
        execute_action(first_relevant_action, plan.jobs, plan.resources, plan.kb, arguments);
    end
    nothing;
end

"""
    get_result(plan, action)

Return the planning problem after executing the given high-level action 'action'.
"""
function get_result(plan::HighLevelPDDL, action::PlanningHighLevelAction)
    execute_action(plan, action);
    return plan;
end

function get_result(plan::HighLevelPDDL, action::Nothing)
    return plan;
end

"""
    refinements(hla, state, dict)

Return a action sequence such that each refinement's preconditions of 'hla' are satisfied by
the given state of the planning problem 'state'.
"""
function refinements(hla::PlanningHighLevelAction, state::HighLevelPDDL, dict::Dict)
    local solution::Array{PlanningHighLevelAction, 1} = Array{PlanningHighLevelAction, 1}();
    local indices::AbstractVector = collect(i for (i, x) in enumerate(dict["HLA"]) if (expr(x).operator == hla.name));
    for i in indices
        local action::PlanningHighLevelAction = PlanningHighLevelAction(expr(dict["steps"][i][1]),
                                                                        (collect(expr(x) for x in dict["precondition_positive"][i]), collect(expr(x) for x in dict["precondition_negated"][i])),
                                                                        (collect(expr(x) for x in dict["effect_add_list"][i]), collect(expr(x) for x in dict["effect_delete_list"][i])));
        if (check_precondition(hla, state.kb, action.arguments))
            push!(solution, action);
        end
    end
    return solution;
end

"""
    hierarchical_search(problem, hierarchy)

Use the breadth-first implementation of the Hierarchical Search algorithm (Fig. 11.5) to the
given high-level planning problem 'problem' and 'hierarchy'. Return the solution or 'nothing'
on failure.
"""
function hierarchical_search(problem::HighLevelPDDL, hierarchy)
    local act::Node = Node(problem.actions[1]);
    local frontier::FIFOQueue();
    push!(frontier, act);
    while (true)
        if (length(frontier) == 0)
            return nothing;
        end
        local plan = pop!(frontier);
        hla = plan.state;
        if (!(plan.parent === nothing))
            prefix = plan.parent.action;
        else
            prefix = nothing;
        end
        local outcome::HighLevelPDDL = get_result(problem, prefix);
        if (typeof(hla) <: Nothing)
            if (goal_test(outcome))
                return path(plan);
            end
        else
            for sequence in refinements(hla, outcome, hierachy)
                push!(frontier, Node(plan.state, parent=plan.parent, action=sequence));
            end
        end
    end
    nothing;
end

function job_shop_scheduling_pddl_goal_test(kb::FirstOrderLogicKnowledgeBase)
    return all((function(ans)
                    if (typeof(ans) <: Bool)
                        return ans;
                    else
                        if (length(ans) == 0)   # length of Tuple
                            return false;
                        else
                            return true;
                        end
                    end
                end),
                collect(ask(kb, q) for q in map(expr, ["Has(C1, W1)", "Has(C1, E1)", "Inspected(C1)",
                                                        "Has(C2, W2)", "Has(C2, E2)", "Inspected(C2)",
                                                        "Car(C1)", "Car(C2)",
                                                        "Wheels(W1)", "Wheels(W2)",
                                                        "Engine(E1)", "Engine(E2)"])));
end

"""
    job_shop_scheduling_pddl()

Return a HighLevelPDDL representing the job-shop scheduling planning problem (Fig. 11.1).
"""
function job_shop_scheduling_pddl()
    local initial::Array{Expression, 1} = map(expr, ["Car(C1)", "Car(C2)",
                                                    "Wheels(W1)", "Wheels(W2)",
                                                    "Engine(E1)", "Engine(E2)"]);
    local resources::Dict = Dict([Pair("EngineHoists", 1),
                                Pair("WheelStations", 2),
                                Pair("Inspectors", 2),
                                Pair("LugNuts", 500)]);
    # AddEngine1 high-level action schema
    local precondition_positive::Array{Expression, 1} = [];
    local precondition_negated::Array{Expression, 1} = [expr("Has(C1, E1)")];
    local effect_add_list::Array{Expression, 1} = [expr("Has(C1, E1)")];
    local effect_delete_list::Array{Expression, 1} = [];
    local add_engine_1::PlanningHighLevelAction = PlanningHighLevelAction(expr("AddEngine1"),
                                                                        (precondition_positive, precondition_negated),
                                                                        (effect_add_list, effect_delete_list),
                                                                        duration=30, uses=Dict([Pair("EngineHoists", 1)]));
    # AddEngine2 high-level action schema
    precondition_positive = [];
    precondition_negated = [expr("Has(C2, E2)")];
    effect_add_list = [expr("Has(C2, E2)")];
    effect_delete_list = [];
    local add_engine_2::PlanningHighLevelAction = PlanningHighLevelAction(expr("AddEngine2"),
                                                                        (precondition_positive, precondition_negated),
                                                                        (effect_add_list, effect_delete_list),
                                                                        duration=60, uses=Dict([Pair("EngineHoists", 1)]));
    # AddWheels1 high-level action schema
    precondition_positive = [];
    precondition_negated = [expr("Has(C1, W1)")];
    effect_add_list = [expr("Has(C1, W1)")];
    effect_delete_list = [];
    local add_wheels_1::PlanningHighLevelAction = PlanningHighLevelAction(expr("AddWheels1"),
                                                                        (precondition_positive, precondition_negated),
                                                                        (effect_add_list, effect_delete_list),
                                                                        duration=30, consumes=Dict([Pair("LugNuts", 20)]), uses=Dict([Pair("WheelStations", 1)]));
    # AddWheels2 high-level action schema
    precondition_positive = [];
    precondition_negated = [expr("Has(C2, W2)")];
    effect_add_list = [expr("Has(C2, W2)")];
    effect_delete_list = [];
    local add_wheels_2::PlanningHighLevelAction = PlanningHighLevelAction(expr("AddWheels2"),
                                                                        (precondition_positive, precondition_negated),
                                                                        (effect_add_list, effect_delete_list),
                                                                        duration=15, consumes=Dict([Pair("LugNuts", 20)]), uses=Dict([Pair("WheelStations", 1)]));
    # Inspect1 high-level action schema
    precondition_positive = [];
    precondition_negated = [expr("Inspected(C1)")];
    effect_add_list = [expr("Inspected(C1)")];
    effect_delete_list = [];
    local inspect_1::PlanningHighLevelAction = PlanningHighLevelAction(expr("Inspect1"),
                                                                        (precondition_positive, precondition_negated),
                                                                        (effect_add_list, effect_delete_list),
                                                                        duration=10, uses=Dict([Pair("Inspectors", 1)]));
    # Inspect2 high-level action schema
    precondition_positive = [];
    precondition_negated = [expr("Inspected(C2)")];
    effect_add_list = [expr("Inspected(C2)")];
    effect_delete_list = [];
    local inspect_2::PlanningHighLevelAction = PlanningHighLevelAction(expr("Inspect2"),
                                                                        (precondition_positive, precondition_negated),
                                                                        (effect_add_list, effect_delete_list),
                                                                        duration=10, uses=Dict([Pair("Inspectors", 1)]));
    local job_group_1::Array{PlanningHighLevelAction, 1} = [add_engine_1, add_wheels_1, inspect_1];
    local job_group_2::Array{PlanningHighLevelAction, 1} = [add_engine_2, add_wheels_2, inspect_2];
    return HighLevelPDDL(initial, [add_engine_1, add_engine_2, add_wheels_1, add_wheels_2, inspect_1, inspect_2], job_shop_scheduling_pddl_goal_test, jobs=[job_group_1, job_group_2], resources=resources);
end


const _BitInteger = (Int == Int64 ? Base.BitInteger64 : Base.BitInteger32)
const _BitSigned = (Int == Int64 ? Base.BitSigned64 : Base.BitSigned32)
const _BitUnsigned = (Int == Int64 ? Base.BitUnsigned64 : Base.BitUnsigned32)

### Mag
for (jf, af) in [(:+, :add!), (:-, :sub!), (:*, :mul!), (:/, :div!)]
    @eval Base.$jf(x::MagOrRef, y::MagOrRef) = $af(zero(x), x, y)
end
Base.:+(x::MagOrRef, y::Integer) = add!(zero(x), x, convert(UInt, y))
Base.:+(x::Integer, y::MagOrRef) = add!(zero(y), y, convert(UInt, x))
Base.:*(x::MagOrRef, y::Integer) = mul!(zero(x), x, convert(UInt, y))
Base.:*(x::Integer, y::MagOrRef) = mul!(zero(y), y, convert(UInt, x))
Base.:/(x::MagOrRef, y::Integer) = div!(zero(x), x, convert(UInt, y))

Base.:(^)(x::MagOrRef, e::Integer) = pow!(zero(x), x, convert(UInt, e))
rsqrt(x::MagOrRef) = rsqrt!(zero(x), x)
Base.hypot(x::MagOrRef, y::MagOrRef) = hypot!(zero(x), x, y)
root(x::MagOrRef, n::Integer) = root!(zero(x), x, convert(UInt, n))
neglog(x::MagOrRef) = neg_log!(zero(x), x)
expinv(x::MagOrRef) = expinv!(zero(x), x)
for f in [:inv, :sqrt, :log, :log1p, :exp, :expm1, :atan, :cosh, :sinh]
    @eval Base.$f(x::MagOrRef) = $(Symbol(f, :!))(zero(x), x)
end

### Arf
function Base.sign(x::ArfOrRef)
    isnan(x) && return Arf(NaN) # Follow Julia and return NaN
    return Arf(sgn(x))
end

Base.abs(x::ArfOrRef) = abs!(zero(x), x)
Base.:(-)(x::ArfOrRef) = neg!(zero(x), x)
for (jf, af) in [(:+, :add!), (:-, :sub!), (:*, :mul!), (:/, :div!)]
    @eval function Base.$jf(x::ArfOrRef, y::Union{ArfOrRef,_BitInteger})
        z = Arf(prec = _precision((x, y)))
        $af(z, x, y)
        return z
    end
end
function Base.:+(x::_BitInteger, y::ArfOrRef)
    z = zero(y)
    add!(z, y, x)
    return z
end
function Base.:*(x::_BitInteger, y::ArfOrRef)
    z = zero(y)
    mul!(z, y, x)
    return z
end
function Base.:/(x::_BitUnsigned, y::ArfOrRef)
    z = zero(y)
    ui_div!(z, x, y)
    return z
end
function Base.:/(x::_BitSigned, y::ArfOrRef)
    z = zero(y)
    si_div!(z, x, y)
    return z
end

function Base.sqrt(x::ArfOrRef)
    y = zero(x)
    sqrt!(y, x)
    return y
end
function rsqrt(x::ArfOrRef)
    y = zero(x)
    rsqrt!(y, x)
    return y
end
function root(x::ArfOrRef, k::Integer)
    y = zero(x)
    root!(y, x, convert(UInt, k))
    return y
end

### Arb and Acb
for (jf, af) in [(:+, :add!), (:-, :sub!), (:*, :mul!), (:/, :div!)]
    @eval Base.$jf(x::ArbOrRef, y::Union{ArbOrRef,ArfOrRef,_BitInteger}) =
        $af(Arb(prec = _precision((x, y))), x, y)
    @eval Base.$jf(x::AcbOrRef, y::Union{AcbOrRef,ArbOrRef,_BitInteger}) =
        $af(Acb(prec = _precision((x, y))), x, y)
    if jf == :(+) || jf == :(*)
        @eval Base.$jf(x::Union{ArfOrRef,_BitInteger}, y::ArbOrRef) =
            $af(Arb(prec = _precision((x, y))), y, x)
        @eval Base.$jf(x::Union{ArbOrRef,_BitInteger}, y::AcbOrRef) =
            $af(Acb(prec = _precision((x, y))), y, x)
    end
end

Base.:(-)(x::Union{ArbOrRef,AcbOrRef}) = neg!(zero(x), x)
Base.abs(x::ArbOrRef) = abs!(zero(x), x)
Base.:(/)(x::_BitUnsigned, y::ArbOrRef) = ui_div!(zero(y), x, y)

Base.:(^)(x::ArbOrRef, y::ArbOrRef) = pow!(Arb(prec = _precision((x, y))), x, y)
function Base.:(^)(x::ArbOrRef, y::_BitInteger)
    z = zero(x)
    x, n = (y >= 0 ? (x, y) : (inv!(z, x), -y))
    return pow!(z, x, convert(UInt, n))
end
Base.:(^)(x::AcbOrRef, y::Union{AcbOrRef,ArbOrRef,_BitInteger}) =
    pow!(Acb(prec = _precision((x, y))), x, y)

# We define the same special cases as Arb does, this avoids some
# overhead
Base.literal_pow(::typeof(^), x::Union{ArbOrRef,AcbOrRef}, ::Val{-2}) =
    (y = inv(x); sqr!(y, y))
#Base.literal_pow(::typeof(^), x::Union{ArbOrRef,AcbOrRef}, ::Val{-1}) - implemented in Base.intfuncs.jl
Base.literal_pow(::typeof(^), x::Union{ArbOrRef,AcbOrRef}, ::Val{0}) = one(x)
Base.literal_pow(::typeof(^), x::Union{ArbOrRef,AcbOrRef}, ::Val{1}) = copy(x)
Base.literal_pow(::typeof(^), x::Union{ArbOrRef,AcbOrRef}, ::Val{2}) = sqr(x)

Base.hypot(x::ArbOrRef, y::ArbOrRef) = hypot!(Arb(prec = _precision((x, y))), x, y)

root(x::Union{ArbOrRef,AcbOrRef}, k::Integer) = root!(zero(x), x, convert(UInt, k))

# Unary methods in Base
for f in [
    :inv,
    :sqrt,
    :log,
    :log1p,
    :exp,
    :expm1,
    :sin,
    :cos,
    :tan,
    :cot,
    :sec,
    :csc,
    :atan,
    :asin,
    :acos,
    :sinh,
    :cosh,
    :tanh,
    :coth,
    :sech,
    :csch,
    :atanh,
    :asinh,
    :acosh,
]
    @eval Base.$f(x::Union{ArbOrRef,AcbOrRef}) = $(Symbol(f, :!))(zero(x), x)
end

sqrtpos(x::ArbOrRef) = sqrtpos!(zero(x), x)
sqrt1pm1(x::ArbOrRef) = sqrt1pm1!(zero(x), x)
rsqrt(x::Union{ArbOrRef,AcbOrRef}) = rsqrt!(zero(x), x)
sqr(x::Union{ArbOrRef,AcbOrRef}) = sqr!(zero(x), x)

Base.sinpi(x::Union{ArbOrRef,AcbOrRef}) = sin_pi!(zero(x), x)
Base.cospi(x::Union{ArbOrRef,AcbOrRef}) = cos_pi!(zero(x), x)
tanpi(x::Union{ArbOrRef,AcbOrRef}) = tan_pi!(zero(x), x)
cotpi(x::Union{ArbOrRef,AcbOrRef}) = cot_pi!(zero(x), x)
cscpi(x::Union{ArbOrRef,AcbOrRef}) = csc_pi!(zero(x), x)
# Julias definition of sinc is equivalent to Arbs definition of sincpi
Base.sinc(x::Union{ArbOrRef,AcbOrRef}) = sinc_pi!(zero(x), x)
Base.atan(y::ArbOrRef, x::ArbOrRef) = atan2!(Arb(prec = _precision((y, x))), y, x)

function Base.sincos(x::Union{ArbOrRef,AcbOrRef})
    s, c = zero(x), zero(x)
    sin_cos!(s, c, x)
    return (s, c)
end
function sincospi(x::Union{ArbOrRef,AcbOrRef})
    s, c = zero(x), zero(x)
    sin_cos_pi!(s, c, x)
    return (s, c)
end
function sinhcosh(x::Union{ArbOrRef,AcbOrRef})
    s, c = zero(x), zero(x)
    sinh_cosh!(s, c, x)
    return (s, c)
end

### Acb
function Base.:(*)(x::AcbOrRef, y::Complex{Bool})
    if real(y)
        if imag(y)
            z = mul_onei!(zero(x), x)
            return add!(z, x, z)
        else
            return Acb(x)
        end
    end
    imag(y) && return mul_onei!(zero(x), x)
    return zero(x)
end
Base.:(*)(x::Complex{Bool}, y::AcbOrRef) = y * x

Base.real(z::AcbLike; prec = _precision(z)) = get_real!(Arb(prec = prec), z)
Base.imag(z::AcbLike; prec = _precision(z)) = get_imag!(Arb(prec = prec), z)
Base.conj(z::AcbLike) = conj!(Acb(prec = _precision(z)), z)
Base.abs(z::AcbLike) = abs!(Arb(prec = _precision(z)), z)

### min and max
for T in [MagOrRef, ArfOrRef, ArbOrRef]
    for op in [:min, :max]
        # min/max
        @eval Base.$op(x::$T, y::$T) = $(Symbol(op, :!))(zero(x), x, y)
        # minimum/maximum
        # The default implementation of minimum/maximum in Julia is
        # not correct for Arb, we implemented a correct version when
        # no specific dimension is given. The default give correct
        # results for Mag and Arf but wrong results for Arb in some
        # cases, be careful!
        @eval function Base.$(Symbol(op, :imum))(A::AbstractArray{<:$T})
            isempty(A) &&
                throw(ArgumentError("reducing over an empty collection is not allowed"))
            res = copy(first(A))
            for x in Iterators.drop(A, 1)
                $(Symbol(op, :!))(res, res, x)
            end
            return res
        end
    end
    @eval Base.minmax(x::$T, y::$T) = (min(x, y), max(x, y))

    @eval function Base.extrema(A::AbstractArray{<:$T})
        isempty(A) &&
            throw(ArgumentError("reducing over an empty collection is not allowed"))
        l = copy(first(A))
        u = copy(first(A))
        for x in Iterators.drop(A, 1)
            min!(l, l, x)
            max!(u, u, x)
        end
        return (l, u)
    end
end


const cosmic_fl = joinpath(homedir(), "haplox/Bone/data/cosmic/CosmicCompleteExport.tsv")

function numsamplesofcancer(cancer)
    sampleids =  Set{ASCIIString}()
    open(cosmic_fl) do file
        header = readline(file)
        header  = split(header, "\t")
        while !eof(file)
            line = readline(file)
            row  = split(line, "\t")
            if row[8] != cancer
                continue
            end
            sample_id = convert(ASCIIString,row[6])
            push!(sampleids, sample_id)
        end
    end
    #@show sampleids
    length(sampleids)
end

function samplerecordsofcancer(cancer, threshold)
    sampleid_numrecords =  Dict{ASCIIString,Int64}()
    open(cosmic_fl) do file
        header = readline(file)
        header  = split(header, "\t")
        while !eof(file)
            line = readline(file)
            row  = split(line, "\t")
            if row[8] != cancer
                continue
            end
            sample_id = convert(ASCIIString,row[6])
            if in(sample_id, keys(sampleid_numrecords))
                sampleid_numrecords[sample_id] += 1
            else
                sampleid_numrecords[sample_id] = 1
            end
        end
    end
    #@show sampleids
    length(filter(x->x[2]>threshold, sort(collect(sampleid_numrecords),by=x->x[2],rev=true)))
end

@show samplerecordsofcancer("small_intestine", 4)