function _readgml(io::IO, line, ::Type{G}) where G
    mapping = Dict{Int,Int}()
    i = 0
    while startswith(line, "node")
        i += 1
        line = readline(io) |> strip
        line == "[" && (line = readline(io) |> strip)
        while !startswith(line, "]")
            name, valstr = splitgml(line)
            if name == "id"
                mapping[parse(Int, valstr)] = i
            end
            line = readline(io) |> strip
        end
        line = readline(io) |> strip #skip ]
    end
    g = G(i)
    while startswith(line, "edge")
        line = readline(io) |> strip
        line == "[" && (line = readline(io) |> strip)
        u = -1
        v = -1
        while !startswith(line, "]")
            name, valstr = splitgml(line)
            if name == "source"
                id = parse(Int, valstr)
                u = mapping[id]
            elseif name == "target"
                id = parse(Int, valstr)
                v = mapping[id]
            end
            line = readline(io) |> strip
        end
        @assert u > 0 && v > 0
        add_edge!(g, u, v)
        line = readline(io) |> strip #skip ]
    end
    return g
end

function readgml(io::IO, ::Type{G}) where G
    H = G
    line = readline(io) |> strip
    @assert startswith(line, "graph")
    line = readline(io) |> strip
    line == "[" && (line = readline(io) |> strip)
    while !startswith(line, "node") && !isempty(line)
        if startswith(line, "directed")
            H = parse(Int, line[10:end]) == 1 ? digraphtype(G) : graphtype(G)
        end
        line = readline(io) |> strip
    end
    return _readgml(io, line, H)
end

function gmltypeof(valstr)
    str = strip(valstr, '\"')
    if length(str) < length(valstr)
        return String
    else
        return Float64
    end
end

gmlval(::Type{T}, x) where {T} = parse(T, x)
gmlval(::Type{String}, x) = strip(x, '\"')
function splitgml(s::AbstractString)
    i = findfirst(isequal(' '), s)
    return SubString(s, 1, i-1), SubString(s, i+1, length(s))
end

gmlprintval(x) = x
gmlprintval(x::String) = "\"" * x * "\""

function _readnetgml(io::IO, line, g)
    mapping = Dict{Int,Int}()
    i = 0
    while startswith(line, "node")
        i += 1
        line = readline(io) |> strip
        line == "[" && (line = readline(io) |> strip)
        @assert startswith(line, "id")
        name, valstr = splitgml(line)
        mapping[parse(Int, valstr)] = i
        add_vertex!(g)
        line = readline(io) |> strip
        while !startswith(line, "]")
            name, valstr = splitgml(line)
            !has_vprop(g, name) && vprop!(g, name, String)
            T = valtype(vprop(g, name))
            vprop(g, name)[i] = gmlval(T, valstr)
            line = readline(io) |> strip
        end

        line = readline(io) |> strip #skip ]
    end

    while startswith(line, "edge")
        line = readline(io) |> strip
        line == "[" && (line = readline(io) |> strip)
        startswith(line, "id") && (line = readline(io) |> strip)

        @assert startswith(line, "source")
        name, valstr = splitgml(line)
        u = mapping[parse(Int, valstr)]
        line = readline(io) |> strip
        @assert startswith(line, "target")
        name, valstr = splitgml(line)
        v = mapping[parse(Int, valstr)]
        line = readline(io) |> strip
        ok, e = add_edge!(g, u, v)

        while !startswith(line, "]")
            name, valstr = splitgml(line)
            !has_eprop(g, name) && eprop!(g, name, gmltypeof(valstr))
            T = valtype(eprop(g, name))
            eprop(g, name)[e] = gmlval(T, valstr)
            line = readline(io) |> strip
        end

        line = readline(io) |> strip #skip ]
    end
    return g
end

function readnetgml(io::IO, ::Type{G}) where G<:ANetOrDiNet
    H = G
    line = readline(io) |> strip
    @assert startswith(line, "graph")
    line = readline(io) |> strip
    line == "[" && (line = readline(io) |> strip)
    gdict = Dict{String, String}()
    while !startswith(line, "node") && !isempty(line)
        if startswith(line, "directed")
            H = parse(Int, line[10:end]) == 1 ? digraphtype(G) : graphtype(G)
        else
            name, valstr = splitgml(line)
            gdict[name] = gmlval(String, valstr)
        end
        line = readline(io) |> strip
    end
    g = H()
    for (name, val) in gdict
        gprop!(g, name, val)
    end
    return _readnetgml(io, line, g)
end

"""
    writegml(f, g)

Writes a graph `g` to a file `f` in the
[GML](https://en.wikipedia.org/wiki/Graph_Modelling_Language) format.
"""
function writegml(io::IO, g::AGraphOrDiGraph)
    println(io, "graph [")
    # length(gname) > 0 && println(io, "label \"$gname\"")
    is_directed(g) && println(io, "\tdirected 1")
    for i=1:nv(g)
        println(io,"\tnode [")
        println(io,"\t\tid $i")
        println(io,"\t]")
    end
    for (s,t) in edges(g)
        println(io,"\tedge [")
        println(io,"\t\tsource $s")
        println(io,"\t\ttarget $t")
        println(io,"\t]")
    end
    println(io, "]")
    return 1
end

function writenetgml(io::IO, g::ANetOrDiNet)
    println(io, "graph [")
    is_directed(g) && println(io, "\tdirected 1")
    for (pname, p) in gprop(g)
        println(io,"\t$pname $(gmlprintval(p))")
    end
    for i=1:nv(g)
        println(io,"\tnode [")
        println(io,"\t\tid $i")
        for (name, val) in vprop(g, i)
            println(io,"\t\t$name $(gmlprintval(val))")
        end
        println(io,"\t]")
    end
    for e in edges(g)
        println(io,"\tedge [")
        println(io,"\t\tsource $(src(e))")
        println(io,"\t\ttarget $(dst(e))")
        for (name, val) in eprop(g, e)
            println(io,"\t\t$name $(gmlprintval(val))")
        end
        println(io,"\t]")
    end
    println(io, "]")
    return 1
end

filemap[:gml] = (readgml, writegml, readnetgml, writenetgml)


"Abstract type for ASPRS LAS point data record formats 0 - 3"
abstract type LasPoint end

"""
Custom PointVector struct for memory mapped LasPoints.
Inspiration taken from UnalignedVector.jl
and extended it with custom indexing and packing.
"""
struct PointVector{T} <: AbstractArray{T,1}
    io::IOBuffer
    n::Int
    pointsize::Int

    function PointVector{T}(data::Vector{UInt8}, pointsize::Integer) where {T}
        n = length(data) ÷ pointsize

        # IOBuffer takes (data, readable, writable)
        new{T}(IOBuffer(data; read=true, write=false), n, pointsize)
    end
end

Base.IndexStyle(::Type{PointVector{T}}) where {T} = IndexLinear()
Base.length(pv::PointVector) = pv.n
Base.size(pv::PointVector) = (length(pv),)

function Base.getindex(pv::PointVector{T}, i::Int) where {T}
    offset = (i-1) * pv.pointsize  # seeking uses 0 based indexing!
    position(pv.io) != offset && seek(pv.io, offset)
    read(pv.io, T)
end

function Base.setindex!(pv::PointVector{T}, val::T, i::Int) where {T}
    # offset = (i-1) * pv.pointsize  # seeking uses 0 based indexing!
    # position(pv.io) != offset && seek(pv.io, offset)
    # write(pv.io, val)
    error("Can't write to read only memory mapped file.")
end

function Base.show(io::IO, pointdata::AbstractVector{<:LasPoint})
    n = size(pointdata, 1)
    println(io, "$(typeof(pointdata)) with $n points.")
end

"ASPRS LAS point data record format 0"
@gen_io struct LasPoint0 <: LasPoint
    x::Int32
    y::Int32
    z::Int32
    intensity::UInt16
    flag_byte::UInt8
    raw_classification::UInt8
    scan_angle::Int8
    user_data::UInt8
    pt_src_id::UInt16
end

"ASPRS LAS point data record format 1"
@gen_io struct LasPoint1 <: LasPoint
    x::Int32
    y::Int32
    z::Int32
    intensity::UInt16
    flag_byte::UInt8
    raw_classification::UInt8
    scan_angle::Int8
    user_data::UInt8
    pt_src_id::UInt16
    gps_time::Float64
end

"ASPRS LAS point data record format 2"
@gen_io struct LasPoint2 <: LasPoint
    x::Int32
    y::Int32
    z::Int32
    intensity::UInt16
    flag_byte::UInt8
    raw_classification::UInt8
    scan_angle::Int8
    user_data::UInt8
    pt_src_id::UInt16
    red::N0f16
    green::N0f16
    blue::N0f16
end

"ASPRS LAS point data record format 3"
@gen_io struct LasPoint3 <: LasPoint
    x::Int32
    y::Int32
    z::Int32
    intensity::UInt16
    flag_byte::UInt8
    raw_classification::UInt8
    scan_angle::Int8
    user_data::UInt8
    pt_src_id::UInt16
    gps_time::Float64
    red::N0f16
    green::N0f16
    blue::N0f16
end

# for convenience in function signatures
const LasPointColor = Union{LasPoint2,LasPoint3}
const LasPointTime = Union{LasPoint1,LasPoint3}

function Base.show(io::IO, p::LasPoint)
    x = Int(p.x)
    y = Int(p.y)
    z = Int(p.z)
    cl = Int(classification(p))
    println(io, "LasPoint(x=$x, y=$y, z=$z, classification=$cl)")
end

# Extend base by enabling reading/writing relevant FixedPointNumbers from IO.
Base.read(io::IO, ::Type{N0f16}) = reinterpret(N0f16, read(io, UInt16))
Base.write(io::IO, t::N0f16) = write(io, reinterpret(UInt16, t))

# functions for IO on points

"X coordinate (Float64), apply scale and offset according to the header"
xcoord(p::LasPoint, h::LasHeader) = muladd(p.x, h.x_scale, h.x_offset)
"Y coordinate (Float64), apply scale and offset according to the header"
ycoord(p::LasPoint, h::LasHeader) = muladd(p.y, h.y_scale, h.y_offset)
"Z coordinate (Float64), apply scale and offset according to the header"
zcoord(p::LasPoint, h::LasHeader) = muladd(p.z, h.z_scale, h.z_offset)

# inverse functions of the above
"X value (Int32), as represented in the point data, reversing the offset and scale from the header"
xcoord(x::Real, h::LasHeader) = round(Int32, (x - h.x_offset) / h.x_scale)
"Y value (Int32), as represented in the point data, reversing the offset and scale from the header"
ycoord(y::Real, h::LasHeader) = round(Int32, (y - h.y_offset) / h.y_scale)
"Z value (Int32), as represented in the point data, reversing the offset and scale from the header"
zcoord(z::Real, h::LasHeader) = round(Int32, (z - h.z_offset) / h.z_scale)

# functions to access common LasPoint fields
"Integer representation of the pulse return magnitude."
intensity(p::LasPoint) = p.intensity
"Angle at which the laser point was output, including the roll of the aircraft."
scan_angle(p::LasPoint) = p.scan_angle
"This field may be used at the user’s discretion."
user_data(p::LasPoint) = p.user_data
"This value indicates the file from which this point originated."
pt_src_id(p::LasPoint) = p.pt_src_id

# time
# 1.3159648e9 = 315964800.0 + 1.0e9
# 1.0e9: see LAS spec (for float accuracy)
# 315964800: seconds between 1970 (Unix epoch) and 1980 (GPS epoch), from http://stackoverflow.com/a/20528332
"""Get the DateTime that the point was collected.
Assumes time is recorded in Adjusted Standard GPS Time; see `is_standard_gps`.
Note that DateTime has millisecond precision, any higher precision is lost."""
Dates.DateTime(p::LasPointTime) = Dates.unix2datetime(p.gps_time) + Dates.Second(1315964800)
# A conversion of the GPS Week Time format to DateTime is not yet implemented

"""Time tag value at which the point was aquired,
a Float64; see `is_standard_gps` for what it represents"""
gps_time(p::LasPointTime) = p.gps_time
"""Convert DateTime to GPS Float64, as represented in the point data,
assumes time is recorded in Adjusted Standard GPS Time; see `is_standard_gps`"""
gps_time(d::DateTime) = Dates.datetime2unix(d - Dates.Second(1315964800))

# color
"The red image channel value associated with this point"
ColorTypes.red(p::LasPointColor) = p.red
"The green image channel value associated with this point"
ColorTypes.green(p::LasPointColor) = p.green
"The blue image channel value associated with this point"
ColorTypes.blue(p::LasPointColor) = p.blue
"The RGB color associated with this point"
ColorTypes.RGB(p::LasPointColor) = RGB(red(p), green(p), blue(p))

# functions to extract sub-byte items from a LasPoint's flag_byte
"The pulse return number for a given output pulse, starting at one."
return_number(p::LasPoint) = (p.flag_byte & 0b00000111)
"The total number of returns for a given pulse."
number_of_returns(p::LasPoint) = (p.flag_byte & 0b00111000) >> 3
"If true, the scanner mirror was traveling from left to right at the time of the output pulse."
scan_direction(p::LasPoint) = Bool((p.flag_byte & 0b01000000) >> 6)
"If true, it is the last point before the scanner changes direction."
edge_of_flight_line(p::LasPoint) = Bool((p.flag_byte & 0b10000000) >> 7)

"Flag byte, contains return number, number of returns, scan direction flag and edge of flight line"
flag_byte(p::LasPoint) = p.flag_byte
"Flag byte, as represented in the point data, built up from components"
function flag_byte(return_number::UInt8, number_of_returns::UInt8,
                   scan_direction::Bool, edge_of_flight_line::Bool)
    # Bool to UInt8 conversion because a bit shift on a Bool produces an Int
    UInt8(edge_of_flight_line) << 7 | UInt8(scan_direction) << 6 | number_of_returns << 3 | return_number
end

# functions to extract sub-byte items from a LasPoint's raw_classification
"Classification value as defined in the ASPRS classification table."
classification(p::LasPoint) = (p.raw_classification & 0b00011111)
"If true, the point was not created from lidar collection"
synthetic(p::LasPoint) = Bool((p.raw_classification & 0b00100000) >> 5)
"If true, this point is considered to be a model key-point."
key_point(p::LasPoint) = Bool((p.raw_classification & 0b01000000) >> 6)
"If true, this point should not be included in processing"
withheld(p::LasPoint) = Bool((p.raw_classification & 0b10000000) >> 7)

"Raw classification byte, contains classification, synthetic, key point and withheld"
raw_classification(p::LasPoint) = p.raw_classification
"Raw classification byte, as represented in the point data, built up from components"
function raw_classification(classification::UInt8, synthetic::Bool,
                            key_point::Bool, withheld::Bool)
    UInt8(withheld) << 7 | UInt8(key_point) << 6 | UInt8(synthetic) << 5 | classification
end

function Base.convert(::Type{Point{3, Float64}}, p::LasPoint, h::LasHeader)
    Point{3, Float64}(xcoord(p, h), ycoord(p, h), zcoord(p, h))
end

# beware of the limited precision, for instance with UTM coordinates
function Base.convert(::Type{Point{3, Float32}}, p::LasPoint, h::LasHeader)
    Point{3, Float32}(xcoord(p, h), ycoord(p, h), zcoord(p, h))
end


using Q.K_KeyTable
@testset "table tests" begin
  @test_throws ArgumentError K_KeyTable(kj(0))
  @test_throws ArgumentError K_KeyTable(xD(ktn(0,0),ktn(0,0)))
  let x = K_Table(a=Int[]), p = kpointer(x)
    @test_throws ArgumentError K_KeyTable(xD(r1(p),ktn(0,0)))
    @test (kt = K_KeyTable(xD(r1(p), r1(p))); xt(kpointer(kt)) == XD)
  end
  @test (x = K_KeyTable(1; a=Int[], b=Int[]); size(x) == (0, 2))
  @test begin
    t = K_KeyTable(1; a=[0, 1, 2], b=[0., 10., 20.])
    show_to_string(t) == """
    3×2 Q.K_KeyTable
    │ Row │ a │ b    │
    ├─────┼───┼──────┤
    │ 1   │ 0 │ 0.0  │
    │ 2   │ 1 │ 10.0 │
    │ 3   │ 2 │ 20.0 │"""
  end
  @test (x = K_KeyTable(1;a=Int[],b=Int[]); x[:a] == x[:b] == Int[])
end


module XGBoost

using Printf
using Random: randperm, seed!
using SparseArrays: SparseMatrixCSC, nnz
using Statistics: mean, std

export DMatrix, Booster
export xgboost, predict, save, nfold_cv, slice, get_info, set_info, dump_model, importance
export rabit_init, rabit_finalize, rabit_is_distributed, rabit_get_rank, rabit_get_world_size, rabit_get_version_number

global const build_version = "0.82"
include("../deps/deps.jl")

function __init__()
    check_deps()
end
include("xgboost_wrapper_h.jl")
include("rabit_wrapper.jl")
include("xgboost_lib.jl")

end # module XGBoost


############################################################
# judiRHS ## #################################################
############################################################

# Authors: Philipp Witte (pwitte@eos.ubc.ca), Henryk Modzelewski (hmodzelewski@eos.ubc.ca)
# Date: January 2017

export judiRHS, judiRHSexception

############################################################

struct judiRHS{vDT<:Number} <: joAbstractLinearOperator{vDT,vDT}
    name::String
    m::Integer
    n::Integer
    info::Info
    geometry::Geometry
    data
end

mutable struct judiRHSexception <: Exception
    msg :: String
end

############################################################

## outer constructors

"""
    judiRHS
        name::String
        m::Integer
        n::Integer
        info::Info
        geometry::Geometry
        data

Abstract sparse vector for right-hand-sides of the modeling operators. The `judiRHS` vector has the\\
dimensions of the full time history of the wavefields, but contains only the data defined at the \\
source or receiver positions (i.e. wavelets or shot records).

Constructor
==========

    judiRHS(info, geometry, data)

Examples
========

Assuming `Pr` and `Ps` are projection operators of type `judiProjection` and `dobs` and `q` are\\
seismic vectors of type `judiVector`, then a `judiRHS` vector can be created as follows:

    rhs = Pr'*dobs    # right-hand-side with injected observed data
    rhs = Ps'*q    # right-hand-side with injected wavelet

"""
function judiRHS(info,geometry,data;vDT::DataType=Float32)
    vDT == Float32 || throw(judiRHSexception("Domain type not supported"))
    # length of vector
    m = info.n * sum(info.nt)
    n = 1
    return judiRHS{Float32}("judiRHS",m,n,info,geometry,data)
end

####################################################################
## overloaded Base functions

# conj(jo)
conj(A::judiRHS{vDT}) where vDT =
    judiRHS{vDT}("conj("*A.name*")",A.m,A.n,A.info,A.geometry,A.data)

# transpose(jo)
transpose(A::judiRHS{vDT}) where vDT =
    judiRHS{vDT}(""*A.name*".'",A.n,A.m,A.info,A.geometry,A.data)

# adjoint(jo)
adjoint(A::judiRHS{vDT}) where vDT =
    judiRHS{vDT}(""*A.name*"'",A.n,A.m,A.info,A.geometry,A.data)

####################################################################

# +(judiRHS,judiRHS)
function +(A::judiRHS{avDT}, B::judiRHS{bvDT}) where {avDT, bvDT}

    # Error checking
    size(A) == size(B) || throw(judiRHSexception("shape mismatch"))
    compareInfo(A.info, B.info) == true || throw(judiRHSexception("info mismatch"))
    isequal(A.geometry.nt,B.geometry.nt) == true || throw(judiRHSexception("sample number mismatch"))
    isequal(A.geometry.dt,B.geometry.dt) == true || throw(judiRHSexception("sample interval mismatch"))
    isequal(A.geometry.t,B.geometry.t) == true || throw(judiRHSexception("recording time mismatch"))

    # Size
    m = A.info.n * sum(A.info.nt)
    n = 1

    # merge geometries and data
    xloc = Array{Any}(undef, A.info.nsrc)
    yloc = Array{Any}(undef, A.info.nsrc)
    zloc = Array{Any}(undef, A.info.nsrc)
    dt = Array{Any}(undef, A.info.nsrc)
    nt = Array{Any}(undef, A.info.nsrc)
    t = Array{Any}(undef, A.info.nsrc)
    data = Array{Any}(undef, A.info.nsrc)

    for j=1:A.info.nsrc
        xloc[j] = [vec(A.geometry.xloc[j])' vec(B.geometry.xloc[j])']
        yloc[j] = [vec(A.geometry.yloc[j])' vec(B.geometry.yloc[j])']
        zloc[j] = [vec(A.geometry.zloc[j])' vec(B.geometry.zloc[j])']
        dt[j] = A.geometry.dt[j]
        nt[j] = A.geometry.nt[j]
        t[j] = A.geometry.t[j]
        data[j] = [A.data[j] B.data[j]]
    end
    geometry = Geometry(xloc,yloc,zloc,dt,nt,t)
    nvDT = promote_type(avDT,bvDT)

    return judiRHS{nvDT}("judiRHS",m,n,A.info,geometry,data)
end

# -(judiRHS,judiRHS)
function -(A::judiRHS{avDT}, B::judiRHS{bvDT}) where {avDT, bvDT}

    # Error checking
    size(A) == size(B) || throw(judiRHSexception("shape mismatch"))
    compareInfo(A.info, B.info) == true || throw(judiRHSexception("info mismatch"))
    isequal(A.geometry.nt,B.geometry.nt) == true || throw(judiRHSexception("sample number mismatch"))
    isequal(A.geometry.dt,B.geometry.dt) == true || throw(judiRHSexception("sample interval mismatch"))
    isequal(A.geometry.t,B.geometry.t) == true || throw(judiRHSexception("recording time mismatch"))

    # Size
    m = A.info.n * sum(A.info.nt)
    n = 1

    # merge geometries and data
    xloc = Array{Any}(undef, A.info.nsrc)
    yloc = Array{Any}(undef, A.info.nsrc)
    zloc = Array{Any}(undef, A.info.nsrc)
    dt = Array{Any}(undef, A.info.nsrc)
    nt = Array{Any}(undef, A.info.nsrc)
    t = Array{Any}(undef, A.info.nsrc)
    data = Array{Any}(undef, A.info.nsrc)

    for j=1:A.info.nsrc
        xloc[j] = [vec(A.geometry.xloc[j])' vec(B.geometry.xloc[j])']
        yloc[j] = [vec(A.geometry.yloc[j])' vec(B.geometry.yloc[j])']
        zloc[j] = [vec(A.geometry.zloc[j])' vec(B.geometry.zloc[j])']
        dt[j] = A.geometry.dt[j]
        nt[j] = A.geometry.nt[j]
        t[j] = A.geometry.t[j]
        data[j] = [A.data[j] -B.data[j]]
    end
    geometry = Geometry(xloc,yloc,zloc,dt,nt,t)
    nvDT = promote_type(avDT,bvDT)

    return judiRHS{nvDT}("judiRHS",m,n,A.info,geometry,data)
end

function subsample(a::judiRHS{avDT},srcnum) where avDT
    info = Info(a.info.n, length(srcnum), a.info.nt[srcnum])
    geometry = subsample(a.geometry,srcnum)     # Geometry of subsampled data container
    return judiRHS(info,geometry,a.data[srcnum];vDT=avDT)
end

getindex(x::judiRHS,a) = subsample(x,a)


## Learning a connect four agent for the JuliaCon 2020 demo

ENV["JULIA_CUDA_MEMORY_POOL"] = "split" # "binned" / "split"

using AlphaZero

include("../games/connect-four/main.jl")
using .ConnectFour: Game, Training

session = Session(
  Game,
  Training.Network{Game},
  Training.params,
  Training.netparams,
  benchmark=Training.benchmark,
  dir="sessions/connect-four",
  autosave=true,
  save_intermediate=false)

resume!(session)


"""
**AnalyticVI**

Variational Inference solver for conjugate or conditionally conjugate likelihoods (non-gaussian are made conjugate via augmentation)
All data is used at each iteration (use AnalyticSVI for Stochastic updates)

```julia
AnalyticVI(;ϵ::T=1e-5)
```
**Keywords arguments**

    - `ϵ::T` : convergence criteria
"""
mutable struct AnalyticVI{T,N} <: VariationalInference{T}
    ϵ::T #Convergence criteria
    nIter::Integer #Number of steps performed
    Stochastic::Bool #Use of mini-batches
    nSamples::Int64
    nMinibatch::Int64 #Size of mini-batches
    ρ::T #Stochastic Coefficient
    HyperParametersUpdated::Bool #To know if the inverse kernel matrix must updated
    vi_opt::NTuple{N,AVIOptimizer}
    MBIndices::Vector{Int64} #Indices of the minibatch
    xview::SubArray{T,2,Matrix{T}}
    yview::AbstractVector

    function AnalyticVI{T}(ϵ::T,optimizer::Optimizer,nMinibatch::Int,Stochastic::Bool) where {T}
        return new{T,1}(ϵ,0,Stochastic,0,nMinibatch,1.0,true,(AVIOptimizer{T}(0,optimizer),))
    end
    function AnalyticVI{T,1}(ϵ::T,Stochastic::Bool,nFeatures::Int,nSamples::Int,nMinibatch::Int,nLatent::Int,optimizer::Optimizer) where {T}
        vi_opts = ntuple(_->AVIOptimizer{T}(nFeatures,optimizer),nLatent)
        new{T,nLatent}(ϵ,0,Stochastic,nSamples,nMinibatch,nSamples/nMinibatch,true,vi_opts,collect(1:nMinibatch))
    end
end

function AnalyticVI(;ϵ::T=1e-5) where {T<:Real}
    AnalyticVI{Float64}(ϵ,VanillaGradDescent(η=1.0),0,false)
end

"""
**AnalyticSVI**
Stochastic Variational Inference solver for conjugate or conditionally conjugate likelihoods (non-gaussian are made conjugate via augmentation)

```julia
AnalyticSVI(nMinibatch::Integer;ϵ::T=1e-5,optimizer::Optimizer=InverseDecay())
```
    - `nMinibatch::Integer` : Number of samples per mini-batches

**Keywords arguments**

    - `ϵ::T` : convergence criteria
    - `optimizer::Optimizer` : Optimizer used for the variational updates. Should be an Optimizer object from the [GradDescent.jl](https://github.com/jacobcvt12/GradDescent.jl) package. Default is `InverseDecay()` (ρ=(τ+iter)^-κ)
"""
function AnalyticSVI(nMinibatch::Integer;ϵ::T=1e-5,optimizer::Optimizer=InverseDecay()) where {T<:Real}
    AnalyticVI{T}(ϵ,optimizer,nMinibatch,true)
end

function Base.show(io::IO,inference::AnalyticVI{T}) where T
    print(io,"Analytic$(inference.Stochastic ? " Stochastic" : "") Variational Inference")
end

## Initialize the final version of the inference object ##
function tuple_inference(i::TInf,nLatent::Integer,nFeatures::Integer,nSamples::Integer,nMinibatch::Integer) where {TInf <: AnalyticVI}
    return TInf(i.ϵ,i.Stochastic,nFeatures,nSamples,nMinibatch,nLatent,i.vi_opt[1].optimizer)
end

## Generic method for variational updates using analytical formulas ##
function variational_updates!(model::AbstractGP{T,L,<:AnalyticVI}) where {T,L}
    local_updates!(model.likelihood,get_y(model),mean_f(model),diag_cov_f(model))
    natural_gradient!.(
        ∇E_μ(model.likelihood,model.inference.vi_opt[1],get_y(model)),
        ∇E_Σ(model.likelihood,model.inference.vi_opt[1],get_y(model)),
        model.inference,model.inference.vi_opt,get_Z(model),model.f)
    global_update!(model)
end

function variational_updates!(model::MOSVGP{T,L,<:AnalyticVI}) where {T,L}
    local_updates!.(model.likelihood,get_y(model),mean_f(model),diag_cov_f(model)) # Compute the local updates given the expectations of f
    # for i in 1:model.nLatent
    #     local_updates!.(model.likelihood,get_y(model),mean_f(model),diag_cov_f(model)) # Compute the local updates given the expectations of f
    #     natural_gradient!(∇E_μ(model)[i],∇E_Σ(model)[i],model.inference,model.inference.vi_opt[i],get_Z(model)[i],model.f[i])
    #     global_update!(model) # Update η₁ and η₂
    # end
    natural_gradient!.(∇E_μ(model), ∇E_Σ(model), model.inference,
        model.inference.vi_opt, get_Z(model), model.f) # Compute the natural gradients of u given the weighted sum of the gradient of f
    global_update!(model) # Update η₁ and η₂
end

## Wrappers for tuple of 1 element
local_updates!(l::Likelihood,y,μ::Tuple{<:AbstractVector{T}},Σ::Tuple{<:AbstractVector{T}}) where {T} = local_updates!(l,y,first(μ),first(Σ))
expec_log_likelihood(l::Likelihood,i::AnalyticVI,y,μ::Tuple{<:AbstractVector{T}},Σ::Tuple{<:AbstractVector{T}}) where {T} = expec_log_likelihood(l,i,y,first(μ),first(Σ))

## Coordinate ascent updates on the natural parameters ##
function natural_gradient!(∇E_μ::AbstractVector,∇E_Σ::AbstractVector,i::AnalyticVI,opt::AVIOptimizer,X::AbstractMatrix,gp::_VGP{T}) where {T,L}
    gp.η₁ .= ∇E_μ .+ gp.K \ gp.μ₀(X)
    gp.η₂ .= -Symmetric(Diagonal(∇E_Σ) + 0.5 * inv(gp.K).mat)
end

#Computation of the natural gradient for the natural parameters
function natural_gradient!(∇E_μ::AbstractVector{T},∇E_Σ::AbstractVector{T},i::AnalyticVI,opt::AVIOptimizer,Z::AbstractMatrix,gp::_SVGP{T}) where {T<:Real,L}
    opt.∇η₁ .= ∇η₁(∇E_μ, i.ρ, gp.κ, gp.K, gp.μ₀(Z), gp.η₁)
    opt.∇η₂ .= ∇η₂(∇E_Σ, i.ρ, gp.κ, gp.K, gp.η₂)
end

function ∇η₁(∇μ::AbstractVector{T},ρ::Real,κ::AbstractMatrix{T},K::PDMat{T,Matrix{T}},μ₀::AbstractVector,η₁::AbstractVector{T}) where {T <: Real}
    transpose(κ)*(ρ * ∇μ) + (K \ μ₀) - η₁
end

function ∇η₂(θ::AbstractVector{T},ρ::Real,κ::AbstractMatrix{<:Real},K::PDMat{T,Matrix{T}},η₂::Symmetric{T,Matrix{T}}) where {T<:Real}
    -(ρκdiagθκ(ρ, κ, θ) + 0.5 * inv(K).mat) - η₂
end

global_update!(model::VGP{T,L,<:AnalyticVI}) where {T,L} = global_update!.(model.f)

global_update!(gp::_VGP,opt::AVIOptimizer,i::AnalyticVI) = global_update!(gp)

#Update of the natural parameters and conversion from natural to standard distribution parameters
function global_update!(model::Union{SVGP{T,L,TInf},MOSVGP{T,L,TInf}}) where {T,L,TInf<:AnalyticVI}
    global_update!.(model.f,model.inference.vi_opt,model.inference)
end

function global_update!(gp::_SVGP,opt::AVIOptimizer,i::AnalyticVI)
    if isstochastic(i)
        Δ = GradDescent.update(opt.optimizer,vcat(opt.∇η₁,opt.∇η₂[:]))
        gp.η₁ .+= Δ[1:gp.dim]
        gp.η₂ .= Symmetric(gp.η₂ + reshape(Δ[(gp.dim+1):end],gp.dim,gp.dim))
    else
        gp.η₁ .+= opt.∇η₁
        gp.η₂ .= Symmetric(opt.∇η₂ + gp.η₂)
    end
    global_update!(gp)
end

function ELBO(model::AbstractGP{T,L,<:AnalyticVI}) where {T,L}
    tot = zero(T)
    tot += model.inference.ρ*expec_log_likelihood(model.likelihood,model.inference,get_y(model),mean_f(model),diag_cov_f(model))
    tot -= GaussianKL(model)
    tot -= model.inference.ρ*AugmentedKL(model.likelihood,get_y(model))
end


#=
Time steps for the MixedSolver module.
Since the FE and FV portions may use different steppers, this must work for both,
but they can be done separately by only passing one set of variables.
For all of them:
input
- fe_var
- fv_var
- vol_lhs
- vol_rhs
- surf_lhs
- surf_rhs
- step_info = (t, dt, dofs_per_node_fe, dofs_per_loop_fe, assemble_func_fe, dofs_per_node_fv, dofs_per_loop_fv, assemble_func_fv)
- fe_storage = (A, b, fe_sol, allocated_vecs, b_order, b_sizes)
- fv_storage = (fv_sol, allocated_vecs_fv)
=#

function mixed_euler_explicit_step(fe_var, fv_var, vol_lhs, vol_rhs, surf_lhs, surf_rhs, t, step_info, fe_storage, fv_storage)
    fe_stepper = step_info[1];
    fv_stepper = step_info[2];
    dofs_per_node_fe = step_info[3];
    dofs_per_loop_fe = step_info[4];
    fe_assemble_func = step_info[5];
    dofs_per_node_fv = step_info[6];
    dofs_per_loop_fv = step_info[7];
    fv_assemble_func = step_info[8];
    dt = fe_stepper.dt;
    assemble_t = 0;
    linsolve_t = 0;

    if !(fv_var === nothing) && config.num_partitions > 1 exchange_ghosts(fv_var, fv_grid, i); end
    pre_step_function();

    ### First step the FE part #######################################################################################
    if !(fe_var === nothing)
        # Unpack things
        A = fe_storage[1];
        b = fe_storage[2];
        fe_sol = fe_storage[3];
        allocated_vecs = fe_storage[4];
        b_order = fe_storage[5];
        b_sizes = fe_storage[6];

        N1 = size(grid_data.allnodes,2);

        assemble_t += @elapsed begin
                                    b = assemble(fe_var, nothing, vol_rhs[1], allocated_vecs, dofs_per_node_fe, dofs_per_loop_fe, t, dt; rhs_only = true, assemble_loops=fe_assemble_func);
                                    b = gather_system(nothing, b, N1, dofs_per_node_fe, b_order, b_sizes);
                                end

        linsolve_t += @elapsed(tmpvec = distribute_solution( linear_system_solve(A,b) , N1, dofs_per_node_fe, b_order, b_sizes));

        # At this point tmpvec holds the boundary values
        # directly write them to the variable values and zero sol.
        copy_bdry_vals_to_variables(fe_var, tmpvec, grid_data, dofs_per_node_fe, zero_vals=true);

        for i=1:length(fe_sol)
            fe_sol[i] = fe_sol[i] + dt * tmpvec[i];
        end

        copy_bdry_vals_to_vector(fe_var, fe_sol, grid_data, dofs_per_node_fe);
        place_sol_in_vars(fe_var, fe_sol);

    end

    ### Then step the FV part ######################################################################################
    if !(fv_var === nothing)
        fv_sol = fv_storage[1];
        fv_allocated_vecs = fv_storage[2];

        assemble_t += @elapsed(tmpvec = fv_assemble(fv_var, vol_lhs[2], vol_rhs[2], surf_lhs[2], surf_rhs[2], fv_allocated_vecs, dofs_per_node_fv, dofs_per_loop_fv, t, dt, assemble_loops=fv_assemble_func));

        for i=1:length(fv_sol)
            fv_sol[i] = fv_sol[i] + dt * tmpvec[i];
        end

        FV_copy_bdry_vals_to_vector(fv_var, fv_sol, fv_grid, dofs_per_node_fv);
        place_sol_in_vars(fv_var, fv_sol);
    end

    post_step_function();

    return (assemble_t, linsolve_t);
end

function mixed_euler_implicit_step(fe_var, fv_var, vol_lhs, vol_rhs, surf_lhs, surf_rhs, t, step_info, fe_storage, fv_storage)
    fe_stepper = step_info[1];
    fv_stepper = step_info[2];
    dofs_per_node_fe = step_info[3];
    dofs_per_loop_fe = step_info[4];
    fe_assemble_func = step_info[5];
    dofs_per_node_fv = step_info[6];
    dofs_per_loop_fv = step_info[7];
    fv_assemble_func = step_info[8];
    dt = fe_stepper.dt;
    assemble_t = 0;
    linsolve_t = 0;

    if !(fv_var === nothing) && config.num_partitions > 1 exchange_ghosts(fv_var, fv_grid, i); end
    pre_step_function();

    ### First step the FE part #######################################################################################
    if !(fe_var === nothing)
        # Unpack things
        A = fe_storage[1];
        b = fe_storage[2];
        fe_sol = fe_storage[3];
        allocated_vecs = fe_storage[4];
        b_order = fe_storage[5];
        b_sizes = fe_storage[6];

        N1 = size(grid_data.allnodes,2);

        assemble_t += @elapsed begin
                                    b = assemble(fe_var, nothing, vol_rhs[1], allocated_vecs, dofs_per_node_fe, dofs_per_loop_fe, t, dt; rhs_only = true, assemble_loops=fe_assemble_func);
                                    b = gather_system(nothing, b, N1, dofs_per_node_fe, b_order, b_sizes);
                                end

        linsolve_t += @elapsed(tmpvec = distribute_solution( linear_system_solve(A,b) , N1, dofs_per_node_fe, b_order, b_sizes));

        # At this point tmpvec holds the boundary values
        # directly write them to the variable values and zero sol.
        copy_bdry_vals_to_variables(fe_var, tmpvec, grid_data, dofs_per_node_fe, zero_vals=true);

        for i=1:length(fe_sol)
            fe_sol[i] = tmpvec[i];
        end

        copy_bdry_vals_to_vector(fe_var, fe_sol, grid_data, dofs_per_node_fe);
        place_sol_in_vars(fe_var, fe_sol);
    end

    ### Then step the FV part ######################################################################################
    if !(fv_var === nothing)
        fv_sol = fv_storage[1];
        fv_allocated_vecs = fv_storage[2];

        ## TODO

        FV_copy_bdry_vals_to_vector(fv_var, fv_sol, fv_grid, dofs_per_node_fv);
        place_sol_in_vars(fv_var, fv_sol);
    end

    post_step_function();

    return (assemble_t, linsolve_t);
end

function mixed_crank_nicholson_step(fe_var, fv_var, vol_lhs, vol_rhs, surf_lhs, surf_rhs, t, step_info, fe_storage, fv_storage)
    fe_stepper = step_info[1];
    fv_stepper = step_info[2];
    dofs_per_node_fe = step_info[3];
    dofs_per_loop_fe = step_info[4];
    fe_assemble_func = step_info[5];
    dofs_per_node_fv = step_info[6];
    dofs_per_loop_fv = step_info[7];
    fv_assemble_func = step_info[8];
    dt = fe_stepper.dt;
    assemble_t = 0;
    linsolve_t = 0;

    if !(fv_var === nothing) && config.num_partitions > 1 exchange_ghosts(fv_var, fv_grid, i); end
    pre_step_function();

    ### First step the FE part #######################################################################################
    if !(fe_var === nothing)
        # Unpack things
        A = fe_storage[1];
        b = fe_storage[2];
        fe_sol = fe_storage[3];
        allocated_vecs = fe_storage[4];
        b_order = fe_storage[5];
        b_sizes = fe_storage[6];

        N1 = size(grid_data.allnodes,2);

        assemble_t += @elapsed begin
            b = assemble(fe_var, nothing, vol_rhs[1], allocated_vecs, dofs_per_node_fe, dofs_per_loop_fe, t, dt; rhs_only = true, assemble_loops=fe_assemble_func);
            b = gather_system(nothing, b, N1, dofs_per_node_fe, b_order, b_sizes);
        end

        linsolve_t += @elapsed(tmpvec = distribute_solution( linear_system_solve(A,b) , N1, dofs_per_node_fe, b_order, b_sizes));

        # At this point tmpvec holds the boundary values
        # directly write them to the variable values and zero sol.
        copy_bdry_vals_to_variables(fe_var, tmpvec, grid_data, dofs_per_node_fe, zero_vals=true);

        for i=1:length(fe_sol)
        fe_sol[i] = tmpvec[i];
        end

        copy_bdry_vals_to_vector(fe_var, fe_sol, grid_data, dofs_per_node_fe);
        place_sol_in_vars(fe_var, fe_sol);
    end

    ### Then step the FV part ######################################################################################
    if !(fv_var === nothing)
        fv_sol = fv_storage[1];
        fv_allocated_vecs = fv_storage[2];

        ## TODO

        FV_copy_bdry_vals_to_vector(fv_var, fv_sol, fv_grid, dofs_per_node_fv);
        place_sol_in_vars(fv_var, fv_sol);
    end

    post_step_function();

    return (assemble_t, linsolve_t);
end

function mixed_lsrk4_step(fe_var, fv_var, vol_lhs, vol_rhs, surf_lhs, surf_rhs, t, step_info, fe_storage, fv_storage)
    fe_stepper = step_info[1];
    fv_stepper = step_info[2];
    dofs_per_node_fe = step_info[3];
    dofs_per_loop_fe = step_info[4];
    fe_assemble_func = step_info[5];
    dofs_per_node_fv = step_info[6];
    dofs_per_loop_fv = step_info[7];
    fv_assemble_func = step_info[8];
    dt = fe_stepper.dt;
    assemble_t = 0;
    linsolve_t = 0;

    ### First step the FE part #######################################################################################
    if !(fe_var === nothing)
        # Unpack things
        A = fe_storage[1];
        b = fe_storage[2];
        fe_sol = fe_storage[3];
        allocated_vecs = fe_storage[4];
        b_order = fe_storage[5];
        b_sizes = fe_storage[6];
        tmppi = fe_storage[7];
        tmpki = fe_storage[8];

        N1 = size(grid_data.allnodes,2);

        # Low storage RK4:
        # p0 = u
        #   ki = ai*k(i-1) + dt*f(p(i-1), t+ci*dt)
        #   pi = p(i-1) + bi*ki
        # u = p5

        tmppi = get_var_vals(fe_var, tmppi);
        for rki=1:fe_stepper.stages
            rktime = t + fe_stepper.c[rki]*fe_stepper.dt;
            # p(i-1) is currently in u
            pre_step_function();

            ### First step the FE part #######################################################################################
            assemble_t += @elapsed begin
                                b = assemble(fe_var, nothing, vol_rhs[1], allocated_vecs, dofs_per_node_fe, dofs_per_loop_fe, rktime, fe_stepper.dt; rhs_only = true, assemble_loops=fe_assemble_func);
                                b = gather_system(nothing, b, N1, dofs_per_node_fe, b_order, b_sizes);
                            end

            linsolve_t += @elapsed(fe_sol = distribute_solution( linear_system_solve(A,b) , N1, dofs_per_node_fe, b_order, b_sizes));

            # At this point sol holds the boundary values
            # directly write them to the variable values and zero sol.
            copy_bdry_vals_to_variables(fe_var, fe_sol, grid_data, dofs_per_node_fe, zero_vals=true);

            if rki == 1 # because a1 == 0
                tmpki = fe_stepper.dt .* fe_sol;
            else
                tmpki = fe_stepper.a[rki].*tmpki + fe_stepper.dt.*fe_sol;
            end
            tmppi = tmppi + fe_stepper.b[rki].*tmpki

            copy_bdry_vals_to_vector(fe_var, tmppi, grid_data, dofs_per_node_fe);
            place_sol_in_vars(fe_var, tmppi);

            post_step_function();
        end

    end

    ### Then step the FV part ######################################################################################
    if !(fv_var === nothing)
        fv_sol = fv_storage[1];
        fv_allocated_vecs = fv_storage[2];
        fv_tmppi = fv_storage[3];
        fv_tmpki = fv_storage[4];

        fv_tmppi = get_var_vals(fv_var, fv_tmppi);
        for rki=1:fv_stepper.stages
            rktime = t + fv_stepper.c[rki]*fv_stepper.dt;
            # p(i-1) is currently in u
            if config.num_partitions > 1 exchange_ghosts(fv_var, fv_grid, i); end
            pre_step_function();

            assemble_t += @elapsed(fv_sol = fv_assemble(fv_var, vol_lhs[2], vol_rhs[2], surf_lhs[2], surf_rhs[2], fv_allocated_vecs, dofs_per_node_fv, dofs_per_loop_fv, rktime, dt, assemble_loops=fv_assemble_func));

            if rki == 1 # because a1 == 0
                fv_tmpki = fv_stepper.dt .* fv_sol;
            else
                fv_tmpki = fv_stepper.a[rki].*fv_tmpki + fv_stepper.dt.*fv_sol;
            end
            fv_tmppi = fv_tmppi + fv_stepper.b[rki].*fv_tmpki

            FV_copy_bdry_vals_to_vector(fv_var, fv_tmppi, fv_grid, dofs_per_node_fv);
            place_sol_in_vars(fv_var, fv_tmppi);

            post_step_function();
        end
    end

    return (assemble_t, linsolve_t);
end

# This may be removed.
function mixed_rk4_step(fe_var, fv_var, vol_lhs, vol_rhs, surf_lhs, surf_rhs, t, step_info, fe_storage, fv_storage)
    return mixed_multistage_step(fe_var, fv_var, vol_lhs, vol_rhs, surf_lhs, surf_rhs, t, step_info, fe_storage, fv_storage);
end

function mixed_multistage_step(fe_var, fv_var, vol_lhs, vol_rhs, surf_lhs, surf_rhs, t, step_info, fe_storage, fv_storage)
    fe_stepper = step_info[1];
    fv_stepper = step_info[2];
    dofs_per_node_fe = step_info[3];
    dofs_per_loop_fe = step_info[4];
    fe_assemble_func = step_info[5];
    dofs_per_node_fv = step_info[6];
    dofs_per_loop_fv = step_info[7];
    fv_assemble_func = step_info[8];
    dt = fe_stepper.dt;
    assemble_t = 0;
    linsolve_t = 0;

    ### First step the FE part #######################################################################################
    if !(fe_var === nothing)
        # Unpack things
        A = fe_storage[1];
        b = fe_storage[2];
        fe_sol = fe_storage[3];
        allocated_vecs = fe_storage[4];
        b_order = fe_storage[5];
        b_sizes = fe_storage[6];
        tmpresult = fe_storage[7];
        tmpki = fe_storage[8];

        N1 = size(grid_data.allnodes,2);

        # solution will be placed in var.values for each stage
        for stage=1:fe_stepper.stages
            stime = t + fe_stepper.c[stage]*fe_stepper.dt;

            # Update the values in vars to be used in this stage
            if stage > 1
                initialized_tmpresult = false;
                for j=1:stage
                    if fe_stepper.a[stage, j] > 0
                        if !initialized_tmpresult
                            initialized_tmpresult = true;
                            for k=1:length(fe_sol)
                                tmpresult[k] = fe_sol[k] + fe_stepper.dt * fe_stepper.a[stage, j] * tmpki[k,j];
                            end
                        else
                            for k=1:length(fe_sol)
                                tmpresult[k] += fe_stepper.dt * fe_stepper.a[stage, j] * tmpki[k,j];
                            end
                        end
                    end
                end

                if initialized_tmpresult
                    copy_bdry_vals_to_vector(fe_var, tmpresult, grid_data, dofs_per_node_fe);
                    place_sol_in_vars(fe_var, tmpresult);
                end
                post_step_function(); # seems weird, but imagine this is happening after stage-1
            end

            pre_step_function();

            ### First step the FE part #######################################################################################
            assemble_t += @elapsed begin
                                b = assemble(fe_var, nothing, vol_rhs[1], allocated_vecs, dofs_per_node_fe, dofs_per_loop_fe, stime, fe_stepper.dt; rhs_only = true, assemble_loops=fe_assemble_func);
                                b = gather_system(nothing, b, N1, dofs_per_node_fe, b_order, b_sizes);
                            end

            linsolve_t += @elapsed(tmpki[:,stage] = distribute_solution( linear_system_solve(A,b) , N1, dofs_per_node_fe, b_order, b_sizes));

            # At this point tmpki[:,stage] holds the boundary values
            # directly write them to the variable values and zero sol.
            copy_bdry_vals_to_variables(fe_var, tmpki[:,stage], grid_data, dofs_per_node_fe, zero_vals=true);

        end
        for i=1:length(fe_sol)
            for stage=1:fe_stepper.stages
                fe_sol[i] += fe_stepper.dt * fe_stepper.b[stage] * tmpki[i, stage];
            end
        end

        copy_bdry_vals_to_vector(fe_var, fe_sol, grid_data, dofs_per_node_fe);
        place_sol_in_vars(fe_var, fe_sol);

        post_step_function();
    end

    ### Then step the FV part ######################################################################################
    if !(fv_var === nothing)
        fv_sol = fv_storage[1];
        fv_allocated_vecs = fv_storage[2];
        fv_tmpresult = fv_storage[3];
        fv_tmpki = fv_storage[4];

        # solution will be placed in var.values for each stage
        for stage=1:fv_stepper.stages
            stime = t + fv_stepper.c[stage]*fv_stepper.dt;

            # Update the values in vars to be used in this stage
            if stage > 1
                initialized_tmpresult = false;
                for j=1:stage
                    if fv_stepper.a[stage, j] > 0
                        if !initialized_tmpresult
                            initialized_tmpresult = true;
                            for k=1:length(fv_sol)
                                fv_tmpresult[k] = fv_sol[k] + fv_stepper.dt * fv_stepper.a[stage, j] * fv_tmpki[k,j];
                            end
                        else
                            for k=1:length(fv_sol)
                                fv_tmpresult[k] += fv_stepper.dt * fv_stepper.a[stage, j] * fv_tmpki[k,j];
                            end
                        end
                    end
                end

                if initialized_tmpresult
                    FV_copy_bdry_vals_to_vector(fv_var, fv_tmpresult, fv_grid, dofs_per_node_fv);
                    place_sol_in_vars(fv_var, fv_tmpresult);
                end
                post_step_function(); # seems weird, but imagine this is happening after stage-1
            end

            if config.num_partitions > 1 exchange_ghosts(fv_var, fv_grid, i); end
            pre_step_function();

            tmpki[:,stage] = fv_assemble(fv_var, vol_lhs[2], vol_rhs[2], surf_lhs[2], surf_rhs[2], fv_allocated_vecs, dofs_per_node_fv, dofs_per_loop_fv, stime, fv_stepper.dt, assemble_loops=fv_assemble_func);
        end
        for i=1:length(fv_sol)
            for stage=1:fv_stepper.stages
                fv_sol[i] += fv_stepper.dt * fv_stepper.b[stage] * fv_tmpki[i, stage];
            end
        end
        FV_copy_bdry_vals_to_vector(fv_var, fv_sol, fv_grid, dofs_per_node_fv);
        place_sol_in_vars(fv_var, fv_sol);

        post_step_function();
    end

    return (assemble_t, linsolve_t);
end


# Replace Flux's LSTMCell with BasicLSTMCell from TensorFlow 1.
# It's implementation is slightly different.
# https://github.com/tensorflow/tensorflow/blob/v2.4.1/tensorflow/python/keras/layers/legacy_rnn/rnn_cell_impl.py#L786-L787
# https://github.com/FluxML/Flux.jl/blob/master/src/layers/recurrent.jl#L141-L142
# The replacement is required because the pre-trained models use BasicLSTMCell.
struct BasicLSTMCell{A,V,S}
    Wi::A
    Wh::A
    b::V
    state0::S
end

function BasicLSTMCell(in::Integer, out::Integer;
                       init = Flux.glorot_uniform,
                       initb = zeros,
                       init_state = zeros)
    cell = BasicLSTMCell(init(out * 4, in), init(out * 4, out), initb(out * 4), (init_state(out,1), init_state(out,1)))
    cell.b[gate(out, 2)] .= 1
    return cell
end

function (m::BasicLSTMCell)((h, c), x) where {A,V,T}
    b, o = m.b, size(h, 1)
    g = m.Wi*x .+ m.Wh*h .+ b
    input = σ.(gate(g, o, 1))
    cell = tanh.(gate(g, o, 2))
    forget = σ.(gate(g, o, 3))
    output = σ.(gate(g, o, 4))
    c = forget .* c .+ input .* cell
    h′ = output .* tanh.(c)
    sz = size(x)
    return (h′, c), reshape(h′, :, sz[2:end]...)
end

Flux.@functor BasicLSTMCell

Base.show(io::IO, l::BasicLSTMCell) =
    print(io, "BasicLSTMCell(", size(l.Wi, 2), ", ", size(l.Wi, 1)÷4, ")")

"""
    BasicLSTM(in::Integer, out::Integer)
Implements the BasicLSTMCell from TensorFlow 1
https://github.com/tensorflow/tensorflow/blob/v2.4.1/tensorflow/python/keras/layers/legacy_rnn/rnn_cell_impl.py#L648-L814
"""
BasicLSTM(a...; ka...) = Flux.Recur(BasicLSTMCell(a...; ka...))
Flux.Recur(m::BasicLSTMCell) = Flux.Recur(m, m.state0)


#------- SET UP DESIGN PARAMETERS --------#

struct plyproperties
    names#::Array{String,1}
    plies::Array{Composites.material,1}
end

struct geometry
    t::Array{Float64,1}
    s::Array{Float64,1}
    x::Array{Float64,1} #x-location of airfoil section (positive x = downstream)
    y::Array{Float64,1} #y-location of airfoil section (positive y = spanwise)
    z::Array{Float64,1} #z-location of airfoil section (positive z = up)
    chord::Array{Float64,1}
    normalchord::Array{Float64,1} #normal chord length
    twist::Array{Float64,1} #airfoil twist in degrees
    sweep::Array{Float64,1}
    dihedral::Array{Float64,1}
    airfoilthickness::Array{Float64,1}
    xaf::Array{Float64,2}
    yaf::Array{Float64,2}
end

struct compstructure
    material::Array{Array{Composites.material,1},1}
    laminate::Array{Composites.laminate,1}
    A::Array{Array{Float64,2},1}
    B::Array{Array{Float64,2},1}
    D::Array{Array{Float64,2},1}
    bucklingstrain::Array{Float64,2}
    precompinput::Array{PreComp.input,1}
    precompoutput::Array{PreComp.output,1}
end

struct designparameters
    # configuration assumptions
    p_n_akima::Int64
    p_radii::Array{Float64,1}
    p_webloc::Array{Float64,1}

    # operating point assumptions
    altitude::Float64 #m

    # material properties
    plyprops::plyproperties
    p_usedmaterials
    p_orientation::Array{Float64,1}

    # Propeller Design
    p_lam_t::Array{Float64,1}

    p_chord::Array{Float64,1}
    p_sweep_d::Array{Float64,1}
    p_twist_d::Array{Float64,1}
    p_pitch_d::Float64
    p_dihedral_d::Array{Float64,1}
    p_airfoilthickness::Array{Float64,1}
    p_aerocenter::Array{Float64,1}
    p_x_offset::Array{Float64,1}
    velocity::Float64
    p_Rtip::Float64
    blades::Float64
    numprops::Int64
    machtip::Float64
    p_rpm::Float64
    kv::Float64
    i0::Float64
    batt_mass::Float64

    # Wing Design
    w_n_linear::Int64
    w_webloc::Array{Float64,1}
    w_lam_t::Array{Float64,1}
    w_chord::Array{Float64,1}
    w_aoa_d::Float64
    w_twist_d::Array{Float64,1}
    w_halfspan::Float64
    w_nondim_halfspany::Array{Float64,1}
    w_sweep_d::Array{Float64,1}
    w_dihedral_d::Array{Float64,1}
    w_usedmaterials #array of strings???
    w_orientation::Array{Float64,1}
    w_x_offset::Array{Float64,1}
    w_aerocenter::Array{Float64,1}

    n_CP::Float64
    payload::Float64
    max_mass::Float64
    gravity::Float64
    h_set::Float64
    specif_energy::Float64
    required_range::Float64
    required_takeoff_dist::Float64
    altitude_accel::Float64
    altitude_climb::Float64
    altitude_cruise::Float64
    required_noise::Float64
    required_cruise::Float64

end

function designparameters(;
    p_n_akima = 20,
    p_radii::Array{Float64,1}=Float64[0.0,0.2,.4,.6,.8,1.0],
    p_webloc::Array{Float64,1}=[1.0],
    altitude::Float64=18288.0,
    plyprops::plyproperties=plyproperties(),
    p_usedmaterials = ["highmodulus_uni","highmodulus_weave","taylor_foam"],
    p_orientation=[0.0,-45],
    # structural design variables
    p_lam_t = ones(length(p_radii)*length(p_usedmaterials)),
    # geometric design variables
    p_chord = Float64[0.1, 0.1, 0.1, 0.1, 0.1],
    p_sweep_d = Float64[0.0,0.0,0.0,0.0,0.0],
    p_twist_d = Float64[30.0, 30.0, 30.0, 30.0],
    p_pitch_d = 0.0,
    p_dihedral_d = fill(0.0,5),
    p_airfoilthickness = Float64[0.147538, 0.132307, 0.127404, 0.130604, 0.245836, 0.0629818],
    p_aerocenter = ones(p_radii)*0.25,
    p_x_offset = zeros(p_radii),
    # operating point design variables
    velocity = 15.40662113981876,
    # propulsion design variables
    p_Rtip = 2.25,
    blades = 2,
    numprops = 4,
    machtip = 0.8,
    p_rpm = 1020.45,
    kv = 4.271513933940295,
    i0 = 0.1,
    batt_mass = 1E8,

    w_n_linear = 20,
    w_halfspan = 10.0,
    w_webloc::Array{Float64,1}=[0.40],
    w_usedmaterials = ["highmodulus_uni","highmodulus_weave","taylor_foam"],
    w_nondim_halfspany = linspace(0,w_halfspan,length(w_chord)),
    w_lam_t = ones(length(w_nondim_halfspany)*length(w_usedmaterials)),
    w_chord = [0.5,0.5],
    w_aoa_d = 0.0,
    w_twist_d = [0.0,0.0],
    w_sweep_d = [0.0,0.0],
    w_dihedral_d = [0.0,0.0],
    w_orientation = [0.0,45.0,45.0], #uni, weave, webweave
    w_x_offset = ones(w_nondim_halfspany)*0.25,
    w_aerocenter = ones(w_nondim_halfspany)*0.275,

    n_CP = 60.0,
    payload = 684.0,
    max_mass = 1500.0,
    gravity = 9.81,
    h_set = 20.0, #meters
    specif_energy = 300.0, #wh/kg
    required_range = 150000.0, #km
    required_takeoff_dist = 395.0, #meters
    altitude_accel = 0.0,
    altitude_climb = 0.0,
    altitude_cruise = 3000.0,
    required_noise = 68.0,
    required_cruise = 28.0,

    )

    parameters = designparameters(p_n_akima,p_radii,p_webloc,altitude,plyprops,p_usedmaterials,p_orientation,p_lam_t,
    p_chord,p_sweep_d,p_twist_d,p_pitch_d,p_dihedral_d,p_airfoilthickness,p_aerocenter,p_x_offset,velocity,p_Rtip,
    blades,numprops,machtip,p_rpm,kv,i0,batt_mass,w_n_linear,w_webloc,w_lam_t,w_chord,w_aoa_d,w_twist_d,w_halfspan,w_nondim_halfspany,w_sweep_d,w_dihedral_d,w_usedmaterials,
    w_orientation,w_x_offset,w_aerocenter,n_CP,payload,max_mass,gravity,h_set,
    specif_energy,required_range,required_takeoff_dist,altitude_accel,altitude_climb,
    altitude_cruise,required_noise,required_cruise)
    return parameters
end

struct propwingdesign
    #Design Variables at this scope
    batt_mass::Float64
    w_aoa::Float64
    p_rpm::Float64
    p_pitch::Float64
    velocity::Float64

    #Design Parameters at this scope
    constraints
    af
    p_xaf
    p_yaf
    p_xafstrain
    p_yafstrain
    w_xaf
    w_yaf
    w_xafstrain
    w_yafstrain
    spline_3D_freestream
    spline_3D_blown

    # Design variables unique to the system function
    p_lam_tin
    p_orientation::Array{Float64,1}
    p_chord_pts::Array{Float64,1}
    p_twist_pts::Array{Float64,1}
    p_Rtip::Float64
    kv::Float64
    i0::Float64
    w_chord_pts::Array{Float64,1}
    w_twist_pts::Array{Float64,1}
    w_halfspan::Float64
    w_sweep_pts::Array{Float64,1}
    w_dihedral_pts::Array{Float64,1}
    w_lam_tin
    w_orientation::Array{Float64,1}
    w_airfoilthickness::Array{Float64,1}

    #Design variables unique to the system function
    etaprop_center::Array{Float64,1}
    p_n_akima::Int64
    p_radii::Array{Float64,1}
    altitude::Float64
    numprops::Int64
    p_x_offset::Array{Float64,1}
    p_aerocenter::Array{Float64,1}
    p_usedmaterials
    p_webloc::Array{Float64,1}
    plyprops
    blades::Float64
    w_n_linear::Int64
    w_x_offset::Array{Float64,1}
    w_aerocenter::Array{Float64,1}
    w_nondim_halfspany::Array{Float64,1}
    w_usedmaterials
    w_webloc::Array{Float64,1}
    prop_tilt::Float64

    #Control parameters
    detailed::Bool
    VTKfilename
    savefinalvtk::Bool
    verification::Bool
    plots::Bool
    n_CP::Int64

end

function propwingdesign(;
    #Design Variables at this scope
    batt_mass = 0.0,
    w_aoa = 0.0,
    p_rpm = 0.0,
    p_pitch = 0.0,
    velocity = 0.0,

    #Design Parameters at this scope
    constraints = [],
    af = [],
    p_xaf = [],
    p_yaf = [],
    p_xafstrain = [],
    p_yafstrain = [],
    w_xaf = [],
    w_yaf = [],
    w_xafstrain = [],
    w_yafstrain = [],
    spline_3D_freestream = [],
    spline_3D_blown = [],

    # Design variables unique to the system function
    p_lam_tin= [],
    p_orientation = zeros(2),
    p_chord_pts = zeros(2),
    p_twist_pts = zeros(2),
    p_Rtip =0.0,
    kv =0.0,
    i0 =0.0,
    w_chord_pts = zeros(2),
    w_twist_pts = zeros(2),
    w_halfspan =0.0,
    w_sweep_pts = zeros(2),
    w_dihedral_pts = zeros(2),
    w_lam_tin= [],
    w_orientation = zeros(2),
    w_airfoilthickness = zeros(2),

    #Design variables unique to the system function
    etaprop_center = zeros(2),
    p_n_akima = 10 ,
    p_radii = zeros(2),
    altitude =0.0,
    numprops = 10 ,
    p_x_offset = zeros(2),
    p_aerocenter = zeros(2),
    p_usedmaterials = [],
    p_webloc = zeros(2),
    plyprops = [],
    blades = 10 ,
    w_n_linear = 10 ,
    w_x_offset = zeros(2),
    w_aerocenter = zeros(2),
    w_nondim_halfspany = zeros(2),
    w_usedmaterials = [],
    w_webloc = zeros(2),
    prop_tilt = 0.0,

    #Control parameters
    detailed = false,
    VTKfilename = "notdefined.vtk",
    savefinalvtk = false,
    verification = false,
    plots = false,
    n_CP = 60,
    )

    return propwingdesign(batt_mass, w_aoa, p_rpm, p_pitch, velocity,
        constraints, af, p_xaf, p_yaf, p_xafstrain, p_yafstrain, w_xaf, w_yaf,
        w_xafstrain, w_yafstrain, spline_3D_freestream, spline_3D_blown,
        p_lam_tin, p_orientation, p_chord_pts, p_twist_pts, p_Rtip, kv, i0,
        w_chord_pts, w_twist_pts, w_halfspan, w_sweep_pts, w_dihedral_pts,
         w_lam_tin, w_orientation, w_airfoilthickness, etaprop_center,
        p_n_akima, p_radii, altitude, numprops, p_x_offset, p_aerocenter,
        p_usedmaterials, p_webloc, plyprops, blades, w_n_linear, w_x_offset,
        w_aerocenter, w_nondim_halfspany, w_usedmaterials, w_webloc, prop_tilt,
        detailed, VTKfilename, savefinalvtk, verification, plots, n_CP)
end

struct propwing_out
    #Propulsion
    ETA_total::Float64
    ETA_m::Float64
    ETA_p::Float64
    # Propeller
    CT::Float64
    CQ::Float64
    Np::Array{Float64,1}
    Tp::Array{Float64,1}
    p_alphas::Array{Float64,1}
    uprop::Array{Float64,1}
    vprop::Array{Float64,1}
    total_thrust::Float64
    advance_ratio::Float64
    torque::Float64
    db_test::Array{Float64,1}

    # Motor
    V_m::Float64
    I_m::Float64
    # Masses
    propwing_mass::Float64
    mass_prop::Float64
    mass_ESC::Float64
    mass_motor::Float64
    mass_wing::Float64

    # Wing
    panels#::Array{Float64,1}
    Lift::Float64
    Drag::Float64
    viscousDrag::Float64
    a::Float64
    Fp#::Array{Float64,1}
    cllocal::Array{Float64,1}
    cl::Array{Float64,1}
    Vinfeff::Array{Float64,1}
    alphaeff::Array{Float64,1}

    #Structures
    p_c_stress #::Array{Float64,2}
    w_c_stress #::Array{Float64,2}
    p_c_buckling #::Array{Float64,2}
    w_c_buckling #::Array{Float64,2}

    fail::Bool
end

function propwing_out(;
    #Propulsion
    ETA_total = 0.0,
    ETA_m = 0.0,
    ETA_p = 0.0,
    # Propeller
    CT = 0.0,
    CQ = 0.0,
    Np = zeros(2),
    Tp = zeros(2),
    p_alphas = zeros(2),
    uprop = zeros(2),
    vprop = zeros(2),
    total_thrust = 0.0,
    advance_ratio = 0.0,
    torque = 0.0,
    db_test = zeros(2),

    # Motor
    V_m = 0.0,
    I_m = 0.0,
    # Masses
    propwing_mass = 0.0,
    mass_prop = 0.0,
    mass_ESC = 0.0,
    mass_motor = 0.0,
    mass_wing = 0.0,

    # Wing
    panels = zeros(2),
    Lift = 0.0,
    Drag = 0.0,
    viscousDrag = 0.0,
    a = 0.0,
    Fp = [],# = zeros(2)
    cllocal = zeros(2),
    cl = zeros(2),
    Vinfeff = zeros(2),
    alphaeff = zeros(2),

    #Structures
    p_c_stress = zeros(2),
    w_c_stress = zeros(2),
    p_c_buckling = zeros(2),
    w_c_buckling = zeros(2),
    fail = false,
)

    return propwing_out(ETA_total,ETA_m,ETA_p,CT,CQ,Np,Tp,p_alphas,uprop,vprop,total_thrust,
    advance_ratio,torque,db_test,V_m,I_m,propwing_mass,mass_prop,mass_ESC,
    mass_motor,mass_wing,panels,Lift,Drag,viscousDrag,a,Fp,cllocal,cl,Vinfeff,alphaeff,
    p_c_stress,w_c_stress,p_c_buckling,w_c_buckling, fail)

end

struct system_output
    range_true::Float64
    dist_accel::Float64
    dist_climb::Float64

    energy_accel::Float64
    energy_climb::Float64
    energy_cruise::Float64

    t_accel::Float64
    t_transition::Float64
    t_climb::Float64
    t_cruise::Float64

    total_mass::Float64
    total_weight::Float64

    gama::Float64
    radius::Float64
    xtr::Float64
    ytr::Float64
    H_left::Float64
    x_left::Float64

    optconvals::Dict{Symbol,Array{Float64,1}}

    payload::Float64
    max_mass::Float64
    gravity::Float64
    h_set::Float64
    specif_energy::Float64
    required_takeoff_dist::Float64
    altitude_accel::Float64
    altitude_climb::Float64
    altitude_cruise::Float64
    required_noise::Float64
    etaprop_center::Array{Float64,1}
    batt_mass_accel::Float64
    batt_mass_climb::Float64
    batt_mass_cruise::Float64
    Fl::Float64
end

function system_output(;
    range_true = 0.0,
    dist_accel = 0.0,
    dist_climb = 0.0,

    energy_accel = 0.0,
    energy_climb = 0.0,
    energy_cruise = 0.0,

    t_accel = 0.0,
    t_transition = 0.0,
    t_climb = 0.0,
    t_cruise = 0.0,

    total_mass = 0.0,
    total_weight = 0.0,

    gama = 0.0,
    radius = 0.0,
    xtr = 0.0,
    ytr = 0.0,
    H_left = 0.0,
    x_left = 0.0,

    optconvals = Dict{Symbol,Array{Float64,1}}(),

    payload = 0.0,
    max_mass = 0.0,
    gravity = 0.0,
    h_set = 0.0,
    specif_energy = 0.0,
    required_takeoff_dist = 0.0,
    altitude_accel = 0.0,
    altitude_climb = 0.0,
    altitude_cruise = 0.0,
    required_noise = 0.0,
    etaprop_center = [0.0,0.0],
    batt_mass_accel = 0.0,
    batt_mass_climb = 0.0,
    batt_mass_cruise = 0.0,
    Fl = 0.0,
    )

    return system_output(range_true,dist_accel,dist_climb, energy_accel,
    energy_climb,energy_climb, t_accel,t_transition,t_climb,t_cruise,
    total_mass,total_weight, gama,radius,xtr,ytr,H_left,x_left, optconvals,
    payload,max_mass,gravity,h_set,specif_energy,required_takeoff_dist,
    altitude_accel,altitude_climb,altitude_cruise,required_noise,
    etaprop_center,batt_mass_accel,batt_mass_climb,batt_mass_cruise,Fl)

end

# This file sets up default values and bounds for design variables
function getoptxbounds!(paramdict,designparams)

    ##############################################################################
    #----------------------- Prop Design Variables ------------------------#
    ##############################################################################

    lb_scale = 1.0 #1E-6
    ub_scale = 1E3
    # Skin Fabric Ply Thickness Design Variables
    p_nstations = length(designparams.p_radii)
    matnames = designparams.plyprops.names
    p_nplies = length(designparams.p_usedmaterials)

    name = :p_lam_t
    default = zeros(designparams.p_lam_t)
    lowerbound = zeros(designparams.p_lam_t)
    upperbound = zeros(designparams.p_lam_t)
    scaling = zeros(designparams.p_lam_t)
    for i = 1:p_nplies
        idx = find(matnames -> matnames == designparams.p_usedmaterials[i],matnames)
        material = designparams.plyprops.plies[idx[1]]

        default1 = designparams.p_lam_t[(i-1)*p_nstations+1:(i)*p_nstations]
        lowerbound1 = fill(designparams.plyprops.plies[idx[1]].t*lb_scale,p_nstations)
        upperbound1 = fill(designparams.plyprops.plies[idx[1]].t*ub_scale,p_nstations)
        scaling1 = fill(0.01/designparams.plyprops.plies[idx[1]].t,p_nstations)

        default[(i-1)*p_nstations+1:(i)*p_nstations] = default1
        lowerbound[(i-1)*p_nstations+1:(i)*p_nstations] = lowerbound1
        upperbound[(i-1)*p_nstations+1:(i)*p_nstations] = upperbound1
        scaling[(i-1)*p_nstations+1:(i)*p_nstations] = scaling1

    end
    OptParams.addparam!(paramdict,name,default,lowerbound,upperbound,scaling)

    name = :p_orientation
    default = designparams.p_orientation
    lowerbound = fill(-90.0,length(default))
    upperbound = fill(90.0,length(default))
    scaling = fill(1.0./90.0,length(default))
    OptParams.addparam!(paramdict,name,default,lowerbound,upperbound,scaling)

    name = :p_chord
    default = designparams.p_chord
    lowerbound = fill(0.01,p_nstations)
    lowerbound[end] = 0.001 #tip, works well for splines
    upperbound = fill(0.8,p_nstations)
    scaling = fill(1E2,p_nstations)
    OptParams.addparam!(paramdict,name,default,lowerbound,upperbound,scaling)

    name = :p_sweep_d
    default = designparams.p_sweep_d
    lowerbound = fill(0.0,p_nstations-1)
    upperbound = fill(45.0,p_nstations-1)
    scaling = fill(pi/180.0,p_nstations-1)
    OptParams.addparam!(paramdict,name,default,lowerbound,upperbound,scaling)

    OptParams.addparam!(paramdict,:p_pitch_d_accel, designparams.p_pitch_d*10.0, -25.0, 45.0, 1E1)
    OptParams.addparam!(paramdict,:p_pitch_d_climb, designparams.p_pitch_d*10.0, -25.0, 45.0, 1E0)
    OptParams.addparam!(paramdict,:p_pitch_d_cruise, designparams.p_pitch_d*-2, -5.0, 80.0, 1E2)

    name = :p_twist_d
    default = designparams.p_twist_d
    lowerbound = fill(-0.0,p_nstations)
    upperbound = fill(45.0,p_nstations)
    scaling = fill(0.1,p_nstations)
    OptParams.addparam!(paramdict,name,default,lowerbound,upperbound,scaling)

    name = :p_dihedral_d
    default = designparams.p_dihedral_d
    lowerbound = fill(0.0,p_nstations-1)
    upperbound = fill(10.0,p_nstations-1)
    scaling = fill(pi/180.0,p_nstations-1)
    OptParams.addparam!(paramdict,name,default,lowerbound,upperbound,scaling)

    name = :p_airfoilthickness
    default = designparams.p_airfoilthickness
    lowerbound = fill(0.050,p_nstations)
    upperbound = fill(0.250,p_nstations)
    scaling = fill(10.0,p_nstations)
    OptParams.addparam!(paramdict,name,default,lowerbound,upperbound,scaling)

    OptParams.addparam!(paramdict,:p_Rtip,designparams.p_Rtip,0.1,5.0,1E-1)

    ##############################################################################
    #----------------------- Wing Design Variables ------------------------#
    ##############################################################################

    # Skin Fabric Ply Thickness Design Variables
    w_nstations = length(designparams.w_chord)

    name = :w_lam_t
    default = zeros(designparams.w_lam_t)
    lowerbound = zeros(designparams.w_lam_t)
    upperbound = zeros(designparams.w_lam_t)
    scaling = zeros(designparams.w_lam_t)
    for i = 1:length(designparams.w_usedmaterials)
        idx = find(matnames -> matnames == designparams.w_usedmaterials[i],matnames)
        material = designparams.plyprops.plies[idx[1]]

        default1 = designparams.w_lam_t[(i-1)*w_nstations+1:(i)*w_nstations]
        lowerbound1 = fill(designparams.plyprops.plies[idx[1]].t*lb_scale,w_nstations)
        upperbound1 = fill(designparams.plyprops.plies[idx[1]].t*ub_scale,w_nstations)
        scaling1 = fill(0.01/designparams.plyprops.plies[idx[1]].t,w_nstations)

        default[(i-1)*w_nstations+1:(i)*w_nstations] = default1
        lowerbound[(i-1)*w_nstations+1:(i)*w_nstations] = lowerbound1
        upperbound[(i-1)*w_nstations+1:(i)*w_nstations] = upperbound1
        scaling[(i-1)*w_nstations+1:(i)*w_nstations] = scaling1

    end
    OptParams.addparam!(paramdict,name,default,lowerbound,upperbound,scaling)

    name = :w_orientation
    default = designparams.w_orientation
    lowerbound = fill(-90.0,length(default))
    upperbound = fill(90.0,length(default))
    scaling = fill(1.0./90.0,length(default))
    OptParams.addparam!(paramdict,name,default,lowerbound,upperbound,scaling)

    name = :w_chord
    default = designparams.w_chord
    lowerbound = fill(0.1,w_nstations)
    upperbound = fill(5.0,w_nstations)
    scaling = fill(100.0/designparams.w_halfspan,w_nstations)
    OptParams.addparam!(paramdict,name,default,lowerbound,upperbound,scaling)

    OptParams.addparam!(paramdict,:w_aoa_d_accel,designparams.w_aoa_d*1.1,2.0,40.0,1E-2)
    OptParams.addparam!(paramdict,:w_aoa_d_climb,designparams.w_aoa_d*0.5,5.0,40.0,1E0)
    OptParams.addparam!(paramdict,:w_aoa_d_cruise,designparams.w_aoa_d*0.5,0.20,20.0,1E0)

    name = :w_twist_d
    default = designparams.w_twist_d
    lowerbound = fill(-10.0,w_nstations)
    upperbound = fill(20.0,w_nstations)
    scaling = fill(10.0,w_nstations)
    OptParams.addparam!(paramdict,name,default,lowerbound,upperbound,scaling)

    name = :w_halfspan
    default = designparams.w_halfspan
    lowerbound = 5.699
    upperbound = 12.0
    scaling = 1.0/designparams.w_halfspan
    OptParams.addparam!(paramdict,name,default,lowerbound,upperbound,scaling)

    name = :w_sweep_d
    default = designparams.w_sweep_d
    lowerbound = fill(0.0,w_nstations)
    upperbound = fill(20.0,w_nstations)
    scaling = fill(10.0,w_nstations)
    OptParams.addparam!(paramdict,name,default,lowerbound,upperbound,scaling)

    name = :w_dihedral_d
    default = designparams.w_dihedral_d
    lowerbound = fill(0.0,w_nstations)
    upperbound = fill(20.0,w_nstations)
    scaling = fill(10.0,w_nstations)
    OptParams.addparam!(paramdict,name,default,lowerbound,upperbound,scaling)

    name = :w_airfoilthickness
    default = designparams.p_airfoilthickness
    lowerbound = fill(0.050,p_nstations)
    upperbound = fill(0.250,p_nstations)
    scaling = fill(10.0,p_nstations)
    OptParams.addparam!(paramdict,name,default,lowerbound,upperbound,scaling)

    ##############################################################################
    #-----------------  Operating Point Design Variables ------------------------#
    ##############################################################################

    OptParams.addparam!(paramdict,:velocity_accel,designparams.velocity/1.4,5.0,28.0,1E1)
    OptParams.addparam!(paramdict,:velocity_climb,designparams.velocity*1.0,5.0,60.0,1E-2)
    OptParams.addparam!(paramdict,:velocity_cruise,designparams.velocity*1.0,20.0,100.0,1E1)

    ##############################################################################
    #-------------------------  Propulsion Design Variables ---------------------#
    ##############################################################################

    OptParams.addparam!(paramdict,:p_rpm_accel,designparams.p_rpm*4.5,200.0,40000.0,1E-1)
    OptParams.addparam!(paramdict,:p_rpm_climb,designparams.p_rpm*4.6,200.0,40000.0,1E-2)
    OptParams.addparam!(paramdict,:p_rpm_cruise,designparams.p_rpm*2.5,100.0,10000.0,1E0)
    OptParams.addparam!(paramdict,:kv,designparams.kv,5.0,5000.0,1E-0)
    OptParams.addparam!(paramdict,:i0,designparams.i0,0.1,6.0,1E0)
    OptParams.addparam!(paramdict,:batt_mass_accel,designparams.batt_mass,0.001,100.0,1E0)
    OptParams.addparam!(paramdict,:batt_mass_climb,designparams.batt_mass,0.01,100.0,1E-1)
    OptParams.addparam!(paramdict,:batt_mass_cruise,designparams.batt_mass,5.0,1000.0,1E-1)
    OptParams.addparam!(paramdict,:number_blades,2.0,2.0,5.0,1E-0)
    OptParams.addparam!(paramdict,:radius,100.0,2.0,2000.0,1E-1)
    OptParams.addparam!(paramdict,:gama_d,30.0,0.1,89.9,1E-1)
end

#------ SET UP OPTIMIZATION PARAMETERS -------#

mutable struct controlparams
    objective::Symbol
    printfreq::Int
    savevtk::Int
    printobjective::Bool
    printdetailedoutput::Bool
    printconstraintviolations::Bool
end

function controlparams(;
    objective::Symbol = :mass,
    printfreq::Int = 1,
    savevtk::Int = 3,
    printobjective::Bool = true,
    printdetailedoutput::Bool = true,
    printconstraintviolations::Bool = true,
    )
    controlparams(objective,printfreq,savevtk,printobjective,printdetailedoutput,
    printconstraintviolations)
end

# Define design variables
function getoptparams()
    designvar =  [
    :p_lam_t, #1-12
    # :p_orientation,
    :p_chord, #13-18
    :p_twist_d, #19-24
    :p_pitch_d_accel, #25
    :p_pitch_d_climb, #25
    :p_pitch_d_cruise, #25

    :velocity_accel,
    :velocity_climb,
    :velocity_cruise,
    :p_Rtip,
    :p_rpm_accel,
    :p_rpm_climb,
    :p_rpm_cruise,
    # :kv, #28
    # :i0, #29

    # :w_orientation,
    :w_chord, #30-32
    :w_aoa_d_accel,   #multi
    :w_aoa_d_climb,   #multi
    :w_aoa_d_cruise,   #multi
    # :w_twist_d, #33-35
    # :w_halfspan,
    # :w_sweep_d,
    # :w_dihedral_d,
    # :w_lam_t, #36-end
    :batt_mass_accel,
    :batt_mass_climb,
    :batt_mass_cruise,
    ]

    # Define general constraints
    constraints = [
    :p_materialfailure,
    :p_localbuckling,
    # :w_materialfailure,
    # :w_localbuckling,

    :totalmass,
    :machtip,
    :Lift,
    :Drag,
    :noise,
    # :p_chordlamthick,
    # :range,
    :takeoff_dist,
    :stall,
    :capacity,
    :p_alphas,
    ]

    # define run controls
    ctlparams = controlparams(
    objective= :range, #:mass,#:range #:ETA_total_cruise,
    printfreq = 50,
    savevtk = 500,
    printobjective=true,
    printdetailedoutput=true,
    printconstraintviolations=true,
    )

    return designvar, constraints, ctlparams
end

#------- MATERIAL PROPERTIES -------#
function plyproperties()
    names = ["highmodulus_uni",
    "standard_uni",
    "MR60H",
    "T3900_uni",
    "T700_uni",
    "ELT5500",
    "UDCarbon",
    "highmodulus_weave",
    "standard_weave",
    "T3900_weave",
    "T700_weave",
    "Gelcoat",
    "Triax",
    "Saertex",
    "taylor_foam",
    "SNL_foam"]

    e1  =zeros(16)
    e2  =zeros(16)
    g12 =zeros(16)
    anu =zeros(16)
    rho =zeros(16)
    xt  =zeros(16)
    xc  =zeros(16)
    yt  =zeros(16)
    yc  =zeros(16)
    s   =zeros(16)
    plythickness =zeros(16)

    # "highmodulus_uni"
    e1[1]  = 175.0e9
    e2[1]  = 8.0e9
    g12[1] = 5.0e9
    anu[1] = 0.30
    rho[1] = 1600.0
    xt[1]  = min(1.0,1355.656/1682.011)*1000e6 #mean -> A-basis
    xc[1]  = min(1.0,1103.943/1396.504)*850e6 #mean -> A-basis
    yt[1]  = min(1.0,39.226/52.975)*40e6 #mean -> A-basis
    yc[1]  = min(1.0,235.434/282.439)*200e6 #mean -> A-basis
    s[1]   = min(1.0,142.411/159.516)*60e6 #mean -> A-basis
    plythickness[1] = 0.152e-3
    # "standard_uni"
    e1[2]  = 135.0e9
    e2[2]  = 10.0e9
    g12[2] = 5.0e9
    anu[2] = 0.30
    rho[2] = 1600.0
    xt[2]  = min(1.0,1355.656/1682.011)*1500e6 #mean -> A-basis
    xc[2]  = min(1.0,1103.943/1396.504)*1200e6 #mean -> A-basis
    yt[2]  = min(1.0,39.226/52.975)*50e6 #mean -> A-basis
    yc[2]  = min(1.0,235.434/282.439)*250e6 #mean -> A-basis
    s[2]   = min(1.0,142.411/159.516)*70e6 #mean -> A-basis
    plythickness[2] = 0.152e-3
    # "MR60H"
    e1[3]  = (165e9+150e9)/2.0
    e2[3]  = 8.56e9
    g12[3] = 4.39e9
    anu[3] = 0.326
    rho[3] = 1810.0
    xt[3]  = min(1.0,1355.656/1682.011)*3190e6 #mean -> A-basis
    xc[3]  = min(1.0,1103.943/1396.504)*1440e6 #mean -> A-basis
    yt[3]  = min(1.0,39.226/52.975)*82.0e6 #mean -> A-basis
    yc[3]  = min(1.0,235.434/282.439)*200.0e6 #mean -> A-basis
    s[3]   = min(1.0,142.411/159.516)*141e6 #mean -> A-basis
    plythickness[3] = 0.152e-3
    # "T3900_uni"
    e1[4]  = (148e9+131e9)/2.0
    e2[4]  = (9.7e9+9.7e9)/2.0
    g12[4] = 4.83e9
    anu[4] = 0.33
    rho[4] = 1573.0
    xt[4]  = min(1.0,1355.656/1682.011)*2830e6 #CTD mean -> A-basis
    xc[4]  = min(1.0,1103.943/1396.504)*1772e6 #CTD mean -> A-basis
    yt[4]  = min(1.0,39.226/52.975)*56.9e6 #CTD mean -> A-basis
    yc[4]  = min(1.0,235.434/282.439)*303e6 #CTD mean -> A-basis
    s[4]   = min(1.0,142.411/159.516)*89.6e6 #CTD mean -> A-basis
    plythickness[4] = 0.191e-3
    # "T700_uni"
    e1[5]=120.8e9
    e2[5]=11.57e9
    g12[5]=5.219e9
    anu[5]=0.350
    rho[5]=1525.0
    xt[5]=1356e6
    xc[5]=1104e6
    yt[5]=39.23e6
    yc[5]=235.4e6
    s[5]=142.4e6
    plythickness[5] = 0.152e-3
    # "ELT5500"
    e1[6]=41.8e9
    e2[6]=14.00e9
    g12[6]=2.630e9
    anu[6]=0.280
    rho[6]=1920.0
    xt[6]=972.0e6
    xc[6]=702.0e6
    yt[6]=100.0e6 #made up
    yc[6]=100.0e6 #made up
    s[6]=100.0e6 #made up
    plythickness[6] = 0.47e-3
    # "UDCarbon"
    e1[7]=114.5e9
    e2[7]=8.39e9
    g12[7]=5.990e9
    anu[7]=0.270
    rho[7]=1220.0
    xt[7]=1546.0e6
    xc[7]=1047.0e6
    yt[7]=100.0e6 #made up
    yc[7]=100.0e6 #made up
    s[7]=100.0e6 #made up
    plythickness[7] = 0.47e-3

    # FABRICS
    # "highmodulus_weave"
    e1[8]  = 85.0e9
    e2[8]  = 85.0e9
    g12[8] = 5.0e9
    anu[8] = 0.10
    rho[8] = 1600.0
    xt[8]  = min(1.0,1355.656/1682.011)*350e6 #mean -> A-basis
    xc[8]  = min(1.0,1103.943/1396.504)*150e6 #mean -> A-basis
    yt[8]  = min(1.0,39.226/52.975)*350e6 #mean -> A-basis
    yc[8]  = min(1.0,235.434/282.439)*150e6 #mean -> A-basis
    s[8]   = min(1.0,142.411/159.516)*35e6 #mean -> A-basis
    plythickness[8] = 0.218e-3
    # "standard_weave"
    e1[9]  = 70.0e9
    e2[9]  = 70.0e9
    g12[9] = 5.0e9
    anu[9] = 0.10
    rho[9] = 1600.0
    xt[9]  = min(1.0,1355.656/1682.011)*600e6 #mean -> A-basis
    xc[9]  = min(1.0,1103.943/1396.504)*570e6 #mean -> A-basis
    yt[9]  = min(1.0,39.226/52.975)*600e6 #mean -> A-basis
    yc[9]  = min(1.0,235.434/282.439)*570e6 #mean -> A-basis
    s[9]   = min(1.0,142.411/159.516)*90e6 #mean -> A-basis
    plythickness[9] = 0.218e-3
    # "T3900_weave"
    e1[10]  = (70.3e9+71.0e9)/2.0
    e2[10]  = (68.9e9+67.6e9)/2.0
    g12[10] = 4.6e9
    anu[10] = 0.032
    rho[10] = 1551.0
    xt[10]  = min(1.0,701.302/803.236)*1055e6 #CTD mean -> A-basis
    xc[10]  = min(1.0,549.748/749.955)*676e6 #CTD mean -> A-basis
    yt[10]  = min(1.0,557.575/722.602)*945e6 #CTD mean -> A-basis
    yc[10]  = min(1.0,604.067/741.866)*614e6 #CTD mean -> A-basis
    s[10]   = min(1.0,138.440/154.888)*79.3e6 #CTD mean -> A-basis
    plythickness[10] = 0.218e-3
    # "T700_weave"
    e1[11]=55.82e9
    e2[11]=52.10e9
    g12[11]=4.295e9
    anu[11]=0.085
    rho[11]=1501.0
    xt[11]=701.32e6
    yt[11]=557.59e6
    xc[11]=549.77e6
    yc[11]=604.08e6
    s[11]=138.44e6
    plythickness[11] = 0.218e-3
    # "Gelcoat"
    e1[12]=3.44e9
    e2[12]=3.44e9
    g12[12]=1.38e9
    anu[12]=0.3
    rho[12]=1235.0
    xt[12]=100.0e6 #made up
    xc[12]=100.0e6 #made up
    yt[12]=100.0e6 #made up
    yc[12]=100.0e6 #made up
    s[12]=100.0e6 #made up
    plythickness[12] = 0.05e-3
    # "Triax"
    e1[13]=27.7e9
    e2[13]=13.65e9
    g12[13]=7.2e9
    anu[13]=0.39
    rho[13]=1850.0
    xt[13]=700.0e6
    xc[13]=100.0e6 #made up
    yt[13]=700.0e6
    yc[13]=100.0e6 #made up
    s[13]=100.0e6 #made up
    plythickness[13] = 0.94e-3
    # "Saertex"
    e1[14]=13.6e9
    e2[14]=13.3e9
    g12[14]=11.8e9
    anu[14]=0.49
    rho[14]=1780.0
    xt[14]=144.0e6
    xc[14]=213.0e6
    yt[14]=144.0e6
    yc[14]=213.0e6
    s[14]=100.0e6 #made up
    plythickness[14] = 1.0e-3

    #FOAMS
    # "taylor_foam"
    e1[15]=48.0e6
    e2[15]=48.0e6
    g12[15]=28.0e6
    anu[15]=0.3
    rho[15]=75.0
    xt[15]=100.0e6 #made up
    yt[15]=100.0e6 #made up
    xc[15]=100.0e6 #made up
    yc[15]=100.0e6 #made up
    s[15]=100.0e6 #made up
    plythickness[15]=1.0E-3
    # "SNL_foam"
    e1[16]=256.0e6
    e2[16]=256.0e6
    g12[16]=22.0e6
    anu[16]=0.3
    rho[16]=200.0
    xt[16]=100.0e6 #made up
    yt[16]=100.0e6 #made up
    xc[16]=100.0e6 #made up
    yc[16]=100.0e6 #made up
    s[16]=100.0e6 #made up
    plythickness[16]=1.0E-3

    return plyproperties(names,Composites.material(e1,e2,g12,anu,rho,xt,xc,yt,yc,s,plythickness))
end