using Serialization
using LightGraphs
using SimpleWeightedGraphs
using MetaGraphs
using Bijections
using Statistics
import LightGraphs.Parallel
import Base.Threads.@threads

function save_graph_data(graph::GGraphData)
    serialize(CALCULATED_FOLDER * "GGraphData", graph)
end
function save_graph_data(graph::GGraphData, epsilon, name)
    serialize(CALCULATED_FOLDER * "GGraphData $epsilon $name", graph)
end
function load_graph_data()
    if isfile(CALCULATED_FOLDER * "GGraphData")
        deserialize(CALCULATED_FOLDER * "GGraphData")
    else
        GGraphData()
    end
end
function load_graph_data(epsilon)
    deserialize(CALCULATED_FOLDER * "GGraphData $epsilon")
end
function load_graph_data(epsilon, name)
    deserialize(CALCULATED_FOLDER * "GGraphData $epsilon $name")
end

function get_overall_genotype_distribution(graph_data::GGraphData)
    occurances = Int64[]

    for vertex in 1:nv(graph_data.genotype_graph)
        outgoing = outneighbors(graph_data.genotype_graph, vertex)

        if length(outgoing) == 0
            push!(occurances, 0)
        else
            push!(occurances, sum([graph_data.genotype_graph.weights[u, vertex] for u in outgoing]))
        end
    end

    occurances = occurances ./ sum(occurances)

    return occurances
end

function build_genotype_graph!(graph_data::GGraphData)
    parent_offsprings = deserialize(CALCULATED_FOLDER * "parent_offspring_ocurrances")

    genotype_mapping = Bijection()

    for ((parent, _), _) in parent_offsprings
        if haskey(genotype_mapping, parent) == false
            genotype_mapping[parent] = length(genotype_mapping) + 1
        end
    end

    @info "Mapping created. ($(length(genotype_mapping))) Building graph..."

    genotype_graph = SimpleWeightedDiGraph(length(genotype_mapping))

    count = 0

    for ((parent, offspring), num) in parent_offsprings
        if haskey(genotype_mapping, offspring) == false continue end   # we skip sinks without offsprings

        parent_index = genotype_mapping[parent]
        offspring_index = genotype_mapping[offspring]
        add_edge!(genotype_graph, parent_index, offspring_index, num)

        count += 1
        if count % 1000 == 0
            @info (count / length(parent_offsprings))
        end
    end

    @info "Graph built."

    graph_data.genotype_vertex_mapping = genotype_mapping
    graph_data.genotype_graph = genotype_graph
end

function build_phenotype_graph!(graph_data::GGraphData, min_occurances, tolerance, min_samples, max_samples)
    phenotype_graph = SimpleWeightedDiGraph(nv(graph_data.genotype_graph))

    @info "Collect relevant edges."

    edges_to_test = []

    for edge in edges(graph_data.genotype_graph)
        u, v = src(edge), dst(edge)

        if u == v continue end
        if (v, u) in edges_to_test continue end
        if has_edge(graph_data.genotype_graph, u, v) == false continue end
        if graph_data.genotype_graph.weights[v, u] < min_occurances continue end

        push!(edges_to_test, (u, v))
    end

    @info "Calculate Phenotype Similarities for $(length(edges_to_test)) edges."

    num_edges_to_test = length(edges_to_test)
    phenotype_similarities = SharedArrays.SharedArray{Float64}(num_edges_to_test)

    done = Threads.Atomic{Int}(0)

    @threads for (i, (u, v)) in unique(enumerate(edges_to_test))
        genotype_u, genotype_v = graph_data.genotype_vertex_mapping(u), graph_data.genotype_vertex_mapping(v)
        similarity = get_phenotype_similarity(genotype_u, genotype_v, tolerance, min_samples, max_samples)

        phenotype_similarities[i] = similarity

        Threads.atomic_add!(done, 1)
        @info (done[] / length(edges_to_test))
    end

    phenotype_similarities_dict = Dict()

    for i in 1:num_edges_to_test
        similarity = phenotype_similarities[i]
        u, v = edges_to_test[i]

        phenotype_similarities_dict[(u, v)] = similarity
        phenotype_similarities_dict[(v, u)] = similarity
    end

    @info "Build Phenotype Similarities Graph."

    for edge in edges(graph_data.genotype_graph)
        u, v = src(edge), dst(edge)

        if haskey(phenotype_similarities_dict, (u, v)) == false continue end

        similarity = max(eps(), phenotype_similarities_dict[(u, v)])
        add_edge!(phenotype_graph, u, v, similarity)
    end

    graph_data.phenotype_graph = phenotype_graph
end

function build_neutral_networks!(graph_data::GGraphData, epsilon)
    undirected_phenotype_graph = get_simple_weighted_graph(graph_data.phenotype_graph)

    neutral_networks = []
    already_assigned_nodes = []

    # find all neutral networks

    for (i, edge) in edges(undirected_phenotype_graph) |> enumerate
	    if has_edge(undirected_phenotype_graph, edge) == false continue end

	    node = src(edge)    # we ignore sinks
        if node in already_assigned_nodes continue end

	    neutral_network = _find_neutral_network(undirected_phenotype_graph, node, epsilon)
        push!(neutral_networks, neutral_network)

        append!(already_assigned_nodes, neutral_network)

        if i % 1000 == 0
            @info " - $i, $(length(neutral_networks)) NNs"
        end
    end

    graph_data.neutral_networks = neutral_networks
end

function _find_neutral_network(graph, starting_vertex, epsilon)
    neutral_network = [starting_vertex]

    to_test = [(starting_vertex, n) for n in outneighbors(graph, starting_vertex)]

    while 0 < length(to_test)
        u, v = pop!(to_test)

        if graph.weights[v, u] <= epsilon
            push!(neutral_network, v)

            for n in outneighbors(graph, v)
                if n in neutral_network continue end

                push!(to_test, (v, n))
            end
        end
    end

    return unique(neutral_network)
end

function get_neutral_networks_by_g_sampling(graph_data, epsilon, n, tolerance, min_samples, max_samples)
    num_genotypes = nv(graph_data.genotype_graph)

    neutral_networks = []

    for _ in 1:n
        starting_genotype = rand(1:num_genotypes)
        neutral_network = build_neutral_network(graph_data, starting_genotype, epsilon, tolerance, min_samples, max_samples)
        push!(neutral_networks, neutral_network)
        @info "$(length(neutral_networks))-th generated with $(length(neutral_network)) nodes"
    end

    return neutral_networks
end

function build_neutral_network(graph_data, starting_vertex, epsilon, tolerance, min_samples, max_samples)
    neutral_network = []
    _build_neutral_network(graph_data, starting_vertex, neutral_network, epsilon, tolerance, min_samples, max_samples)
    return neutral_network
end

function _build_neutral_network(graph_data, vertex, already_built_nn, epsilon, tolerance, min_samples, max_samples)
    push!(already_built_nn, vertex)

    neighbors = outneighbors(graph_data.genotype_graph, vertex)

    for n in neighbors
        if n in already_built_nn continue end

        if has_edge(graph_data.phenotype_graph, vertex, n)
            phenotype_similarity = graph_data.phenotype_graph.weights[n, vertex]
        else
            genotype_u, genotype_v = graph_data.genotype_vertex_mapping(vertex), graph_data.genotype_vertex_mapping(n)

            phenotype_similarity = get_phenotype_similarity(genotype_u, genotype_v, tolerance, min_samples, max_samples)
            phenotype_similarity = max(eps(), phenotype_similarity)

            add_edge!(graph_data.phenotype_graph, vertex, n, phenotype_similarity)
        end

        if phenotype_similarity <= epsilon
            _build_neutral_network(graph_data, n, already_built_nn, epsilon, tolerance, min_samples, max_samples)
        end
    end
end


using Revise
using VCFTools
using SnpArrays
using MendelImpute
using Random
using StatsBase
using CodecZlib
using ProgressMeter
using JLSO
using BenchmarkTools
using LinearAlgebra
using CSV
using DataFrames

function filter_and_mask(chr::Int)
    # filter chromosome data for unique snps
#         data = "../beagle_raw/chr$chr.1kg.phase3.v5a.vcf.gz"
#         full_record_index = .!find_duplicate_marker(data)
#         VCFTools.filter(data, full_record_index, 1:nsamples(data),
#             des = "chr$chr.uniqueSNPs.vcf.gz", allow_multiallelic=false)

    # import VCF data with only unique SNPs
    _, vcf_sampleID, _, vcf_record_pos, _, _, _ = convert_ht(Bool,
        "chr$chr.uniqueSNPs.vcf.gz", save_snp_info=true, msg="importing")
    total_snps = length(vcf_record_pos)
    samples = length(vcf_sampleID)

    # generate target panel with all snps
    n = 1000
    sample_idx = falses(samples)
    sample_idx[1:n] .= true
    shuffle!(sample_idx)
    VCFTools.filter("chr$chr.uniqueSNPs.vcf.gz", 1:total_snps,
        sample_idx, des = "target.chr$chr.full.vcf.gz", allow_multiallelic=false)

    # generate reference panel without target samples
    VCFTools.filter("chr$chr.uniqueSNPs.vcf.gz", 1:total_snps,
        .!sample_idx, des = "ref.chr$chr.excludeTarget.vcf.gz", allow_multiallelic=false)

    # generate target file with 1000 samples whose snps are in the omni 5.4 chip
    bed = CSV.read("/home/biona001/omni_chips/InfiniumOmni5-4v1-2_A1.bed",
        skipto = 2, header=false, delim='\t', DataFrame)
    typed_snppos = bed[findall(x -> x == "chr$chr", bed[!, 1]), 3]
    match_idx = indexin(vcf_record_pos, typed_snppos)
    record_idx = falses(total_snps)
    record_idx[findall(!isnothing, match_idx)] .= true
    VCFTools.filter("chr$chr.uniqueSNPs.vcf.gz", record_idx, sample_idx,
        des = "target.chr$chr.typedOnly.vcf.gz", allow_multiallelic=false)

    # unphase and mask 0.1% entries in target file
    p = nrecords("target.chr$chr.typedOnly.vcf.gz")
    masks = falses(p, n)
    missingprop = 0.001
    for j in 1:n, i in 1:p
        rand() < missingprop && (masks[i, j] = true)
    end
    mask_gt("target.chr$chr.typedOnly.vcf.gz", masks,
        des="target.chr$chr.typedOnly.masked.vcf.gz", unphase=true)
end
Random.seed!(2020)
@time filter_and_mask(10)


#=
This file is auto-generated. Do not edit.
=#
"""
    mutable struct SingleMass <: Shaft
        H::Float64
        D::Float64
        ext::Dict{String, Any}
        states::Vector{Symbol}
        n_states::Int
        internal::InfrastructureSystemsInternal
    end

Parameters of single mass shaft model. Typically represents the rotor mass.

# Arguments
- `H::Float64`: Rotor inertia constant in MWs/MVA, validation range: `(0, nothing)`
- `D::Float64`: Rotor natural damping in pu, validation range: `(0, nothing)`
- `ext::Dict{String, Any}`
- `states::Vector{Symbol}`: The states are:
	δ: rotor angle,
	ω: rotor speed
- `n_states::Int`: SingleMass has 1 state
- `internal::InfrastructureSystemsInternal`: power system internal reference, do not modify
"""
mutable struct SingleMass <: Shaft
    "Rotor inertia constant in MWs/MVA"
    H::Float64
    "Rotor natural damping in pu"
    D::Float64
    ext::Dict{String, Any}
    "The states are:
	δ: rotor angle,
	ω: rotor speed"
    states::Vector{Symbol}
    "SingleMass has 1 state"
    n_states::Int
    "power system internal reference, do not modify"
    internal::InfrastructureSystemsInternal
end

function SingleMass(H, D, ext=Dict{String, Any}(), )
    SingleMass(H, D, ext, [:δ, :ω], 2, InfrastructureSystemsInternal(), )
end

function SingleMass(; H, D, ext=Dict{String, Any}(), states=[:δ, :ω], n_states=2, internal=InfrastructureSystemsInternal(), )
    SingleMass(H, D, ext, states, n_states, internal, )
end

# Constructor for demo purposes; non-functional.
function SingleMass(::Nothing)
    SingleMass(;
        H=0,
        D=0,
        ext=Dict{String, Any}(),
    )
end

"""Get [`SingleMass`](@ref) `H`."""
get_H(value::SingleMass) = value.H
"""Get [`SingleMass`](@ref) `D`."""
get_D(value::SingleMass) = value.D
"""Get [`SingleMass`](@ref) `ext`."""
get_ext(value::SingleMass) = value.ext
"""Get [`SingleMass`](@ref) `states`."""
get_states(value::SingleMass) = value.states
"""Get [`SingleMass`](@ref) `n_states`."""
get_n_states(value::SingleMass) = value.n_states
"""Get [`SingleMass`](@ref) `internal`."""
get_internal(value::SingleMass) = value.internal

"""Set [`SingleMass`](@ref) `H`."""
set_H!(value::SingleMass, val) = value.H = val
"""Set [`SingleMass`](@ref) `D`."""
set_D!(value::SingleMass, val) = value.D = val
"""Set [`SingleMass`](@ref) `ext`."""
set_ext!(value::SingleMass, val) = value.ext = val


using Pkg
Pkg.activate(@__DIR__)

using Dates
using HTTP
using MySQL
using JSON3
using StructTypes

struct jsonObj
    id::Int
    randomNumber::Int
end

StructTypes.StructType(::Type{jsonObj}) = StructTypes.Struct()

HTTP.listen("0.0.0.0" , 8080, reuseaddr = true) do http
    target = http.message.target

    HTTP.setstatus(http, 200)
    HTTP.setheader(http, "Server" => "Julia-HTTP")
    HTTP.setheader(http, "Date" => Dates.format(Dates.now(), Dates.RFC1123Format) * " GMT")

    if endswith(target, "/plaintext")
        HTTP.setheader(http, "Content-Type" => "text/plain")
        HTTP.startwrite(http)
        write(http, "Hello, World!")

    elseif endswith(target, "/json")
        HTTP.setheader(http, "Content-Type" => "application/json")
        startwrite(http)
        JSON3.write(http, (;message = "Hello, World!"))

    elseif endswith(target, "/db")
        HTTP.setheader(http, "Content-Type" => "application/json")
        randNum = rand(1:10000)

        conn = DBInterface.connect(MySQL.Connection, "tfb-database", "benchmarkdbuser", "benchmarkdbpass", db="hello_world")
        sqlQuery = "SELECT * FROM World WHERE id = $randNum"
        results = DBInterface.execute(conn, sqlQuery)
        row = first(results)
        dbNumber = row[2]
        jsonString = "{\"id\":$randNum,\"randomNumber\":$dbNumber}"

        startwrite(http)
        JSON3.write(http, (JSON3.read(jsonString)))

        DBInterface.close!(conn)

    elseif occursin("/queries", target)
        HTTP.setheader(http, "Content-Type" => "application/json")
        numQueries = -1

        try
            numQueries = parse(Int64, (split(target, "="))[2])

        catch ArgumentError
            numQueries = 1

        finally
            if numQueries > 500
                numQueries = 500
            end

            if numQueries < 1
                numQueries = 1
            end
        end

        conn = DBInterface.connect(MySQL.Connection, "tfb-database", "benchmarkdbuser", "benchmarkdbpass", db="hello_world")

        responseArray = Array{jsonObj}(undef, numQueries)
        for i in 1:numQueries
            # randNum = randNumList[i]
            randNum = rand(1:10000)
            sqlQuery = "SELECT * FROM World WHERE id = $randNum"
            results = DBInterface.execute(conn, sqlQuery)
            row = first(results)
            dbNumber = row[2]
            responseArray[i] = JSON3.read("{\"id\":$randNum,\"randomNumber\":$dbNumber}", jsonObj)
        end

        startwrite(http)
        JSON3.write(http, responseArray)

        DBInterface.close!(conn)

    elseif occursin("/updates", target)
        HTTP.setheader(http, "Content-Type" => "application/json")
        numQueries = -1

        try
            numQueries = parse(Int64, (split(target, "="))[2])

        catch ArgumentError
            numQueries = 1

        finally
            if numQueries > 500
                numQueries = 500
            end

            if numQueries < 1
                numQueries = 1
            end
        end

        conn = DBInterface.connect(MySQL.Connection, "tfb-database", "benchmarkdbuser", "benchmarkdbpass", db="hello_world")

        responseArray = Array{jsonObj}(undef, numQueries)
        for i in 1:numQueries
            randId = rand(1:10000)
            randNum = rand(1:10000)
            sqlQuery = "SELECT * FROM World WHERE id = $randId"
            results = DBInterface.execute(conn, sqlQuery)
            row = first(results)
            dbNumber = row[2]

            sqlQuery = "UPDATE World SET randomnumber = $randNum WHERE id = $randId"
            results = DBInterface.execute(conn, sqlQuery)
            responseArray[i] = JSON3.read("{\"id\":$randId,\"randomNumber\":$randNum}", jsonObj)
        end

        startwrite(http)
        JSON3.write(http, responseArray)

        DBInterface.close!(conn)

    elseif endswith(target, "/fortunes")
        HTTP.setheader(http, "Content-Type" => "text/html; charset=utf-8")

        fortunesList = []
        sqlQuery = "SELECT * FROM fortune"
        output = "<!DOCTYPE html><title>Fortunes</title>

id	message
"
        conn = DBInterface.connect(MySQL.Connection, "tfb-database", "benchmarkdbuser", "benchmarkdbpass", db="hello_world")
        results = DBInterface.execute(conn, sqlQuery)

        for row in results
            push!(fortunesList, [string(row[1]), row[2]])
        end

        push!(fortunesList, [string(0), "Additional fortune added at request time."])

        sort!(fortunesList, by = x -> x[2])

        for fortune in fortunesList
            id = fortune[1]
            message = HTTP.Strings.escapehtml(fortune[2])
            output = string(output, "
$id	$message
")
        end

        output = string(output, "
")

        write(http, output)

        DBInterface.close!(conn)

    else
        HTTP.setstatus(http, 404)
        startwrite(http)
        write(http, "Not Found")

    end
end


abstract SegmentGeometry <: Gadfly.GeometryElement

# Geometry for vectors/arrows/segments
immutable SegmentGeom <: SegmentGeometry

     arrow::Bool
     filled::Bool
     tag::Symbol

    function SegmentGeom(;arrow=false, filled=false, tag=empty_tag)
         new(arrow, filled, tag)
     end
end

const segment = SegmentGeom

# Leave this as a function, pending extra arguments e.g. arrow attributes
function vector(;filled::Bool=false, tag::Symbol=empty_tag)
    return SegmentGeom(arrow=true, filled=filled, tag=tag)
end

function element_aesthetics(::SegmentGeom)
    return [:x, :y, :xend, :yend, :color]
end

function Gadfly.render(geom::SegmentGeom, theme::Gadfly.Theme, aes::Gadfly.Aesthetics,
                subplot_layer_aess::@compat(Union{(@compat Void), Vector{Gadfly.Aesthetics}}),
                subplot_layer_datas::@compat(Union{(@compat Void), Vector{Gadfly.Data}}),
    scales::Dict{Symbol, Gadfly.ScaleElement})
    render(geom, theme, aes, scales)
end

function render(geom::SegmentGeom, theme::Gadfly.Theme, aes::Gadfly.Aesthetics,
    scales::Dict{Symbol, Gadfly.ScaleElement})

    Gadfly.assert_aesthetics_defined("Geom.segment", aes, :x, :y, :xend, :yend)

    function arrow{T<:Real}(x::T, y::T, xmax::T, ymax::T, xyrange::Vector{T})
        dx = xmax-x
        dy = ymax-y
        vl = 0.225*hypot(dy/xyrange[2], dx/xyrange[1])
        θ =  atan2(dy/xyrange[2], dx/xyrange[1])
        ϕ = pi/15
        xr = -vl*xyrange[1]*[cos(θ+ϕ), cos(θ-ϕ)]
        yr = -vl*xyrange[2]*[sin(θ+ϕ), sin(θ-ϕ)]
        arr = [(xmax+xr[1],ymax+yr[1]), (xmax,ymax), (xmax+xr[2],ymax+yr[2]) ]
        return arr
    end

    n = length(aes.x)
    color = ColorTypes.RGBA{Float32}(theme.default_color)
    default_aes = Gadfly.Aesthetics()
    default_aes.color = DataArrays.DataArray(fill(color,n))

    aes = inherit(aes, default_aes)

#    line_style = Gadfly.get_stroke_vector(theme.line_style)
    line_style = theme.line_style
    if is(line_style, nothing)
        line_style = []
    end

# Geom.vector requires information about scales

if geom.arrow
    xscale = scales[:x]
    yscale = scales[:y]
    check = [xscale.minvalue, xscale.maxvalue, yscale.minvalue, yscale.maxvalue]
    if any( map(x -> is(x,nothing), check) )
        error("For Geom.vector, Scale minvalue and maxvalue must be manually provided for both axes")
    end
    fx = xscale.trans.f
    fy = yscale.trans.f
    xyrange = [fx(xscale.maxvalue)-fx(xscale.minvalue),
        fy(yscale.maxvalue)-fy(yscale.minvalue)]

     arrows = [ arrow(x, y, xend, yend, xyrange)
            for (x, y, xend, yend) in zip(aes.x, aes.y, aes.xend, aes.yend) ]

end

    segments = [ [(x,y), (xend,yend)]
        for (x, y, xend, yend) in zip(aes.x, aes.y, aes.xend, aes.yend) ]

    classes = [string("geometry ", svg_color_class_from_label( aes.color_label([c])[1] ))
        for c in aes.color ]

    ctx = context()

    compose!( ctx, Compose.line(segments, geom.tag), stroke(aes.color), linewidth(theme.line_width),
        strokedash(line_style), svgclass( classes )  )
    if geom.arrow
        if geom.filled
            compose!(ctx, (context(), Compose.polygon(arrows), fill(aes.color), strokedash([])) )
        else
            compose!(ctx, (context(), Compose.line(arrows), stroke(aes.color), linewidth(theme.line_width),
            strokedash([]))  )
        end
    end

    return ctx
end


using NESSie.Format

@testset "writeskel (surface model)" begin
    for T in testtypes
        nodes = Vector{T}[[0, 0, 0], [1, 0, 0], [0, 1, 0], [0, 0, 1]]
        model = Model(
            nodes,
            Triangle{T}[
                Triangle(nodes[1], nodes[2], nodes[3]),
                Triangle(nodes[1], nodes[2], nodes[4])
            ]
        )
        (fname, fh) = mktemp()
        try
            writeskel(fh, model)
            seek(fh, 0)
            @test readline(fh) == "SKEL"
            @test split(readline(fh)) == ["4", "2"]
            for node in nodes
                @test [parse(T, e) for e in split(readline(fh))] == node
            end
            @test split(readline(fh)) == ["4", "0", "1", "2", "0"]
            @test split(readline(fh)) == ["4", "0", "1", "3", "0"]
            @test strip(readline(fh)) == ""
            @test eof(fh)
        finally
            close(fh)
            rm(fname)
        end
    end
end

@testset "writeskel (volume model)" begin
    for T in testtypes
        nodes = Vector{T}[[0, 0, 0], [1, 0, 0], [0, 1, 0], [0, 0, 1], [-2, 0, 0]]
        model = Model(
            nodes,
            Tetrahedron{T}[
                Tetrahedron(nodes[1], nodes[2], nodes[3], nodes[4]),
                Tetrahedron(nodes[1], nodes[2], nodes[3], nodes[5])
            ]
        )
        (fname, fh) = mktemp()
        try
            writeskel(fh, model)
            seek(fh, 0)
            @test readline(fh) == "SKEL"
            @test split(readline(fh)) == ["5", "2"]
            for node in nodes
                @test [parse(T, e) for e in split(readline(fh))] == node
            end
            @test split(readline(fh)) == ["7", "0", "1", "2", "3", "1", "3", "0"]
            @test split(readline(fh)) == ["7", "0", "1", "2", "4", "1", "4", "0"]
            @test strip(readline(fh)) == ""
            @test eof(fh)
        finally
            close(fh)
            rm(fname)
        end
    end
end


__precompile__()

module FastStyleTransfer

using Flux, Metalhead, Images, CuArrays, NNlib, BSON
using BSON: @save, @load
using Flux: @epochs, sub2, expand, initn
using Base.Iterators: partition
using Flux.Tracker: track, data, @grad, nobacksies

export train, stylize

include("utils.jl")
include("layers.jl")
include("vgg.jl")
include("transformer_net.jl")
include("neural_style.jl")

end # module


# VectorAffineDecisionFunction #
# ========================== #
mutable struct VectorAffineDecisionConstraintBridge{T, S} <: MOIB.Constraint.AbstractBridge
    constraint::CI{MOI.VectorAffineFunction{T}, S}
    decision_function::VectorAffineDecisionFunction{T}
    set::S
end

function MOIB.Constraint.bridge_constraint(::Type{VectorAffineDecisionConstraintBridge{T, S}},
                                           model,
                                           f::VectorAffineDecisionFunction{T},
                                           set::S) where {T, S}
    # All decisions have been mapped to the variable part terms
    # at this point.
    F = MOI.VectorAffineFunction{T}
    # Add the bridged constraint
    constraint = MOI.add_constraint(model,
                                    f.variable_part,
                                    set)
    # Save the constraint index, the decision function, and the set, to allow modifications
    return VectorAffineDecisionConstraintBridge{T, S}(constraint, f, set)
end

function MOI.supports_constraint(::Type{<:VectorAffineDecisionConstraintBridge{T}},
                                 ::Type{<:VectorAffineDecisionFunction{T}},
                                 ::Type{<:MOI.AbstractVectorSet}) where {T}
    return true
end
function MOIB.added_constrained_variable_types(::Type{<:VectorAffineDecisionConstraintBridge})
    return Tuple{DataType}[]
end
function MOIB.added_constraint_types(::Type{<:VectorAffineDecisionConstraintBridge{T, S}}) where {T, S}
    return [(MOI.VectorAffineFunction{T}, S)]
end
function MOIB.Constraint.concrete_bridge_type(::Type{<:VectorAffineDecisionConstraintBridge{T}},
                                              ::Type{<:VectorAffineDecisionFunction{T}},
                                              S::Type{<:MOI.AbstractVectorSet}) where T
    return VectorAffineDecisionConstraintBridge{T, S}
end

MOI.get(b::VectorAffineDecisionConstraintBridge{T, S}, ::MOI.NumberOfConstraints{MOI.VectorAffineFunction{T}, S}) where {T, S} = 1
MOI.get(b::VectorAffineDecisionConstraintBridge{T, S}, ::MOI.ListOfConstraintIndices{MOI.VectorAffineFunction{T}, S}) where {T, S} = [b.constraint]

function MOI.get(model::MOI.ModelLike, attr::MOI.ConstraintFunction,
                 bridge::VectorAffineDecisionConstraintBridge{T}) where T
    f = bridge.decision_function
    # Remove mapped variables to properly unbridge
    from_decision(v) = begin
        result = any(t -> t.variable_index == v, f.decision_part.terms)
    end
    g = VectorAffineDecisionFunction(
        MOIU.filter_variables(v -> !from_decision(v), f.variable_part),
        copy(f.decision_part))
    return g
end

function MOI.get(model::MOI.ModelLike, attr::MOI.ConstraintSet,
                 bridge::VectorAffineDecisionConstraintBridge{T}) where T
    return bridge.set
end

function MOI.get(model::MOI.ModelLike, attr::MOI.ConstraintPrimal,
                 bridge::VectorAffineDecisionConstraintBridge{T}) where T
    return MOI.get(model, attr, bridge.constraint)
end

function MOI.get(model::MOI.ModelLike, attr::MOI.ConstraintDual,
                 bridge::VectorAffineDecisionConstraintBridge{T}) where T
    return MOI.get(model, attr, bridge.constraint)
end

function MOI.get(model::MOI.ModelLike, ::DecisionIndex,
                 bridge::VectorAffineDecisionConstraintBridge{T}) where T
    return bridge.constraint
end

function MOI.delete(model::MOI.ModelLike, bridge::VectorAffineDecisionConstraintBridge)
    MOI.delete(model, bridge.constraint)
    return nothing
end

function MOI.set(model::MOI.ModelLike, ::MOI.ConstraintFunction,
                 bridge::VectorAffineDecisionConstraintBridge{T,S},
                 f::VectorAffineDecisionFunction{T}) where {T,S}
    # Update bridge functions and function constant
    bridge.decision_function = f
    # Modify constraint function
    MOI.set(model, MOI.ConstraintFunction(), bridge.constraint,
            f.variable_part)
    return nothing
end

function MOI.set(model::MOI.ModelLike, ::MOI.ConstraintSet,
                 bridge::VectorAffineDecisionConstraintBridge{T,S}, change::S) where {T,S}
    f = bridge.decision_function
    bridge.set = change
    MOI.set(model, MOI.ConstraintSet(), bridge.constraint, change)
    return nothing
end

function MOI.modify(model::MOI.ModelLike, bridge::VectorAffineDecisionConstraintBridge{T,S}, change::MOI.VectorConstantChange) where {T,S}
    f = bridge.decision_function
    # Modify variable part of decision function
    f.variable_part.constants .= change.new_constant
    MOI.set(model, MOI.ConstraintFunction(), bridge.constraint,
            f.variable_part)
    return nothing
end

function MOI.modify(model::MOI.ModelLike, bridge::VectorAffineDecisionConstraintBridge{T,S}, change::MOI.MultirowChange) where {T,S}
    f = bridge.decision_function
    # Modify variable part of decision function
    modify_coefficients!(f.variable_part.terms, change.decision, change.new_coefficients)
    # Modify decision part if variable originated from a decision
    if !isempty(findall(t -> t.scalar_term.variable_index == change.variable, f.decision_part.terms))
        modify_coefficients!(f.decision_part.terms, change.variable, change.new_coefficients)
    end
    # Modify the variable part of the constraint as usual
    MOI.modify(model, bridge.constraint, change)
    return nothing
end

function MOI.modify(model::MOI.ModelLike, bridge::VectorAffineDecisionConstraintBridge{T,S}, change::DecisionMultirowChange) where {T,S}
    f = bridge.decision_function
    # Update the decision coefficient
    modify_coefficients!(f.decision_part.terms, change.decision, change.new_coefficients)
    # Update mapped variable through MultiRowChange
    MOI.modify(model, bridge, MOI.MultiRowChange(change.decision, change.new_coefficients))
    return nothing
end


export TronSparseMatrixCSC

abstract type AbstractTronMatrix end

struct TronSparseMatrixCSC{VI, VD} <: AbstractTronMatrix
    n::Int
    nnz::Int
    colptr::VI
    rowval::VI
    map::VI
    diag_vals::VD
    tril_vals::VD
end

mutable struct TronDenseMatrix{MD} <: AbstractTronMatrix
    n::Int      # the current dimension of submatrix
    max_n::Int  # the maximum dimension this matrix can hold
    vals::MD    # matrix entries
end

function TronDenseMatrix{MD}(n::Int) where {MD}
    @assert n >= 1
    vals = tron_zeros(MD, (n, n))
    return TronDenseMatrix{MD}(n, n, vals)
end

function TronDenseMatrix(A::TronDenseMatrix)
    return TronDenseMatrix{typeof(A.vals)}(A.n, A.max_n, copy(A.vals))
end

function TronDenseMatrix(I::VI, J::VI, V::VD, n) where {VI, VD}
    @assert n >= 1
    @assert length(I) == length(J) == length(V)

    if isa(V, Array)
        A = TronDenseMatrix{Array{Float64, 2}}(n, n, tron_zeros(Array{eltype(V)}, (n, n)))
    else
        A = TronDenseMatrix{CuArray{Float64, 2}}(n, n, tron_zeros(CuArray{eltype(V)}, (n, n)))
    end
    for i=1:length(I)
        @assert 1 <= I[i] <= n && 1 <= J[i] <= n && I[i] >= J[i]
        @inbounds A.vals[I[i], J[i]] += V[i]
    end

    return A
end

# Default constructor to allocate memory
function TronSparseMatrixCSC{VI, VD}(n::Int, nnz::Int) where {VI, VD}
    colptr = tron_zeros(VI, n+1)
    rowval = tron_zeros(VI, nnz)
    diag_vals = tron_zeros(VD, n)
    tril_vals = tron_zeros(VD, nnz)
    map = tron_zeros(VI, nnz)
    return TronSparseMatrixCSC{VI, VD}(
        n, nnz, colptr, rowval, map, diag_vals, tril_vals,
    )
end

# Constructor using sparsity pattern passed as input
function TronSparseMatrixCSC(I::VI, J::VI, V::VD, n) where {VI, VD}
    @assert length(I) == length(J) == length(V)
    # Create a CSC matrix without duplicates.
    nnz = length(I)
    # Temporary arrays for removal of duplicates
    csc_rows = zeros(Int, nnz)
    csc_cols = zeros(Int, n+1)
    imap = zeros(Int, nnz)
    w = zeros(Int, n)

    acol_ptr = tron_zeros(VI, n+1)
    arow_ind = tron_zeros(VI, nnz)
    diag_vals = tron_zeros(VD, n)
    tril_vals = tron_zeros(VD, nnz)
    map = tron_zeros(VI, nnz)

    # Count the number of entries of each column.
    for i in 1:nnz
        csc_cols[J[i]+1] += 1
    end

    # Compute the starting address (zero-based) of each column.
    for i in 2:n
        csc_cols[i+1] += csc_cols[i]
    end

    # Construct a CSC matrix.
    for i=1:nnz
        p = csc_cols[J[i]]
        csc_rows[p+1] = I[i]
        imap[p+1] = i
        csc_cols[J[i]] += 1
    end

    # Reset the starting address (one-based) of each column.
    for i=n+1:-1:2
        csc_cols[i] = csc_cols[i-1] + 1
    end
    csc_cols[1] = 1

    # Remove duplicates.
    fill!(w, -1)
    nz = 1
    for i=1:n
        p = nz
        for j=csc_cols[i]:csc_cols[i+1]-1
            r = csc_rows[j]
            if w[r] >= p  # already seen (r,i)
                if i == r # diagonal term
                    map[imap[j]] = -r
                else
                    map[imap[j]] = w[r]
                end
            else # first time seen (r,i)
                w[r] = nz
                if i == r # diagonal term
                    map[imap[j]] = -r
                else
                    map[imap[j]] = nz
                    arow_ind[nz] = r
                    nz = nz + 1
                end
            end
        end
        acol_ptr[i] = p
    end
    acol_ptr[n+1] = nz
    nnz_a = nz - 1

    return TronSparseMatrixCSC{VI, VD}(
        n, nnz_a, acol_ptr, arow_ind, map, diag_vals, tril_vals,
    )
end

Base.size(A::TronDenseMatrix) = (A.n, A.n)
Base.size(A::TronSparseMatrixCSC) = (A.n, A.n)

function Base.fill!(A::TronDenseMatrix, val)
    fill!(A.vals, val)
end

function Base.fill!(A::TronSparseMatrixCSC, val)
    fill!(A.diag_vals, val)
    fill!(A.tril_vals, val)
end

@inline function Base.fill!(w::CuDeviceArray{Float64,1}, val::Float64)
    tx = threadIdx().x
    ty = threadIdx().y

    if tx <= length(w) && ty == 1
        @inbounds w[tx] = val
    end
    CUDA.sync_threads()

    return
end

function transfer!(A::TronSparseMatrixCSC, rows, cols, values, nnz)
    for i in 1:nnz
        m = A.map[i]
        if m < 0
            A.diag_vals[-m] = values[i]
        else
            A.tril_vals[m] = values[i]
        end
    end
end
function transfer!(A::TronDenseMatrix, rows, cols, values, nnz)
    @inbounds for i in 1:nnz
        # It is assumed that rows[i] >= cols[i] for all i.
        A.vals[rows[i], cols[i]] += values[i]
    end
end

nrm2!(wa, A) = nrm2!(wa, A, A.n)

function nrm2!(wa, A::TronDenseMatrix, n)
    @inbounds for i=1:n
        wa[i] = A.vals[i,i]^2
    end
    @inbounds for j=1:n
        for i=j+1:n
            wa[j] += A.vals[i,j]^2
            wa[i] += A.vals[i,j]^2
        end
    end
    @inbounds for j=1:n
        wa[j] = sqrt(wa[j])
    end

    return
end

function nrm2!(wa, A::TronSparseMatrixCSC, n)
    @inbounds for i=1:n
        wa[i] = A.diag_vals[i]^2
    end
    @inbounds for j=1:n
        for i=A.colptr[j]:A.colptr[j+1]-1
            k = A.rowval[i]
            wa[j] = wa[j] + A.tril_vals[i]^2
            wa[k] = wa[k] + A.tril_vals[i]^2
        end
    end
    @inbounds for j=1:n
        wa[j] = sqrt(wa[j])
    end
end

@inline function nrm2!(wa, A::CuDeviceArray{Float64,2}, n::Int)
    tx = threadIdx().x
    ty = threadIdx().y

    v = 0.0
    if tx <= n && ty == 1
        @inbounds for j=1:n
            v += A[j,tx]^2
        end
        @inbounds wa[tx] = sqrt(v)
    end
    #=
    v = A[tx,ty]^2

    if tx > n || ty > n
        v = 0.0
    end

    # Sum over the x-dimension.
    offset = div(blockDim().x, 2)
    while offset > 0
        v += CUDA.shfl_down_sync(0xffffffff, v, offset)
        offset = div(offset, 2)
    end

    if tx == 1
        wa[ty] = sqrt(v)
    end
    =#
    CUDA.sync_threads()

    return
end

@inline getdiagvalue(A::TronSparseMatrixCSC, i) = A.diag_vals[i]
@inline getdiagvalue(A::TronDenseMatrix, i) = A.vals[i,i]

function dssyax(n, A::TronDenseMatrix, x, y)
    zero = 0.0

    @inbounds for i=1:n
        y[i] = A.vals[i,i]*x[i]
    end

    @inbounds for j=1:n
        rowsum = zero
        for i=j+1:n
            rowsum += A.vals[i,j]*x[i]
            y[i] += A.vals[i,j]*x[j]
        end
        y[j] += rowsum
    end

    return
end

"""
Subroutine dssyax

This subroutine computes the matrix-vector product y = A*x,
where A is a symmetric matrix with the strict lower triangular
part in compressed column storage.
"""
dssyax(A::TronSparseMatrixCSC, x, y) = dssyax(A.n, A, x, y)
function dssyax(n, A::TronSparseMatrixCSC, x, y)
    zero = 0.0

    @inbounds for i in 1:n
        y[i] = A.diag_vals[i]*x[i]
    end

    @inbounds for j in 1:n
        rowsum = zero
        for i in A.colptr[j]:A.colptr[j+1]-1
            row = A.rowval[i]
            rowsum += A.tril_vals[i]*x[row]
            @inbounds y[row] += A.tril_vals[i]*x[j]
        end
        y[j] += rowsum
    end

    return
end

@inline function dssyax(n::Int, A::CuDeviceArray{Float64,2},
                        z::CuDeviceArray{Float64,1},
                        q::CuDeviceArray{Float64,1})
    tx = threadIdx().x
    ty = threadIdx().y

    v = 0.0
    if tx <= n && ty == 1
        @inbounds for j=1:n
            v += A[tx,j]*z[j]
        end
        @inbounds q[tx] = v
    end
    #=
    v = 0.0
    if tx <= n && ty <= n
        v = A[ty,tx]*z[tx]
    end

    # Sum over the x-dimension: v = sum_tx A[ty,tx]*z[tx].
    # The thread with tx=1 will have the sum in v.

    offset = div(blockDim().x, 2)
    while offset > 0
        v += CUDA.shfl_down_sync(0xffffffff, v, offset)
        offset = div(offset, 2)
    end

    if tx == 1
        q[ty] = v
    end
    =#
    CUDA.sync_threads()

    return
end

@inline function reorder!(n::Int, nfree::Int, B::CuDeviceArray{Float64,2},
                          A::CuDeviceArray{Float64,2}, indfree::CuDeviceArray{Int,1},
                          iwa::CuDeviceArray{Int,1})
    tx = threadIdx().x
    ty = threadIdx().y

    #=
    if tx == 1 && ty == 1
        @inbounds for j=1:nfree
            jfree = indfree[j]
            B[j,j] = A[jfree,jfree]
            for i=jfree+1:n
                if iwa[i] > 0
                    B[iwa[i],j] = A[i,jfree]
                    B[j,iwa[i]] = B[iwa[i],j]
                end
            end
        end
    end
    =#
    if tx <= nfree && ty == 1
        @inbounds begin
            jfree = indfree[tx]
            B[tx,tx] = A[jfree,jfree]
            for i=jfree+1:n
                if iwa[i] > 0
                    B[iwa[i],tx] = A[i,jfree]
                    B[tx,iwa[i]] = B[iwa[i],tx]
                end
            end
        end
    end

    CUDA.sync_threads()

    return
end

function reorder!(B::TronDenseMatrix, A::TronDenseMatrix, indfree, nfree, iwa)
    nnz = 0
    @inbounds for j=1:nfree
        jfree = indfree[j]
        B.vals[j,j] = A.vals[jfree,jfree]
        for i=jfree+1:A.n
            if iwa[i] > 0
                nnz += 1
                B.vals[iwa[i],j] = A.vals[i,jfree]
            end
        end
    end
    B.n = nfree
    return nnz
end

# Update matrix B inplace
function reorder!(B::TronSparseMatrixCSC, A::TronSparseMatrixCSC, indfree, nfree, iwa)
    B.colptr[1] = 1
    nnz = 0
    @inbounds for j=1:nfree
        jfree = indfree[j]
        B.diag_vals[j] = A.diag_vals[jfree]
        for ip = A.colptr[jfree]:A.colptr[jfree+1]-1
            if iwa[A.rowval[ip]] > 0
                nnz = nnz + 1
                B.rowval[nnz] = iwa[A.rowval[ip]]
                B.tril_vals[nnz] = A.tril_vals[ip]
            end
        end
        B.colptr[j+1] = nnz + 1
    end
    return nnz
end


using SIIPExamples

using PowerSystems
using PowerSimulations
const PSI = PowerSimulations
using D3TypeTrees

using Dates
using DataFrames

using Cbc #solver

rts_dir = SIIPExamples.download("https://github.com/GridMod/RTS-GMLC")
rts_src_dir = joinpath(rts_dir, "RTS_Data", "SourceData")
rts_siip_dir = joinpath(rts_dir, "RTS_Data", "FormattedData", "SIIP");

rawsys = PowerSystems.PowerSystemTableData(
    rts_src_dir,
    100.0,
    joinpath(rts_siip_dir, "user_descriptors.yaml"),
    timeseries_metadata_file = joinpath(rts_siip_dir, "timeseries_pointers.json"),
    generator_mapping_file = joinpath(rts_siip_dir, "generator_mapping.yaml"),
);
sys = System(rawsys; time_series_resolution = Dates.Hour(1));

branches = Dict{Symbol, DeviceModel}(
    :L => DeviceModel(Line, StaticLine),
    :T => DeviceModel(Transformer2W, StaticTransformer),
    :TT => DeviceModel(TapTransformer, StaticTransformer),
)

devices = Dict(
    :Generators => DeviceModel(ThermalStandard, ThermalStandardUnitCommitment),
    :Ren => DeviceModel(RenewableDispatch, RenewableFullDispatch),
    :Loads => DeviceModel(PowerLoad, StaticPowerLoad),
    :HydroROR => DeviceModel(HydroDispatch, FixedOutput),
    :Hydro => DeviceModel(HydroEnergyReservoir, HydroDispatchRunOfRiver),
    :RenFx => DeviceModel(RenewableFix, FixedOutput),
    :ILoads => DeviceModel(InterruptibleLoad, InterruptiblePowerLoad),
)

services = Dict(
    :ReserveUp => ServiceModel(VariableReserve{ReserveUp}, RangeReserve),
    :ReserveDown => ServiceModel(VariableReserve{ReserveDown}, RangeReserve),
)

template_uc = OperationsProblemTemplate(CopperPlatePowerModel, devices, branches, services);

solver = optimizer_with_attributes(Cbc.Optimizer, "logLevel" => 1, "ratioGap" => 0.5)

horizon = 24;
interval = Dates.Hour(24);
transform_single_time_series!(sys, horizon, interval)

op_problem = OperationsProblem(
    GenericOpProblem,
    template_uc,
    sys;
    optimizer = solver,
    horizon = horizon,
)

# This file was generated using Literate.jl, https://github.com/fredrikekre/Literate.jl


# XXX sets the objective function according to the arguments provided in obj_dic
"""
Set the objective of the model's underlying optimization problem.
```julia
setObjective!(obj_dic::Union{Dict{Symbol,Float64},Symbol},anyM::anyModel)
```
`obj_dic` is a key-word argument that specifies the respective objective. To enable multi-criteria optimization, it can also be a dictionary assigning a weight to each objective. So far, the only supported key-word is `costs`.
"""
function setObjective!(obj_dic::Union{Dict{Symbol,Float64},Symbol},anyM::anyModel,minimize::Bool=true)

    # XXX converts input into dictionary, if only a symbol was provided, if :none keyword was provided returns a dummy objective function
    if typeof(obj_dic) == Symbol
        if obj_dic == :none @objective(anyM.optModel, Min, 1); return, produceMessage(anyM.options,anyM.report, 1," - Set an empty objective function") end
        obj_dic = Dict(obj_dic => 1.0)
    end

    # XXX create empty variables table for objective variables, if already other object defined, these variables and equations are removed from the model
    partObj = anyM.parts.obj

	if !(:objVar in keys(partObj.var))
		partObj.var[:objVar] = DataFrame(name = Symbol[], group = Symbol[], var = AffExpr[])
		partObj.cns[:objEqn] = DataFrame(name = Symbol[], group = Symbol[], cns = ConstraintRef[])
	end

    # XXX create variables and equations required for specified objectives
    for objGrp in setdiff(keys(obj_dic),unique(partObj.var[:objVar][!,:group]))
        createObjective!(objGrp,partObj,anyM)
    end

    # XXX sets overall objective variable with upper limits and according to weights provided in dictionary
	objBd_flt = anyM.options.bound.obj |> (x -> isnan(x) ? NaN : x / anyM.options.scaFac.obj)
	obj_var = JuMP.add_variable(anyM.optModel, JuMP.build_variable(error, VariableInfo(false, NaN, !isnan(objBd_flt), objBd_flt, false, NaN, false, NaN, false, false)),"obj") * anyM.options.scaFac.obj
	obj_eqn = @constraint(anyM.optModel, obj_var == sum(map(x -> sum(filter(r -> r.group == x,partObj.var[:objVar])[!,:var])*obj_dic[x], collectKeys(keys(obj_dic)))))

    if minimize
        @objective(anyM.optModel, Min, obj_var / anyM.options.scaFac.obj)
    else
        @objective(anyM.optModel, Max, obj_var / anyM.options.scaFac.obj)
    end

	produceMessage(anyM.options,anyM.report, 1," - Set objective function according to inputs")
end

createObjective!(objGrp::Symbol, partObj::OthPart,anyM::anyModel) = createObjective!(Val{objGrp}(), partObj::OthPart,anyM::anyModel)

# XXX create variables and equations for cost objective
function createObjective!(objGrp::Val{:costs},partObj::OthPart,anyM::anyModel)

	parObj_arr = collectKeys(keys(partObj.par))
	techIdx_arr = collect(keys(anyM.parts.tech))
	varToPart_dic = Dict(:exc => :exc, :ctr => :bal,:trdSell => :trd, :trdBuy => :trd)

	# computes discount factors from discount rate provided and saves them as new parameter elements
	computeDisFac!(partObj,anyM)

	# XXX add elements for expansion costs of technologies
	for va in (:Conv, :StIn, :StOut, :StSize, :Exc)

		# XXX compute expansion costs
		var_sym = Symbol(:exp,va)
		costPar_sym = Symbol(:costExp,va)

		if !(costPar_sym in parObj_arr) continue end

		# get all variables
        allExp_df = getAllVariables(var_sym,anyM)
		if isempty(allExp_df)
			continue
		else
			allExp_df = rename(allExp_df,:var => :exp)
		end

		# add economic lifetime to table where it is defined
		if Symbol(:lifeEco,va) in parObj_arr
			ecoLife_df = matchSetParameter(allExp_df,partObj.par[Symbol(:lifeEco,va)],anyM.sets,newCol = :life)
			noEcoLife_df = antijoin(allExp_df,ecoLife_df, on = intCol(allExp_df))
			noEcoLife_df[!,:life] .= nothing
			allExp_df = vcat(ecoLife_df,noEcoLife_df)
		else
			allExp_df[!,:life] .= nothing
		end

		techFilt_arr = filter(y -> var_sym in keys(anyM.parts.tech[y].var), techIdx_arr)

		# use technical lifetime where no economic lifetime could be obtained
		if va != :Exc
			allPar_arr = filter(w -> !(isempty(w)),map(x -> anyM.parts.tech[x].par[Symbol(:life,va)].data,filter(y -> var_sym in keys(anyM.parts.tech[y].var), techFilt_arr)))
			union(intCol.(allPar_arr)...) |> (z -> map(x -> map(y -> insertcols!(x,1,(y => fill(0,size(x,1)))) , setdiff(z,intCol(x)) ) ,allPar_arr))
			lifePar_obj = copy(anyM.parts.tech[techFilt_arr[1]].par[Symbol(:life,va)],vcat(allPar_arr...))
		else
			lifePar_obj = anyM.parts.exc.par[:lifeExc]
		end
		techLife_df = matchSetParameter(filter(x -> isnothing(x.life),allExp_df)[!,Not(:life)],lifePar_obj,anyM.sets,newCol = :life)
		allExp_df = vcat(techLife_df,filter(x -> !isnothing(x.life),allExp_df))

		# gets expansion costs and interest rate to compute annuity
		allExp_df = matchSetParameter(convertExcCol(allExp_df),partObj.par[costPar_sym],anyM.sets,newCol = :costExp)
		if isempty(allExp_df) continue end

		# uses tech specific discount rate and fall back on general discount rate as default
		if Symbol(:rateExp,va) in keys(partObj.par)
			techRate_df = matchSetParameter(allExp_df,partObj.par[Symbol(:rateExp,va)],anyM.sets,newCol = :rate)
		else
			techRate_df = filter(x -> false,allExp_df); techRate_df[!,:rate] .= Float64[]
		end
		# obtains general discount rate
		generRate_df = rename(antijoin(allExp_df,techRate_df,on = intCol(techRate_df)),:Ts_expSup => :Ts_disSup, :Ts_disSup => :Ts_expSup)
		if va != :Exc
			generRate_df = matchSetParameter(generRate_df, partObj.par[:rateDisc],anyM.sets,newCol = :rate)
		else
			rateB_arr = matchSetParameter(rename(generRate_df,:R_a => :R_exp), partObj.par[:rateDisc],anyM.sets,newCol = :rateA)[!,:rateA]
			rateA_arr = matchSetParameter(rename(generRate_df,:R_b => :R_exp), partObj.par[:rateDisc],anyM.sets,newCol = :rateB)[!,:rateB]
			generRate_df[!,:rate] = 0.5 .* (rateA_arr .+ rateB_arr)
		end
		allExp_df = vcat(techRate_df,rename(generRate_df, :Ts_expSup => :Ts_disSup, :Ts_disSup => :Ts_expSup))

		# compute annuity costs
		allExp_df[!,:costAnn] = map(x -> x.costExp * (x.rate == 0.0 ? 1/x.life : (x.rate * (1 + x.rate)^x.life) / ((1 + x.rate)^x.life-1)), eachrow(allExp_df))
		select!(allExp_df,Not([:costExp,:life,:rate]))
		allExp_df = flatten(allExp_df,:Ts_disSup)

		# adds discount factor and computes cost expression
		allExp_df = matchSetParameter(convertExcCol(allExp_df),partObj.par[va != :Exc ? :disFac : :disFacExc],anyM.sets,newCol = :disFac)

		# XXX groups cost expressions by technology, scales groups expression and creates a variables for each grouped entry
		allExp_df = rename(combine(x -> (expr = sum(x.disFac .* x.exp .* x.costAnn),) ,groupby(allExp_df,va != :Exc ? [:Ts_disSup,:R_exp,:Te] : [:Ts_disSup,:C])),:Ts_disSup => :Ts_exp)
		transferCostEle!(allExp_df, partObj,costPar_sym,anyM.optModel,anyM.lock,anyM.sets,anyM.options.coefRng,anyM.options.scaFac.costCapa,anyM.options.checkRng)
	end
	produceMessage(anyM.options,anyM.report, 3," - Created expression for expansion costs")

	# XXX add elements for operational costs of technologies
	# if decommissioning is enabled, capacity costs depend on commissioned and not on installed capacities
	capaTyp_sym =  anyM.options.decomm != :none ? :oprCapa : :capa
	for va in (:Conv, :StIn, :StOut, :StSize, :Exc)

		var_sym = Symbol(capaTyp_sym,va)
		costPar_sym = Symbol(:costOpr,va)

		if !(costPar_sym in parObj_arr) continue end

		# get all variables
        allCapa_df = getAllVariables(var_sym,anyM)
        if isempty(allCapa_df)
            continue
        else
            allCapa_df = rename(allCapa_df,:var => :capa)
        end

		# joins costs and discount factors to create cost expression
		allCapa_df = matchSetParameter(convertExcCol(allCapa_df),partObj.par[costPar_sym],anyM.sets,newCol = :costOpr)
		allCapa_df = matchSetParameter(convertExcCol(allCapa_df),partObj.par[va != :Exc ? :disFac : :disFacExc],anyM.sets,newCol = :disFac)

		if isempty(allCapa_df) continue end

		# XXX groups cost expressions by technology, scales groups expression and creates a variables for each grouped entry
		allCapa_df = combine(x -> (expr = sum(x.disFac .* x.capa .* x.costOpr),), groupby(allCapa_df,va != :Exc ? [:Ts_disSup,:R_exp,:Te] : [:Ts_disSup,:C]))
		transferCostEle!(allCapa_df, partObj,costPar_sym,anyM.optModel,anyM.lock,anyM.sets,anyM.options.coefRng,anyM.options.scaFac.costCapa,anyM.options.checkRng)
	end
	produceMessage(anyM.options,anyM.report, 3," - Created expression for capacity costs")

	# XXX add elements for variable costs of technologies
	for va in (:use,:gen,:stIn,:stOut,:exc)

		costPar_sym = string(va) |> (x -> Symbol(:costVar,uppercase(x[1]),x[2:end]))

		if !(costPar_sym in parObj_arr || (va == :use && :emissionPrc in parObj_arr && :emissionFac in keys(anyM.parts.lim.par))) continue end

		# obtain all variables
		allDisp_df = getAllVariables(va,anyM)
		if isempty(allDisp_df)
			continue
		else
			allDisp_df = rename(allDisp_df,:var => :disp)
		end

		# special case for variable costs of exchange (direct and symmetric values need to be considered both) and of use (emission price needs to be considered)
		if va == :exc
			if :costVarExcDir in parObj_arr
				dirCost_df = matchSetParameter(convertExcCol(allDisp_df),anyM.parts.obj.par[:costVarExcDir],anyM.sets,newCol = :costVar)
			else
				dirCost_df = convertExcCol(allDisp_df[[],:])
			end
			noDirCost_df = matchSetParameter(antijoin(convertExcCol(allDisp_df),dirCost_df, on = intCol(dirCost_df)),anyM.parts.obj.par[costPar_sym],anyM.sets,newCol = :costVar)

			allDisp_df = rename(convertExcCol(vcat(dirCost_df,noDirCost_df)),:var => :disp)
		elseif va == :use && :emissionPrc in parObj_arr && :emissionFac in keys(anyM.parts.lim.par)
			# get emission prices as a costs entry
			emPrc_df = matchSetParameter(select(allDisp_df,Not(:disp)),partObj.par[:emissionPrc],anyM.sets, newCol = :prc)
			emPrc_df = matchSetParameter(emPrc_df,anyM.parts.lim.par[:emissionFac],anyM.sets, newCol = :fac)
			emPrc_df[!,:costEms] = emPrc_df[!,:prc] .*  emPrc_df[!,:fac] ./ 1000
			select!(emPrc_df,Not([:prc,:fac]))
			# merge emission costs with other variable costs or just use emission costs if there are not any other
			if costPar_sym in parObj_arr
				otherVar_df = matchSetParameter(select(allDisp_df,Not(:disp)),anyM.parts.obj.par[costPar_sym],anyM.sets,newCol = :costVar)
				allCost_df = joinMissing(otherVar_df,emPrc_df,intCol(emPrc_df),:outer,merge(Dict{Symbol,Any}(:costVar => 0.0, :costEms => 0.0),Dict{Symbol,Any}(x => 0 for x in intCol(emPrc_df))) )
				allCost_df[!,:costVar] = allCost_df[!,:costVar] .+ allCost_df[!,:costEms]
				select!(allCost_df,Not(:costEms))
			else
				allCost_df = emPrc_df
				rename!(allCost_df,:costEms => :costVar)
			end

			allDisp_df = innerjoin(allCost_df,allDisp_df, on = intCol(allDisp_df))
		else
			allDisp_df = matchSetParameter(allDisp_df,anyM.parts.obj.par[costPar_sym],anyM.sets,newCol = :costVar)
		end
		if isempty(allDisp_df) continue end

		# renames dispatch regions to enable join with discount factors
		if va != :exc rename!(allDisp_df,:R_dis => :R_exp) end
		allDisp_df = matchSetParameter(allDisp_df,partObj.par[va != :exc ? :disFac : :disFacExc],anyM.sets,newCol = :disFac)

		# XXX groups cost expressions by technology, scales groups expression and creates a variables for each grouped entry
		allDisp_df = combine(x -> (expr = sum(x.disFac .* x.disp .* x.costVar) ./ 1000.0 .* anyM.options.redStep,) ,groupby(allDisp_df,va != :exc ? [:Ts_disSup,:R_exp,:Te] : [:Ts_disSup,:C]))
		transferCostEle!(allDisp_df, partObj,costPar_sym,anyM.optModel,anyM.lock,anyM.sets,anyM.options.coefRng,anyM.options.scaFac.costDisp,anyM.options.checkRng)
	end
	produceMessage(anyM.options,anyM.report, 3," - Created expression for variables costs")

	# XXX add elements for curtailment and loss of load costs of energy carriers
	for varType in [:crt,:lss]
		if varType in keys(anyM.parts.bal.var)
			cost_sym = varType == :crt ? :costCrt : :costLss
			# compute discounted curtailment costs
			allVar_df = rename(matchSetParameter(anyM.parts.bal.var[varType],anyM.parts.bal.par[cost_sym],anyM.sets,newCol = :cost),:var => varType)
			allVar_df = matchSetParameter(rename(allVar_df,:R_dis => :R_exp),partObj.par[:disFac],anyM.sets,newCol = :disFac)
			# groups cost expressions by carrier, scales groups expression and creates a variables for each grouped entry
			allVar_df = combine(x -> (expr = sum(x.disFac .* x[!,varType] .* x.cost) ./ 1000.0 .* anyM.options.redStep,) ,groupby(allVar_df, [:Ts_disSup,:R_exp,:C]))
			transferCostEle!(allVar_df, partObj,cost_sym,anyM.optModel,anyM.lock,anyM.sets,anyM.options.coefRng,anyM.options.scaFac.costDisp,anyM.options.checkRng, NaN)
		end
	end

	# XXX add elements for trade costs of energy carriers (buy and sell)
	for va in (:trdBuy, :trdSell)
		if va in keys(anyM.parts.trd.var)
			# compute discounted trade costs
			allTrd_df = rename(matchSetParameter(anyM.parts.trd.var[va],anyM.parts.trd.par[Symbol(va,:Prc)],anyM.sets,newCol = :costTrd),:var => :trd)
			allTrd_df = matchSetParameter(rename(allTrd_df,:R_dis => :R_exp),partObj.par[:disFac],anyM.sets,newCol = :disFac)
			# groups cost expressions by carrier, scales groups expression and creates a variables for each grouped entry
			allTrd_df = combine(x -> (expr = sum(x.disFac .* x.trd .* x.costTrd) ./ (va == :trdSell ? -1000.0 : 1000.0) .* anyM.options.redStep,), groupby(allTrd_df, [:Ts_disSup,:R_exp,:C]))
			transferCostEle!(allTrd_df, partObj,Symbol(:cost,uppercase(string(va)[1]),string(va)[2:end]),anyM.optModel,anyM.lock,anyM.sets,anyM.options.coefRng,anyM.options.scaFac.costDisp,anyM.options.checkRng,(va == :trdSell ? NaN : 0.0))
		end
	end
	produceMessage(anyM.options,anyM.report, 3," - Created expression for curtailment and trade costs")

	# XXX creates overall costs variable considering scaling parameters
	relBla = filter(x -> x != :objVar, collectKeys(keys(partObj.var)))
	objVar_arr = map(relBla) do varName
		# sets lower limit of zero, except for curtailment and revenues from selling, because these can incure "negative" costs
		lowBd_tup = !(varName in (:costCrt,:costTrdSell)) |> (x -> (x,x ? 0.0 : NaN))
		info = VariableInfo(lowBd_tup[1], lowBd_tup[2], false, NaN, false, NaN, false, NaN, false, false)
		return JuMP.add_variable(anyM.optModel, JuMP.build_variable(error, info),string(varName))
	end

	# create dataframe with for overall cost equations and scales it
	objExpr_arr = [objVar_arr[idx] - sum(partObj.var[name][!,:var]) for (idx, name) in enumerate(relBla)]
	cns_df = DataFrame(group = fill(:costs,length(objExpr_arr)), name = relBla, cnsExpr = objExpr_arr)

	# add variables and equations to overall objective dataframes
	partObj.cns[:objEqn] = vcat(partObj.cns[:objEqn],createCns(cnsCont(cns_df,:equal),anyM.optModel))
	partObj.var[:objVar] = vcat(partObj.var[:objVar],DataFrame(group = fill(:costs,length(objVar_arr)), name = relBla, var = objVar_arr))
end

# XXX transfers provided cost dataframe into dataframe of overall objective variables and equations (and scales them)
function transferCostEle!(cost_df::DataFrame, partObj::OthPart,costPar_sym::Symbol,optModel::Model,lock_::ReentrantLock,sets_dic::Dict{Symbol,Tree},
												coefRng_tup::NamedTuple{(:mat,:rhs),Tuple{Tuple{Float64,Float64},Tuple{Float64,Float64}}}, scaCost_fl::Float64, checkRng_fl::Float64, lowBd::Float64 = 0.0)

	# create variables for cost entry and builds corresponding expression for equations controlling them
	cost_df = createVar(cost_df,string(costPar_sym),NaN,optModel,lock_,sets_dic, scaFac = scaCost_fl, lowBd = lowBd)
	cost_df[!,:cnsExpr] = map(x -> x.expr - x.var, eachrow(cost_df))
	select!(cost_df,Not(:expr))

	# scales cost expression
	scaleCnsExpr!(cost_df,coefRng_tup,checkRng_fl)
	cost_df[!,:var] = cost_df[!,:var]

	# writes equations and variables
	partObj.cns[costPar_sym] = createCns(cnsCont(select(cost_df,Not(:var)),:equal),optModel)
	partObj.var[costPar_sym] = select(cost_df,Not(:cnsExpr))
end


module lmp_str

    import Dates
    using Printf
    using Statistics
    import LinearAlgebra.norm

    abstract type Str end
    abstract type Potential end
    abstract type Potential_Unit end
    abstract type Data end
    abstract type Unit end

    # Main Files
    include("Data.jl")
    export Atom, Bond, Angle, Dihedral, Improper
    export Data_Cell, Data_Basic, Data_Unit, Data_Sum

    include("useful_funcs.jl")
    export max, min, get_data, diag, conv, dist, norm_vec, rot_mat
    export add!, set!, central_point_atom, central_point_box, copy_array

    include("Constants.jl")
    export const_list
    #=
    export Const_k_b, Const_n_a, Const_density_wat,
        Const_kcal2j, Const_kcalm2j, Const_kcalm2t,
        Const_g2kg, Const_kg2g, Const_gm2g, Const_gm2kg,
        Const_an2m, Const_an2nm, Const_nm2m, Const_cm2an, Const_dm2m, Const_m2dm,
        Const_fs2s, Const_ps2s, Const_ns2s
    =#

    # Structure Files
    include("./Structure/Structure_Basic.jl")
    export Structure_Basic, transform
    export structure_sc, structure_bcc, structure_fcc, structure_dc

    include("./Structure/Structure_Wat.jl")
    export Structure_Wat
    export structure_tip3p, structure_spc, structure_spce
    export structure_tip4p_cut, structure_tip4p_long
    export structure_tip4p_2005, structure_tip4p_ice
    export structure_tip4p_ew, structure_tip4p_fq
    export structure_tip5p, structure_tip4p_2005, structure_tip5p_2018
    export structure_tip7p

    include("./Structure/Structure_Si.jl")
    export Structure_Si
    export structure_si, structure_sio2
    export structure_si3n4, structure_si3n4_ort, structure_si3n4_pdb, structure_si3n4_ort_pdb

    include("./Structure/Structure_C.jl")
    export Structure_C
    export structure_graphene, structure_graphene_ort

    include("./Structure/Structure_Ti.jl")
    export Structure_Ti
    export structure_tio2_anatase, structure_tio2_rutile

    include("./Structure/Structure_Ca.jl")
    export Structure_Ca
    export structure_caco3

    include("./Structure/Structure_Ion.jl")
    export Structure_Ion

    include("./Structure/Structure_VMD.jl")
    export Structure_VMD

    # generator Files
    include("./generator/genr.jl")
    export genr, genr_cell, genr_atom
    export genr_bond, genr_angle, genr_dihedral, genr_improper

    include("./generator/modify.jl")
    export move, find, select, remove!, remove_else!

    include("./generator/genr_ions.jl")
    export genr_ions

    include("./generator/combine.jl")
    export cat_data, sort_data!

    # Potential Files
    include("./Potential/Potential_Charmm36.jl")
    export Potential_Charmm36, Potential_Charmm36_Unit, potential_charmm36

    # converter Files
    include("./converter/useful_funcs.jl")

    include("converter/converter_vmd.jl")
    export converter_vmd

    # writeer Files
    include("./writer/write_data.jl")
    export write_data, write_info

    include("./writer/write_xyz.jl")
    export write_xyz

    include("./writer/write_pdb.jl")
    export write_pdb

end # module


using PyCall:PyObject, PyReverseDims, pyimport_conda

struct PyModel <: Model
    pyobject::PyObject
end

# lazily import a python model
pyod_import(name::String) = () -> begin
    module_name = lowercase(name)
    # names are not consistently uppercased
    model_name = name == "IForest" ? name : uppercase(name)
    getproperty(pyimport_conda("pyod.models.$(module_name)", "pyod", "conda-forge"), model_name)
end
pyod_import(name::Symbol) = pyod_import(String(name))

make_docs_link(name::String) =
    "<https://pyod.readthedocs.io/en/latest/pyod.models.html#module-pyod.models.$(lowercase(name))>"

"""
    pyod_fit(modelname, params)

Implements the `fit` method for an underlying python model.
"""
function pyod_fit(modelname, params)
    pymodelname = String(modelname)[3:end]
    quote
        function fit(model::$modelname, X::Data)::Fit
            Xt = PyReverseDims(X) # from column-major to row-major
            # load the underlying python model with key-word arguments
            detector = pyod_import($pymodelname)()($((Expr(:kw, p, :(model.$p)) for p in params)...))
            detector.fit(Xt)
            # the underlying python model is out model
            return Fit(PyModel(detector), detector.decision_scores_)
        end
    end
end

"""
    pyod_score(modelname)

Implements the `score` method for an underlying python model.
"""
function pyod_score(modelname)
    quote
        function score(_::$modelname, fitresult::Fit, X::Data)::Result
            Xt = PyReverseDims(X) # change from column-major to row-major
            scores_test = fitresult.model.pyobject.decision_function(Xt)
            return fitresult.scores, scores_test
        end
    end
end

"""
    py_constructor(expr)
Extracts the relevant information from the expr and build the expression
corresponding to the model constructor (see [`_model_constructor`](@ref)).
"""
function py_constructor(expr)
    # similar to @mlj_model
    expr, modelname, params, defaults, constraints = MMI._process_model_def(@__MODULE__, expr)

    # keyword constructor
    const_expr = MMI._model_constructor(modelname, params, defaults)

    # associate the constructor with the definition of the struct
    push!(expr.args[3].args, const_expr)

    # cleaner
    clean_expr = MMI._model_cleaner(modelname, defaults, constraints)

    # return
    return modelname, params, clean_expr, expr
end

macro pymodel(expr)
    modelname, params, clean_expr, expr = py_constructor(expr)
    @assert startswith(String(modelname), "Py") "A python model name has to start with `Py`, e.g. PyKNN"
    expr = quote
        Base.@__doc__($expr)
        $clean_expr
        $(pyod_fit(modelname, params))
        $(pyod_score(modelname))
        MMI.metadata_pkg($modelname,
            package_name="OutlierDetection.jl",
            package_uuid="262411bb-c475-4342-ba9e-03b8c0183ca6",
            package_url="https://github.com/davnn/OutlierDetection.jl",
            is_pure_julia=false,
            package_license="MIT",
            is_wrapper=false
        )
    end
    esc(expr)
end


using ImageView, FixedPointNumbers, Reactive, ColorTypes
using Test

@testset "contrast GUI" begin
    # test for the fix in #119
    imgbw = rand(N0f16, 100, 100)
    imgc = rand(RGB, 100, 100)
    for img in (imgbw, imgc)
        clim = ImageView.default_clim(img)
        imgsig = Signal(img)
        enabled, histsig, imgc = ImageView.prep_contrast(imgsig, clim)
        push!(enabled, true)
        yield()
        ret = ImageView.contrast_gui(enabled, histsig, clim)
        sleep(1.0)
        if isa(ret, Vector) #one gui dict per channel for color images
            for r in ret
                @test isa(r, Dict)
                Gtk.destroy(r["window"])
            end
        else
            @test isa(ret, Dict)
            Gtk.destroy(ret["window"])
        end
        # issue #168
        h = value(value(histsig)[1])
        fill!(h.weights, 0)
        push!(enabled, false)
        yield()
        h = value(value(histsig)[1])
        @test sum(h.weights) > 0
    end
end


# residual_evaluation.jl: some functions used by NonlinearSolvers relating to
# residual evaluation

import Utils.array1DTo3D

"""
  This function computes the vector form of of the residual from the vector
  form of the solution, ie. q_vec -> rhs_vec, for a given physics.
  This is one of the two functions required by [`newtonInner`](@ref).

  Users of newtonInner that are not physics modules (for example, implicit
  time marching schemes) will need to implement their own version of this
  function.  See the Implimentation Notes section below.

  **Inputs**

   * mesh: an AbstractMesh
   * sbp: an AbstractOperator
   * eqn: an AbstractSolutionData (may be modified during this function)
   * opts: the options dictionary
   * ctx_residual: a tuple of values.  ctx_residual[1] must be a function
                   that computes (q -> res).  Typically this is evalResidual.
                   The other entries of the tuple (if any) are not used.

                   The purpose of this argument is to make the signature of
                   the function generic enough so that implict time marching
                   methods can use it.  (If you are confused about this
                   programming pattern, google how callback are implemented
                   in C, this ctx is like a void* in C).

   * t: the time at which to evalute the residual

  **Inputs/Outputs**

   * rhs_vec: vector to put the residual in

  **Output**

   * norm of the residual vector

  **Implementation Notes:**

  This function is really a wrapper around an evalResidual-like function.
  It has to do 5 things:

    1. scatter eqn.q_vec -> eqn.q
    2. start parallel communication if needed
    3. call the evalResidual-like function to compute eqn.q -> eqn.res
    4. assemble the residual into the output vector, ie. eqn.res -> rhs_vec
    5. compute the norm of the rhs_vec

  Any implementation of this function (used with newtonInner) must have the
  following properties:

    1. on exit, eqn.q and eqn.q_vec must be consistent
    2. this function start parallel communication if needed
    3. allow for the possibility that rhs_vec and eqn.res_vec alias

  Is is recommended to use [`calcNorm`](@ref Utils.calcNorm) to compute the norm.

  Other implementations of this function are encouraged to use this function
  to help construct their rhs_vec.
"""
function physicsRhs(mesh, sbp, eqn, opts, rhs_vec, ctx_residual, t=0.0)

  # q_vec -> q
  array1DTo3D(mesh, sbp, eqn, opts, eqn.q_vec)

  # start parallel communication if needed
  time = eqn.params.time
  time.t_send += @elapsed if opts["parallel_type"] == 2 && mesh.npeers > 0
    startSolutionExchange(mesh, sbp, eqn, opts)
  end

  # evaluate (q -> res)
  func = ctx_residual[1]
  func(mesh, sbp, eqn, opts, t)

  # res -> rhs_vec
  assembleResidual(mesh, sbp, eqn, opts, rhs_vec, assemble_edgeres=false)

  # calculate the norm
  rhs_0_norm = calcNorm(eqn, rhs_vec, strongres=true)

  return rhs_0_norm
end

"""
### NonlinearSolvers.assembleResidual

  This function takes the residual in eqn.res and performs an additive reduction
  into res_vec.
  This function wraps array3DTo1D.

  Inputs:
    mesh: AbstractMesh
    sbp:  SBP operator
    eqn:  AbstractEquation object
    opts: options dictionary
    res_vec: residual vector to put the residual into

  Outputs:
    none

  Aliasing restrictions: none

"""
function assembleResidual(mesh, sbp, eqn, opts, res_vec::AbstractArray{T, 1};
                          assemble_edgeres=true, zero_resvec=true) where T
# assembles all of the residuals into res_vec
# no aliasing concerns
#TODO: clarify zero_resvec (why not used for both calls?)

  array3DTo1D(mesh, sbp, eqn, opts, eqn.res, res_vec)

  if assemble_edgeres

    for i=1:size(eqn.res_edge, 4)
      array3DTo1D(mesh, sbp, eqn, opts, sview(eqn.res_edge, :, :, :, i),
                           res_vec, zero_resvec=zero_resvec)
    end
  end

  return nothing
end

@doc """
### NonlinearSolvers.array1DTo3D

  This function performs the scatter q_vec -> eqn.q

  Inputs:
    mesh: AbstractMesh
    sbp:  SBP operator
    eqn:  AbstractEquation object
    opts: options dictionary
    q_vec: vector containing solution variables

  Outputs:
    none

  Aliasing Restrictions: none

"""->
function array1DTo3D(mesh, sbp, eqn, opts, q_vec::AbstractArray{T, 1}) where T
# scatters the q_vec to the 3d array eqn.q
# no aliasing concerns here
  array1DTo3D(mesh, sbp, eqn, opts, q_vec, eqn.q)

  return nothing
end


# This Map has non-trivial domain, thus we need the define testargs
"""
    Reindex(values) -> Map
"""
struct Reindex{A} <: Map
  values::A
end

function testargs(k::Reindex,i)
  @check length(k.values) !=0 "This map has empty domain"
  (one(i),)
end
function testargs(k::Reindex,i::Integer...)
  @check length(k.values) !=0 "This map has empty domain"
  map(one,i)
end
function return_value(k::Reindex,i...)
  length(k.values)!=0 ? evaluate(k,testargs(k,i...)...) : testitem(k.values)
end
return_cache(k::Reindex,i...) = array_cache(k.values)
evaluate!(cache,k::Reindex,i...) = getindex!(cache,k.values,i...)
evaluate(k::Reindex,i...) = k.values[i...]

#"""
#    reindex(i_to_v::AbstractArray, j_to_i::AbstractArray)
#"""
#function reindex(i_to_v::AbstractArray, j_to_i::AbstractArray)
#  lazy_map(Reindex(i_to_v),j_to_i)
#end

function lazy_map(k::Reindex{<:Fill},::Type{T}, j_to_i::AbstractArray) where T
  v = k.values.value
  Fill(v,size(j_to_i)...)
end

function lazy_map(k::Reindex{<:CompressedArray},::Type{T}, j_to_i::AbstractArray) where T
  i_to_v = k.values
  values = i_to_v.values
  ptrs = lazy_map(Reindex(i_to_v.ptrs),j_to_i)
  CompressedArray(values,ptrs)
end

function lazy_map(k::Reindex{<:LazyArray},::Type{T},j_to_i::AbstractArray) where T
  i_to_maps = k.values.maps
  i_to_args = k.values.args
  j_to_maps = lazy_map(Reindex(i_to_maps),eltype(i_to_maps),j_to_i)
  j_to_args = map(i_to_fk->lazy_map(Reindex(i_to_fk),eltype(i_to_fk),j_to_i), i_to_args)
  LazyArray(T,j_to_maps,j_to_args...)
end

# This optimization is important for surface-coupled problems
function lazy_map(k::Reindex{<:LazyArray{<:Fill{<:PosNegReindex}}},::Type{T},j_to_i::AbstractArray) where T
  i_to_iposneg = k.values.args[1]
  ipos_to_value = k.values.maps.value.values_pos
  ineg_to_value = k.values.maps.value.values_neg
  if aligned_with_pos(i_to_iposneg,j_to_i,length(ipos_to_value))
    ipos_to_value
  elseif aligned_with_neg(i_to_iposneg,j_to_i,length(ineg_to_value))
    ineg_to_value
  elseif all_in_pos(i_to_iposneg,j_to_i)
    j_to_ipos = lazy_map(Reindex(get_array(i_to_iposneg)),j_to_i)
    j_to_value = lazy_map(Reindex(ipos_to_value),j_to_ipos)
  elseif all_in_neg(i_to_iposneg,j_to_i)
    j_to_ineg = lazy_map(Reindex(get_array(i_to_iposneg)),j_to_i)
    j_to_value = lazy_map(Reindex(ineg_to_value),lazy_map(ineg->-ineg,j_to_ineg))
  else
    j_to_iposneg = lazy_map(Reindex(get_array(i_to_iposneg)),j_to_i)
    j_to_value = lazy_map(PosNegReindex(ipos_to_value,ineg_to_value),j_to_iposneg)
  end
end

function lazy_map(k::Reindex{<:LazyArray{<:Fill{<:PosNegReindex}}},::Type{T},indices::IdentityVector) where T
  @check length(k.values) == length(indices)
  k.values
end

function lazy_map(k::Reindex{<:LazyArray},::Type{T},indices::IdentityVector) where T
  @check length(k.values) == length(indices)
  k.values
end

function lazy_map(k::Reindex{<:AbstractArray},::Type{T},indices::IdentityVector) where T
  @check length(k.values) == length(indices)
  k.values
end

function lazy_map(k::Reindex{<:Fill},::Type{T},indices::IdentityVector) where T
  @check length(k.values) == length(indices)
  k.values
end

function lazy_map(k::Reindex{<:CompressedArray},::Type{T},indices::IdentityVector) where T
  @check length(k.values) == length(indices)
  k.values
end

@propagate_inbounds function Base.setindex!(a::LazyArray{<:Fill{<:Reindex}},v,j::Integer)
  k = a.maps.value
  i_to_v = k.values
  j_to_i, = a.args
  i = j_to_i[j]
  i_to_v[i]=v
end


using Flux

function get_optimizer(parsed::Dict, prefix="")
    opt_key = prefix == "" ? "opt" : join([prefix, "opt"], "_")
    opt_string = parsed[opt_key]
    get_optimizer(opt_string, parsed, prefix)
end

function get_optimizer(opt_string, parsed::Dict, prefix)

    if opt_string == "Auto+Descent"
        auto = _init_optimizer(Auto, parsed, "$(prefix)_auto")
        sgd = get_optimizer("Descent", parsed, prefix)
        (auto, sgd)
    elseif contains(opt_string, '+')
        opt_1_str, opt_2_str = split(opt_string, '+')
        (get_optimizer(opt_1_str, parsed, prefix), get_optimizer(opt_2_str, parsed, prefix))
    else
        opt_type = if opt_string == "Auto"
            Auto
        else
            getproperty(Flux, Symbol(opt_string))
        end
        _init_optimizer(opt_type, parsed, prefix)
    end
end

_init_optimizer(opt, ::Dict) =
    throw("$(string(opt)) optimizer initialization not found.")

function _init_optimizer(opt_type::Union{Type{Descent}, Type{ADAGrad}, Type{ADADelta}}, parsed::Dict, prefix = "")
    eta_str = prefix == "" ? "eta" : join([prefix, "eta"], "_")
    try
        η = parsed[eta_str]
        opt_type(η)
    catch
        throw("$(opt_type) needs: $(eta_str) (float).")
    end
end

function _init_optimizer(opt_type::Union{Type{RMSProp},
                                         Type{Momentum},
                                         Type{Nesterov}},
                         parsed::Dict, prefix="")

    eta_str = prefix == "" ? "eta" : join([prefix, "eta"], "_")
    rho_str = prefix == "" ? "rho" : join([prefix, "rho"], "_")
    try
        η = parsed[eta_str]
        ρ = parsed[rho_str]
        opt_type(η, ρ)
    catch
        throw("$(opt_type) needs: $(eta_str) (float), and $(rho_str) (float).")
    end
end

function _init_optimizer(opt_type::Union{Type{ADAM},
                                         Type{RADAM},
                                         Type{NADAM},
                                         Type{AdaMax},
                                         Type{AMSGrad}},
                         parsed::Dict, prefix="")

    eta_str = prefix == "" ? "eta" : join([prefix, "eta"], "_")
    beta_str = prefix == "" ? "beta" : join([prefix, "beta"], "_")
    try
        η = parsed[eta_str]
        β = if beta_str ∈ keys(parsed)
            parsed[beta_str]
        else
            (parsed[join([beta_str, "m"], "_")],
             parsed[join([beta_str, "v"], "_")])
        end
        opt_type(η, β)
    catch
        throw("$(opt_type) needs: $(eta_str) (float), and $(beta_str) ((float, float)), or (beta_m, beta_v)).")
    end
end

function _init_optimizer(opt_type::Union{Type{Auto}}, parsed::Dict, prefix="")
    α_str = prefix == "" ? "eta" : join([prefix, "eta"], "_")
    α_init_str = prefix == "" ?  "alpha_init" : join([prefix, "alpha_init"], "_")
    try
        α = parsed[α_str]
        α_init = parsed[α_init_str]
        opt_type(α, α_init)
    catch
        throw("$(opt_type) needs: $(α_str) (float), and $(α_init_str) (float).")
    end
end

function get_linear_learner(parsed::Dict,
                            feature_size::Int,
                            num_actions::Int,
                            num_demons::Int,
                            num_tasks::Int,
                            prefix="",
                            feature_projector = nothing)

    learner_key = prefix == "" ? "learner" : join([prefix, "learner"], "_")
    opt = get_optimizer(parsed, prefix)
    lu = get_learning_update(parsed, opt, prefix)
    w_init_key = prefix == "" ? "w_init" : join([prefix, "w_init"], "_")
    w_init = w_init_key in keys(parsed) ? parsed[w_init_key] : 0

    learner_str = parsed[learner_key]
    demon_learner = if learner_str ∈ ["Q", "QLearner", "q"]
        LinearQLearner(lu,
                       feature_size,
                       num_actions,
                       num_demons,
                       w_init)
    elseif learner_str ∈ ["SR", "SRLearner", "sr"]
        SRLearner(lu,
                  feature_size,
                  num_demons,
                  num_actions,
                  num_tasks,
                  feature_projector,
                  w_init)
    elseif learner_str ∈ ["GPI", "gpi"]
        GPI(lu,
            feature_size,
            num_demons,
            num_actions,
            num_tasks,
            feature_projector,
            w_init)
    elseif learner_str ∈ ["LSTD", "LSTDLearner"]
        eta_str = prefix == "" ? "eta" : join([prefix, "eta"], "_")
        LSTDLearner(lu,
                    parsed[eta_str],
                    feature_size,
                    num_actions,
                    num_demons)
    elseif learner_str ∈ ["NoLearner", "nolearner"]
        NoLearner(collect(1:num_actions), num_demons)
    else
        throw(ArgumentError("Not a valid demon learner"))
    end
end

function get_linear_learner(parsed::Dict,
                            feature_size,
                            num_actions,
                            demons,
                            prefix="",
                            feature_projector=nothing)

    # Don't want to change call now. num_tasks only exists for SRHorde demons.
    num_tasks = if !hasproperty(demons, :num_tasks)
        0
    else
        demons.num_tasks
    end
    return get_linear_learner(parsed::Dict,
        feature_size,
        num_actions,
        length(demons),
        num_tasks,
        prefix,
        feature_projector)
end

function get_learning_update(parsed::Dict, opt, prefix="")
    lu_key = prefix == "" ? "update" : join([prefix, "update"], "_")
    lu = getproperty(Curiosity, Symbol(parsed[lu_key]))
    _init_learning_update(lu, opt, parsed, prefix)
end

_init_learning_update(lu_type, args...) =
    throw("$(string(lu_type)) does not have an init function.")

function _init_learning_update(lu_type::Union{Type{TabularRoundRobin}}, args...)
    lu_type()
end

function _init_learning_update(lu_type::Union{Type{TB}},
                               opt,
                               parsed::Dict,
                               prefix)
    λ_str = prefix == "" ? "lambda" : join([prefix, "lambda"], "_")
    try
        λ = parsed[λ_str]
        lu_type(lambda=λ, opt=opt)
    catch
        throw("$(lu_type) needs: $(λ_str) (float).")
    end
end

function _init_learning_update(lu_type::Union{Type{InterestTB}},
                                opt,
                                parsed::Dict,
                                prefix)
    λ_str = prefix == "" ? "lambda" : join([prefix, "lambda"], "_")
    interest_set_str = prefix == "" ? "interest_set" : join([prefix, "interest_set"], "_")
    norm_interest_str = prefix == "" ? "interest_set" : join([prefix, "normalize_interest"], "_")

    try
        lambda = parsed[λ_str]
        interest_set = parsed[interest_set_str]
        norm_interest = parsed[norm_interest_str]

        hdpi = get_hdpi(parsed, interest_set)
        lu_type(lambda=lambda, opt=opt, hdpi=hdpi, normalize_interest=norm_interest)

    catch e
        throw(e)
        # throw("$(lu_type) needs: $(interest_set_str) (string)")
    end
end

function _init_learning_update(lu_type::Union{Type{TB}, Type{PriorESARSA}, Type{PriorTB}},
                                opt,
                                parsed::Dict,
                                prefix)
    λ_str = prefix == "" ? "lambda" : join([prefix, "lambda"], "_")
    try
        λ = parsed[λ_str]

        if haskey(parsed, "clip_threshold")
            clip_threshold = parsed["clip_threshold"]
        else
            clip_threshold = Inf64
            println("Setting clip threshold to inf")
        end

        lu_type(lambda=λ, opt=opt, clip_threshold=clip_threshold)
    catch
        throw("$(lu_type) needs: $(λ_str) (float).")
    end
end

function _init_learning_update(lu_type::Union{Type{ETB},Type{EmphESARSA},Type{StateETB}},
                                opt,
                                parsed::Dict,
                                prefix)
    λ_str = prefix == "" ? "lambda" : join([prefix, "lambda"], "_")
    interest_set_str = prefix == "" ? "interest_set" : join([prefix, "interest_set"], "_")
    norm_interest_str = prefix == "" ? "interest_set" : join([prefix, "normalize_interest"], "_")

    # try
    lambda = parsed[λ_str]
    interest_set = parsed[interest_set_str]
    norm_interest = parsed[norm_interest_str]

    if haskey(parsed, "emphasis_clip_threshold")
        clip_threshold = parsed["emphasis_clip_threshold"]
    else
        clip_threshold = Inf64
        println("Setting clip threshold to inf")
    end

    hdpi = get_hdpi(parsed, interest_set)
    lu_type(lambda=lambda, opt=opt, hdpi=hdpi, clip_threshold=clip_threshold, normalize_interest=norm_interest)

    # catch e
    #     throw(e)
    # throw("$(lu_type) needs: $(interest_set_str) (string)")
    # end
end

function _init_learning_update(lu_type::Union{Type{ESARSA}, Type{SARSA}},
                                opt,
                                parsed::Dict,
                                prefix)
    λ_str = prefix == "" ? "lambda" : join([prefix, "lambda"], "_")
    trace_str = prefix == "" ? "trace" : join([prefix, "trace"], "_")
    trace = try
        getproperty(Curiosity, Symbol(parsed[trace_str]))
    catch
        throw("$(lu_type) needs trace type")
    end

    try
        λ = parsed[λ_str]
        lu_type(lambda=λ, opt=opt, trace=trace())
    catch
        throw("$(lu_type) needs: $(λ_str) (float).")
    end
end

function get_exploration_strategy(parsed, action_set)
    if parsed["exploration_strategy"] == "epsilon_greedy"
        EpsilonGreedy(parsed["exploration_param"])
    elseif parsed["exploration_strategy"] == "epsilon_greedy_decay"
        ϵGreedyDecay(
            parsed["ϵ_range"],
            parsed["decay_period"],
            parsed["warmup_steps"],
            action_set
        )
    else
        throw(ArgumentError("Not a Valid Exploration Strategy"))
    end
end

Base.@kwdef struct NoLearner <: Learner
    action_set::Array{Int,1}
    num_demons::Int
end

Curiosity.update!(learner::NoLearner, args...) = nothing

Base.get(π::NoLearner; state_t, action_t, kwargs...) =
    get_action_probs(π, state_t, nothing)[action_t]

function Curiosity.get_action_probs(l::NoLearner, features, state)
    return ones(l.num_demons,length(action_set)) ./ length(action_set)
end
(l::NoLearner)(ϕ,a) = zeros(l.num_demons)
(l::NoLearner)(ϕ) = [zeros(l.num_demons) for a in l.action_set]
function zero_eligibility_traces!(l::NoLearner) nothing end

function get_hdpi(parsed, interest_set)
    if (interest_set == "ttmaze")
        if (parsed["exploring_starts"] == false)
            @load "./src/data/dpi/ttmaze_exploring_starts_false.jld2" hdpi
            return hdpi
        end
    elseif (interest_set == "oned_tmaze")
        if (parsed["exploring_starts"] == "beg")
            @load "./src/data/dpi/oned_tmaze_beg.jld2" hdpi
            return hdpi
        end
    elseif (interest_set == "2dOpenWorld_center")
        @load "./src/data/dpi/2dOpenWorld_center.jld2" hdpi
        return hdpi
    end
    throw("no valid hdpi for string $(interest_set)")
end


"""
    Evaluator

An `Evaluator` is used to perform arithmetic and other operations on `Ciphertext` instances. These
include addition, multiplication, relinearization, and modulus switching.

See also: [`Ciphertext`](@ref)
"""
mutable struct Evaluator <: SEALObject
  handle::Ptr{Cvoid}

  function Evaluator(context::SEALContext)
    handleref = Ref{Ptr{Cvoid}}(C_NULL)
    retval = ccall((:Evaluator_Create, libsealc), Clong,
                   (Ptr{Cvoid}, Ref{Ptr{Cvoid}}),
                   context, handleref)
    @check_return_value retval
    return Evaluator(handleref[])
  end

  function Evaluator(handle::Ptr{Cvoid})
    object = new(handle)
    finalizer(destroy!, object)
    return object
  end
end

function destroy!(object::Evaluator)
  if isallocated(object)
    @check_return_value ccall((:Evaluator_Destroy, libsealc), Clong, (Ptr{Cvoid},), object)
    sethandle!(object, C_NULL)
  end

  return nothing
end

function square!(destination::Ciphertext, encrypted::Ciphertext, evaluator::Evaluator)
  retval = ccall((:Evaluator_Square, libsealc), Clong,
                 (Ptr{Cvoid}, Ptr{Cvoid}, Ptr{Cvoid}, Ptr{Cvoid}),
                 evaluator, encrypted, destination, C_NULL)
  @check_return_value retval
  return destination
end

function square_inplace!(encrypted::Ciphertext, evaluator::Evaluator)
  return square!(encrypted, encrypted, evaluator)
end

function relinearize!(destination::Ciphertext, encrypted::Ciphertext, relinkeys::RelinKeys,
                      evaluator::Evaluator)
  retval = ccall((:Evaluator_Relinearize, libsealc), Clong,
                 (Ptr{Cvoid}, Ptr{Cvoid}, Ptr{Cvoid}, Ptr{Cvoid}, Ptr{Cvoid}),
                 evaluator, encrypted, relinkeys, destination, C_NULL)
  @check_return_value retval
  return destination
end

function relinearize_inplace!(encrypted::Ciphertext, relinkeys::RelinKeys, evaluator::Evaluator)
  return relinearize!(encrypted, encrypted, relinkeys, evaluator)
end

function rescale_to_next!(destination::Ciphertext, encrypted::Ciphertext, evaluator::Evaluator)
  retval = ccall((:Evaluator_RescaleToNext, libsealc), Clong,
                 (Ptr{Cvoid}, Ptr{Cvoid}, Ptr{Cvoid}, Ptr{Cvoid}),
                 evaluator, encrypted, destination, C_NULL)
  @check_return_value retval
  return destination
end

function rescale_to_next_inplace!(encrypted::Ciphertext, evaluator::Evaluator)
  return rescale_to_next!(encrypted, encrypted, evaluator)
end

function multiply_plain!(destination::Ciphertext, encrypted::Ciphertext, plain::Plaintext,
                         evaluator::Evaluator)
  retval = ccall((:Evaluator_MultiplyPlain, libsealc), Clong,
                 (Ptr{Cvoid}, Ptr{Cvoid}, Ptr{Cvoid}, Ptr{Cvoid}, Ptr{Cvoid}),
                 evaluator, encrypted, plain, destination, C_NULL)
  @check_return_value retval
  return destination
end

function multiply_plain_inplace!(encrypted::Ciphertext, plain::Plaintext,
                                  evaluator::Evaluator)
  return multiply_plain!(encrypted, encrypted, plain, evaluator)
end

function multiply!(destination::Ciphertext, encrypted1::Ciphertext, encrypted2::Ciphertext,
                   evaluator::Evaluator)
  retval = ccall((:Evaluator_Multiply, libsealc), Clong,
                 (Ptr{Cvoid}, Ptr{Cvoid}, Ptr{Cvoid}, Ptr{Cvoid}, Ptr{Cvoid}),
                 evaluator, encrypted1, encrypted2, destination, C_NULL)
  @check_return_value retval
  return destination
end

function multiply_inplace!(encrypted1::Ciphertext, encrypted2::Ciphertext, evaluator::Evaluator)
  return multiply!(encrypted1, encrypted1, encrypted2, evaluator)
end

function mod_switch_to!(destination::Ciphertext, encrypted::Ciphertext,
                        parms_id::DenseVector{UInt64}, evaluator::Evaluator)
  retval = ccall((:Evaluator_ModSwitchTo1, libsealc), Clong,
                 (Ptr{Cvoid}, Ptr{Cvoid}, Ref{UInt64}, Ptr{Cvoid}, Ptr{Cvoid}),
                 evaluator, encrypted, parms_id, destination, C_NULL)
  @check_return_value retval
  return destination
end

function mod_switch_to_inplace!(encrypted::Ciphertext, parms_id::DenseVector{UInt64},
                                evaluator::Evaluator)
  return mod_switch_to!(encrypted, encrypted, parms_id, evaluator)
end

function mod_switch_to_next!(destination::Ciphertext, encrypted::Ciphertext, evaluator::Evaluator)
  retval = ccall((:Evaluator_ModSwitchToNext1, libsealc), Clong,
                 (Ptr{Cvoid}, Ptr{Cvoid}, Ptr{Cvoid}, Ptr{Cvoid}),
                 evaluator, encrypted, destination, C_NULL)
  @check_return_value retval
  return destination
end

function mod_switch_to_next_inplace!(encrypted::Ciphertext, evaluator::Evaluator)
  return mod_switch_to_next!(encrypted, encrypted, evaluator)
end

function mod_switch_to!(destination::Plaintext, plain::Plaintext, parms_id::DenseVector{UInt64},
                        evaluator::Evaluator)
  retval = ccall((:Evaluator_ModSwitchTo2, libsealc), Clong,
                 (Ptr{Cvoid}, Ptr{Cvoid}, Ref{UInt64}, Ptr{Cvoid}),
                 evaluator, plain, parms_id, destination)
  @check_return_value retval
  return destination
end

function mod_switch_to_inplace!(plain::Plaintext, parms_id::DenseVector{UInt64},
                                evaluator::Evaluator)
  return mod_switch_to!(plain, plain, parms_id, evaluator)
end

function add!(destination::Ciphertext, encrypted1::Ciphertext, encrypted2::Ciphertext,
              evaluator::Evaluator)
  retval = ccall((:Evaluator_Add, libsealc), Clong,
                 (Ptr{Cvoid}, Ptr{Cvoid}, Ptr{Cvoid}, Ptr{Cvoid}),
                 evaluator, encrypted1, encrypted2, destination)
  @check_return_value retval
  return destination
end

function add_inplace!(encrypted1::Ciphertext, encrypted2::Ciphertext, evaluator::Evaluator)
  return add!(encrypted1, encrypted1, encrypted2, evaluator)
end

function add_plain!(destination::Ciphertext, encrypted::Ciphertext, plain::Plaintext,
                    evaluator::Evaluator)
  retval = ccall((:Evaluator_AddPlain, libsealc), Clong,
                 (Ptr{Cvoid}, Ptr{Cvoid}, Ptr{Cvoid}, Ptr{Cvoid}),
                 evaluator, encrypted, plain, destination)
  @check_return_value retval
  return destination
end

function add_plain_inplace!(encrypted::Ciphertext, plain::Plaintext, evaluator::Evaluator)
  return add_plain!(encrypted, encrypted, plain, evaluator)
end

function rotate_vector!(destination::Ciphertext, encrypted::Ciphertext, steps,
                        galois_keys::GaloisKeys, evaluator::Evaluator)
  retval = ccall((:Evaluator_RotateVector, libsealc), Clong,
                 (Ptr{Cvoid}, Ptr{Cvoid}, Cint, Ptr{Cvoid}, Ptr{Cvoid}, Ptr{Cvoid}),
                 evaluator, encrypted, steps, galois_keys, destination, C_NULL)
  @check_return_value retval
  return destination
end

function rotate_vector_inplace!(encrypted::Ciphertext, steps, galois_keys::GaloisKeys,
                                evaluator::Evaluator)
  return rotate_vector!(encrypted, encrypted, steps, galois_keys, evaluator)
end

function rotate_rows!(destination::Ciphertext, encrypted::Ciphertext, steps,
                      galois_keys::GaloisKeys, evaluator::Evaluator)
  retval = ccall((:Evaluator_RotateRows, libsealc), Clong,
                 (Ptr{Cvoid}, Ptr{Cvoid}, Cint, Ptr{Cvoid}, Ptr{Cvoid}, Ptr{Cvoid}),
                 evaluator, encrypted, steps, galois_keys, destination, C_NULL)
  @check_return_value retval
  return destination
end

function rotate_rows_inplace!(encrypted::Ciphertext, steps, galois_keys::GaloisKeys,
                              evaluator::Evaluator)
  return rotate_rows!(encrypted, encrypted, steps, galois_keys, evaluator)
end

function rotate_columns!(destination::Ciphertext, encrypted::Ciphertext, galois_keys::GaloisKeys,
                         evaluator::Evaluator)
  retval = ccall((:Evaluator_RotateColumns, libsealc), Clong,
                 (Ptr{Cvoid}, Ptr{Cvoid}, Ptr{Cvoid}, Ptr{Cvoid}, Ptr{Cvoid}),
                 evaluator, encrypted, galois_keys, destination, C_NULL)
  @check_return_value retval
  return destination
end

function rotate_columns_inplace!(encrypted::Ciphertext, galois_keys::GaloisKeys,
                                 evaluator::Evaluator)
  return rotate_columns!(encrypted, encrypted, galois_keys, evaluator)
end

function complex_conjugate!(destination::Ciphertext, encrypted::Ciphertext, galois_keys::GaloisKeys,
                            evaluator::Evaluator)
  retval = ccall((:Evaluator_ComplexConjugate, libsealc), Clong,
                 (Ptr{Cvoid}, Ptr{Cvoid}, Ptr{Cvoid}, Ptr{Cvoid}, Ptr{Cvoid}),
                 evaluator, encrypted, galois_keys, destination, C_NULL)
  @check_return_value retval
  return destination
end

function complex_conjugate_inplace!(encrypted::Ciphertext, galois_keys::GaloisKeys,
                                    evaluator::Evaluator)
  return complex_conjugate!(encrypted, encrypted, galois_keys, evaluator)
end

function negate!(destination::Ciphertext, encrypted::Ciphertext, evaluator::Evaluator)
  retval = ccall((:Evaluator_Negate, libsealc), Clong,
                 (Ptr{Cvoid}, Ptr{Cvoid}, Ptr{Cvoid}),
                 evaluator, encrypted, destination)
  @check_return_value retval
  return destination
end

function negate_inplace!(encrypted::Ciphertext, evaluator::Evaluator)
  return negate!(encrypted, encrypted, evaluator)
end

function sub!(destination::Ciphertext, encrypted1::Ciphertext, encrypted2::Ciphertext,
              evaluator::Evaluator)
  retval = ccall((:Evaluator_Sub, libsealc), Clong,
                 (Ptr{Cvoid}, Ptr{Cvoid}, Ptr{Cvoid}, Ptr{Cvoid}),
                 evaluator, encrypted1, encrypted2, destination)
  @check_return_value retval
  return destination
end

function sub_inplace!(encrypted1::Ciphertext, encrypted2::Ciphertext, evaluator::Evaluator)
  return sub!(encrypted1, encrypted1, encrypted2, evaluator)
end