# test solver initialization
let
    DefaultAStarSC()
    AStarSC{Int}()
    ISPS()
    CBSSolver()
    TaskGraphsMILPSolver()
    NBSSolver()
    solver = NBSSolver()
    best_cost(solver)
    NBSSolver(TaskGraphsMILPSolver(),CBSSolver())
end
let
    solver = NBSSolver()
    increment_iteration_count!(solver)
    increment_iteration_count!(route_planner(solver))
    @test max_iterations(solver) == 1
    @test iterations(solver) == 1
    reset_solver!(solver)
    @test max_iterations(solver) == 1
    @test iterations(solver) == 0
    hard_reset_solver!(solver)
    @test max_iterations(solver) == 0
    @test max_iterations(route_planner(solver)) == 0
    @test iterations(solver) == 0
end
# in-depth testing of solver stages on a single problem
let
    # init search env
    f = pctapf_problem_4
    cost_model = SumOfMakeSpans()
    solver = NBSSolver(assignment_model = TaskGraphsMILPSolver(GreedyAssignment()))
    pc_tapf = f(solver;cost_function=cost_model,verbose=false)
    base_env = pc_tapf.env
    prob = formulate_assignment_problem(solver.assignment_model,pc_tapf)
    sched, cost = solve_assignment_problem!(solver.assignment_model,prob,pc_tapf)
    @test validate(sched)
    # Test AStarSC
    let
        search_env = construct_search_env(solver,deepcopy(sched),base_env)
        pc_mapf = PC_MAPF(search_env)
        path_planner = DefaultAStarSC()
        node = initialize_root_node(search_env)
        # plan for Robot Start
        node_id = RobotID(1)
        schedule_node = get_node_from_id(search_env.schedule, node_id)
        v = get_vtx(search_env.schedule, node_id)
        @test plan_path!(path_planner, pc_mapf,search_env, node, schedule_node, v)
        # plan for GO
        v2 = outneighbors(search_env.schedule,v)[1]
        schedule_node = get_node_from_vtx(search_env.schedule, v2)
        @test plan_path!(path_planner, pc_mapf,search_env, node, schedule_node, v2)
        # @show convert_to_vertex_lists(node.solution)
    end
    let
        search_env = construct_search_env(solver,deepcopy(sched),base_env)
        pc_mapf = PC_MAPF(search_env)
        node = initialize_root_node(search_env)
        path_planner = ISPS()
        for i in 1:nv(search_env.schedule)
            @test plan_next_path!(path_planner,pc_mapf,node.solution,node)
        end
    end
    # Test ISPS
    let
        search_env = construct_search_env(solver,deepcopy(sched),base_env)
        path_planner = ISPS()
        node = initialize_root_node(search_env)
        @test compute_route_plan!(path_planner,PC_MAPF(search_env),node)#,node)
        # @show convert_to_vertex_lists(node.solution)
        # @show convert_to_vertex_lists(get_route_plan(search_env))
    end
    let
        search_env = construct_search_env(solver,deepcopy(sched),base_env)
        path_planner = ISPS()
        solution, cost = solve!(path_planner,PC_MAPF(search_env))
        # @show convert_to_vertex_lists(solution.route_plan)
    end
    let
        search_env = construct_search_env(solver,deepcopy(sched),base_env)
        path_planner = ISPS()
        node = initialize_root_node(search_env)
        @test low_level_search!(path_planner,PC_MAPF(search_env),node)
        # @show convert_to_vertex_lists(node.solution)
    end
    # Test CBS
    let
        search_env = construct_search_env(solver,deepcopy(sched),base_env)
        pc_mapf = PC_MAPF(search_env)
        path_planner = CBSSolver(ISPS())
        solution, cost = solve!(path_planner,pc_mapf)
        # solution, cost = CRCBS.cbs!(path_planner,pc_mapf)
        # @show convert_to_vertex_lists(solution.route_plan)
    end
    let
        # NOTE Freezing here
        path_planner = CBSSolver(ISPS())
        env, cost = plan_route!(path_planner,deepcopy(sched),pc_tapf)
        # @show convert_to_vertex_lists(get_route_plan(env))
    end
    # test NBS
    let
        solver = NBSSolver(assignment_model = TaskGraphsMILPSolver(GreedyAssignment()))
        # solver = NBSSolver(assignment_model = TaskGraphsMILPSolver())
        set_iteration_limit!(solver,3)
        env, cost = solve!(solver,base_env)
        # @show convert_to_vertex_lists(get_route_plan(env))
        # @show get_logger(solver)
        # @show optimality_gap(solver) > 0
        @test validate(env.schedule)
        @test validate(env.schedule,convert_to_vertex_lists(get_route_plan(env)))
    end
end
# test task assignment
let
    # init search env
    for (i, f) in enumerate(pctapf_test_problems())
        for cost_model in [SumOfMakeSpans(), MakeSpan()]
            let
                costs = Float64[]
                for solver in [
                        NBSSolver(assignment_model = TaskGraphsMILPSolver(AssignmentMILP())),
                        NBSSolver(assignment_model = TaskGraphsMILPSolver(SparseAdjacencyMILP())),
                        NBSSolver(assignment_model = TaskGraphsMILPSolver(GreedyAssignment())),
                        ]
                    # set_verbosity!(assignment_solver(solver),3)
                    # @show i, f, solver
                    pc_tapf = f(solver;cost_function=cost_model,verbose=false);
                    search_env = pc_tapf.env
                    prob = formulate_assignment_problem(solver.assignment_model,pc_tapf)
                    sched, cost = solve_assignment_problem!(solver.assignment_model,prob,pc_tapf)
                    if !isa(solver.assignment_model.milp,GreedyAssignment)
                        push!(costs, cost)
                    end
                    @test validate(sched)
                    @test cost != typemax(Int)
                    @test cost != Inf
                end
                # @show costs
                @test all(costs .== costs[1])
            end
        end
    end
end
# test full solver
let
    # init search env
    for (i, f) in enumerate([
        pctapf_problem_1,
        pctapf_problem_2,
        pctapf_problem_3,
        pctapf_problem_4,
        pctapf_problem_5,
        pctapf_problem_6,
        pctapf_problem_7,
        pctapf_problem_8,
        ])
        for cost_model in [SumOfMakeSpans(), MakeSpan()]
            let
                costs = Float64[]
                for solver in [
                        NBSSolver(assignment_model = TaskGraphsMILPSolver(AssignmentMILP())),
                        NBSSolver(assignment_model = TaskGraphsMILPSolver(SparseAdjacencyMILP())),
                        NBSSolver(assignment_model = TaskGraphsMILPSolver(GreedyAssignment())),
                        NBSSolver(
                            assignment_model = TaskGraphsMILPSolver(GreedyAssignment()),
                            path_planner = PIBTPlanner{NTuple{3,Float64}}()
                            ),
                        ]
                    # set_verbosity!(solver,3)
                    set_iteration_limit!(solver,1)
                    set_iteration_limit!(route_planner(solver),100)
                    pc_tapf = f(solver;cost_function=cost_model,verbose=false);
                    env, cost = solve!(solver,pc_tapf)
                    # @show convert_to_vertex_lists(get_route_plan(env))
                    if !isa(solver.assignment_model.milp,GreedyAssignment)
                        push!(costs, cost[1])
                    end
                    @test validate(env.schedule)
                    @test validate(env.schedule,convert_to_vertex_lists(get_route_plan(env)))
                    @test cost != typemax(Int)
                    @test cost != Inf
                end
                # @show costs
                @test all(costs .== costs[1])
            end
        end
    end
end
# Test that optimal collision-free planners obtain the correct costs
let
    cost_model = MakeSpan()
    for (i, (f,expected_cost)) in enumerate([
        (pctapf_problem_1,5),
        (pctapf_problem_2,8),
        (pctapf_problem_3,10),
        (pctapf_problem_4,3),
        (pctapf_problem_5,4),
        ])
        for solver in [
                NBSSolver(assignment_model = TaskGraphsMILPSolver(AssignmentMILP())),
                NBSSolver(assignment_model = TaskGraphsMILPSolver(SparseAdjacencyMILP())),
                ]
            let
                pc_tapf = f(solver;cost_function=cost_model,verbose=false);
                env, cost = solve!(solver,pc_tapf)
                @test expected_cost == cost
            end
        end
    end
    cost_model = SumOfMakeSpans()
    for (i, (f,expected_cost)) in enumerate([
        (pctapf_problem_1,5),
        (pctapf_problem_2,8),
        (pctapf_problem_3,10),
        (pctapf_problem_4,3),
        (pctapf_problem_5,4),
        (pctapf_problem_8,16),
        ])
        for solver in [
                NBSSolver(assignment_model = TaskGraphsMILPSolver(AssignmentMILP())),
                NBSSolver(assignment_model = TaskGraphsMILPSolver(SparseAdjacencyMILP())),
                ]
            let
                pc_tapf = f(solver;cost_function=cost_model,verbose=false);
                env, cost = solve!(solver,pc_tapf)
                @test expected_cost == cost
            end
        end
    end
end
# verify that ISPS+A*sc avoids a collision by exploiting slack
let
    solver = NBSSolver()
    cost_model = MakeSpan()
    pc_tapf = pctapf_problem_3(solver;cost_function=cost_model,verbose=false);
    base_env = pc_tapf.env

    prob = formulate_assignment_problem(assignment_solver(solver),pc_tapf)
    project_schedule, cost = solve_assignment_problem!(assignment_solver(solver),prob,pc_tapf)
    @test termination_status(prob) == MOI.OPTIMAL

    env = construct_search_env(solver,project_schedule,base_env)
    pc_mapf = PC_MAPF(env)
    node = initialize_root_node(solver,pc_mapf)
    low_level_search!(route_planner(solver),pc_mapf,node)

    path1 = convert_to_vertex_lists(node.solution.route_plan.paths[1])
    path2 = convert_to_vertex_lists(node.solution.route_plan.paths[2])
    @test path2[2] == 5 # test that robot 2 will indeed wait for robot 1
    @test path1 == [2, 6, 10, 14, 18, 22, 26, 30, 31, 32, 32]
    @test path2 == [5, 5, 6,  7,  8,  12, 12, 12, 12, 12, 16]
    # @show path1
    # @show path2
end
# Non-zero process time
let
    """
    Verify that the algorithm correctly handles scenarios where process time
    is non-zero
    """
    println("Test non-zero wait time:")
    for Δt in [0,1,4]
        let
            solver = NBSSolver()
            cost_model=MakeSpan()
            pc_tapf = pctapf_problem_6(solver;
                cost_function=cost_model,
                verbose=false,
                Δt_op=Δt
                );
            env, cost = solve!(solver,pc_tapf)
            paths = convert_to_vertex_lists(get_route_plan(env))
            # @show paths[1]
            # @show paths[2]
            @test paths[1][8+Δt] == 13
            @test paths[1][9+Δt] == 17
        end
    end
end
# Non-zero collection time
let
    """
    Verify that the algorithm correctly handles scenarios where collection time
    and deposit time are non-zero.
    """
    println("Test non-zero collection time:")
    Δt_deposit_1 = 1
    for (Δt_collect_2, true_cost) in zip([0,1,2,3,4],[6,7,8,8,8])
        solver = NBSSolver()
        cost_model = MakeSpan()
        pc_tapf = pctapf_problem_7(solver;
            cost_function=cost_model,
            verbose=false,
            Δt_op=0,
            Δt_collect=[0,Δt_collect_2,0],
            Δt_deposit=[Δt_deposit_1,0,0]
            );
        env, cost = solve!(solver,pc_tapf)
        paths = convert_to_vertex_lists(get_route_plan(env))
        @test cost == true_cost
    end
end
# Collaborative transport problems
let
    cost_model = MakeSpan()
    for (i, (f,expected_cost,expected_paths)) in enumerate([
        (pctapf_problem_11, 6, [
            [1,2,6,10,14,13,9],
            [4,3,7,11,15],
            [15,11,10,9,5,1],
        ]),
        ])
        for solver in [
                NBSSolver(assignment_model = TaskGraphsMILPSolver(SparseAdjacencyMILP())),
                ]
            let
                pc_tapf = f(solver;cost_function=cost_model);
                c_pc_tapf = C_PC_TAPF(pc_tapf.env)
                # set_verbosity!(solver,4)
                env, cost = solve!(solver,c_pc_tapf)
                paths = convert_to_vertex_lists(get_route_plan(env))
                @log_info(-1,verbosity(solver),sprint_route_plan(get_route_plan(env)))
                # @show paths
                @test cost == expected_cost
                for (expected_path, path) in zip(expected_paths,paths)
                    @test length(path) <= maximum(map(length,expected_paths))
                    @test path[1:length(expected_path)] == expected_path
                end
            end
        end
    end
end
# Gap repair problems
let
    solver = NBSSolver()
    set_iteration_limit!(low_level(route_planner(solver)),1)
    pc_tapf = pctapf_problem_12(solver)
    env, cost = solve!(solver,pc_tapf)
    cost
    # @show get_route_plan(env)
end
# Collaborative gap repair problems
let
    solver = NBSSolver()
    set_iteration_limit!(low_level(route_planner(solver)),1)
    pc_tapf = C_PC_TAPF(pctapf_problem_13(solver).env)
    env, cost = solve!(solver,pc_tapf)
    cost
    # @show get_route_plan(env)
end


# Calibration of Gent-McWilliams to a baroclinic adjustment problem
pushfirst!(LOAD_PATH, joinpath(@__DIR__, ".."))

using ParameterEstimocean
using Oceananigans
using Oceananigans.Units
using Oceananigans.Models.HydrostaticFreeSurfaceModels: SliceEnsembleSize
using Oceananigans.TurbulenceClosures: FluxTapering
using LinearAlgebra, CairoMakie, DataDeps, JLD2

using Oceananigans.Architectures: arch_array

# using ElectronDisplay

architecture = CPU()

# filedir = @__DIR__
# filename = "baroclinic_adjustment_double_Lx_zonal_average.jld2"
# filepath = joinpath(filedir, filename)
# Base.download("https://www.dropbox.com/s/f8zsb33vwwwmjjm/$filename", filepath)

# filepath = "/Users/navid/Research/mesoscale-parametrization-OSM2022/baroclinic_adjustment-double_Lx/short_save_often_run/baroclinic_adjustment_double_Lx_zonal_average.jld2"
filepath = "baroclinic_adjustment_double_Lx_zonal_average_80dayrun.jld2"

file = jldopen(filepath)
coriolis = file["serialized/coriolis"]

# number of grid points
Nx, Ny, Nz = file["grid/Nx"], file["grid/Ny"], file["grid/Nz"]

# Domain
const Lx, Ly, Lz = file["grid/Lx"], file["grid/Ly"], file["grid/Lz"]

close(file)

field_names = (:b, :c, :u, :v)
forward_map_names = (:b, :c)

using ParameterEstimocean.Transformations: Transformation

transformation = (b = ZScore(),
                  c = ZScore(),
                  u = ZScore())

transformation = ZScore()

space_transformation = SpaceIndices(x=:, y=2:4:Ny-1, z=20:2:Nz-1)

transformation = (b = Transformation(space = space_transformation, normalization=ZScore()),
                  c = Transformation(space = space_transformation, normalization=ZScore()),
                  u = Transformation(space = space_transformation, normalization=ZScore()),
                  v = Transformation(space = space_transformation, normalization=RescaledZScore(1e-1)))

transformation = ZScore()

transformation = (b = Transformation(space = space_transformation, normalization=ZScore()),
                  c = Transformation(space = space_transformation, normalization=ZScore()))

times = [40days-12hours, 40days]

observations = SyntheticObservations(filepath; transformation, times, field_names, forward_map_names)

#####
##### Simulation
#####

Nensemble = 50

slice_ensemble_size = SliceEnsembleSize(size=(Ny, Nz), ensemble=Nensemble)

ensemble_grid = RectilinearGrid(architecture,
                                size = slice_ensemble_size,
                                topology = (Flat, Bounded, Bounded),
                                y = (-Ly/2, Ly/2),
                                z = (-Lz, 0),
                                halo=(3, 3))

@show ensemble_grid

file = jldopen(filepath)
closures = file["serialized/closure"]
close(file)

gent_mcwilliams_diffusivity = IsopycnalSkewSymmetricDiffusivity(slope_limiter = FluxTapering(1e-2))

gm_closure_ensemble = arch_array(architecture, [deepcopy(gent_mcwilliams_diffusivity) for _ = 1:Nensemble])

closure_ensemble = (gm_closure_ensemble, closures[1], closures[2])

@show "no convective adjustment"
closure_ensemble = (gm_closure_ensemble, closures[1])

ensemble_model = HydrostaticFreeSurfaceModel(grid = ensemble_grid,
                                             tracers = (:b, :c),
                                             buoyancy = BuoyancyTracer(),
                                             coriolis = coriolis,
                                             closure = closure_ensemble,
                                             free_surface = ImplicitFreeSurface())

Δt = 1.0
simulation = Simulation(ensemble_model; Δt, stop_time=times[end])

priors = (
     κ_skew = ScaledLogitNormal(bounds = (300, 3000)),
     κ_symmetric = ScaledLogitNormal(bounds = (300, 3000))
 )

free_parameters = FreeParameters(priors)

using ParameterEstimocean.Observations: FieldTimeSeriesCollector

simulation_fields = fields(simulation.model)
collected_fields = NamedTuple(name => simulation_fields[name] for name in ParameterEstimocean.Observations.forward_map_names(observations))
time_series_collector = FieldTimeSeriesCollector(collected_fields, observation_times(observations), architecture=CPU())

calibration = InverseProblem(observations, simulation, free_parameters; time_series_collector)

eki = EnsembleKalmanInversion(calibration;
                              noise_covariance = 5e-3,
                              resampler = Resampler(acceptable_failure_fraction=1.0))

iterate!(eki; iterations = 10)

# Last, we visualize few metrics regarding how the EKI calibration went about.

y = observation_map(calibration)
#=
θ̅(iteration) = [eki.iteration_summaries[iteration].ensemble_mean...]
varθ(iteration) = eki.iteration_summaries[iteration].ensemble_var

weight_distances = [norm(θ̅(iter)) for iter in 1:eki.iteration]
output_distances = [norm(forward_map(calibration, θ̅(iter))[:, 1] - y) for iter in 1:eki.iteration]
ensemble_variances = [varθ(iter) for iter in 1:eki.iteration]

f = Figure()
lines(f[1, 1], 1:eki.iteration, weight_distances, color = :red, linewidth = 2,
      axis = (title = "Parameter norm",
              xlabel = "Iteration",
              ylabel="|θ̅ₙ|",
              yscale = log10))
lines(f[1, 2], 1:eki.iteration, output_distances, color = :blue, linewidth = 2,
      axis = (title = "Output distance",
              xlabel = "Iteration",
              ylabel="|G(θ̅ₙ) - y|",
              yscale = log10))
ax3 = Axis(f[2, 1:2], title = "Parameter convergence",
           xlabel = "Iteration",
           ylabel = "Ensemble variance",
           yscale = log10)

for (i, pname) in enumerate(free_parameters.names)
    ev = getindex.(ensemble_variances, i)
    lines!(ax3, 1:eki.iteration, ev / ev[1], label = String(pname), linewidth = 2)
end

axislegend(ax3, valign = :top, halign = :right)

save("summary_bca.png", f); nothing #hide
save("summary_bca.svg", f); nothing #hide

# ![](summary_channel.svg)

# And also we plot the the distributions of the various model ensembles for few EKI iterations to see
# if and how well they converge to the true diffusivity values.

=#

f = Figure()

axtop = Axis(f[1, 1])

axmain = Axis(f[2, 1],
              xlabel = "κ_skew [m² s⁻¹]",
              ylabel = "κ_symmetric [m² s⁻¹]")

axright = Axis(f[2, 2])
scatters = []
labels = String[]

for iter in [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
    ## Make parameter matrix
    parameters = eki.iteration_summaries[iter].parameters
    Nensemble = length(parameters)
    Nparameters = length(first(parameters))
    parameter_ensemble_matrix = [parameters[i][j] for i=1:Nensemble, j=1:Nparameters]

    label = iter == 0 ? "Initial ensemble" : "Iteration $iter"
    push!(labels, label)
    push!(scatters, scatter!(axmain, parameter_ensemble_matrix))
    density!(axtop, parameter_ensemble_matrix[:, 1])
    density!(axright, parameter_ensemble_matrix[:, 2], direction = :y)
end

colsize!(f.layout, 1, Fixed(300))
colsize!(f.layout, 2, Fixed(200))

rowsize!(f.layout, 1, Fixed(200))
rowsize!(f.layout, 2, Fixed(300))

Legend(f[1, 2], scatters, labels, valign = :bottom, halign = :left)

hidedecorations!(axtop, grid = false)
hidedecorations!(axright, grid = false)

xlims!(axright, 0, 0.025)
ylims!(axtop, 0, 0.025)

save("distributions_bca_2.png", f); nothing #hide
save("distributions_bca_2.svg", f); nothing #hide

# ![](distributions_channel.svg)


using Distributed
using Images, FileIO
include("mpii.annotation.reader.jl")
include("../models.jl")

# TODO: prepare an image
# then stack them to create whole dataset
# ısValid size check atılabilir

# struct DataItem
#   id::Int
#  path::String
# size::Array{Float64}
# joints::Array
# end

# If w has dimensions (W1,W2,...,Cx,Cy)
# and x has dimensions (X1,X2,...,Cx,N), the result y will have dimensions (Y1,Y2,...,Cy,N)
# where Cx is the number of input channels, Cy is the number of output channels,
# N is the number of instances, and Wi,Xi,Yi are spatial dimensions with Yi determined by:

function make_even(x)
    return isodd(x) ? x + 1 : x
end

function preprocess_dataset(dataset; use_global_scaling = false)
    #max_h = findmax(map(data -> data.size[2], dataset))[1]
    #max_w = findmax(map(data -> data.size[3], dataset))[1]
    deconv_3 = Deconv(3, 3, 1, 1; padding = 1, stride = 2)
    raw_preprocessed = map(
        element -> preprocess_single_image_features(
            element[2];
            use_global_scaling = use_global_scaling,
            counter = element[1],
            mock_deepercut_head = deconv_3,
        ),
        enumerate(dataset),
    )
    return raw_preprocessed
end

# TODO: update global scaling for custom iterator
function preprocess_single_image_features(
    img_data;
    use_global_scaling = false,
    counter,
    mock_deepercut_head,
)

    if counter % 1024 == 1
        println("preprocessing $(counter)")
    end

    img_path = img_data.path
    loaded_image = load(img_path)

    if use_global_scaling
        jitter= rand(scale_jitter_interval[1]:0.01:scale_jitter_interval[2])
        jittered_scale = jitter * global_scale
        scale_h = jittered_scale
        scale_w = jittered_scale
        scales = (scale_h, scale_w)
        resized_image = imresize(loaded_image, ratio = jittered_scale)
    else
        scaled_max_w = read_image_w
        scaled_max_h = read_image_h
        scale_h = convert(Float32, scaled_max_h / img_data.size[3])
        scale_w = convert(Float32, scaled_max_w / img_data.size[2])
        scales = (scale_h, scale_w)
        resized_image = imresize(loaded_image, scaled_max_h, scaled_max_w)
    end

    arranged_img_data = arrange_img_data(resized_image)
    scaled_img_size = size(arranged_img_data)[1:2]

    # for using 3 x 3 deconvolution at the head
    #sm_size = Int.(ceil.(scaled_img_size ./ (preprocess_stride * 2))) .* 2
    sm_size_raw = scaled_img_size ./ (preprocess_stride * 2)

    mock_intermediate = Knet.atype()(randn(Float32, sm_size_raw[1] |> ceil |> Int, sm_size_raw[2] |> ceil |> Int , 1, 1))
    mock_result = mock_deepercut_head(mock_intermediate)
    sm_size = size(mock_result)[1:2]

    temp_holder = img_data.joints[:, 2:3]
    temp_holder = reshape(temp_holder, (:, 2))
    temp = zeros(size(img_data.joints))
    for i = 1:size(temp_holder)[1]
        j1 = temp_holder[i, 1] * scales[1]
        j2 = temp_holder[i, 2] * scales[2]
        temp[i, 2] = j1
        temp[i, 3] = j2
    end

    scaled_joints = temp[:, 2:3]

    joint_ids = img_data.joints[:, 1]

    (scmap, scmap_weights, locref_map, locref_mask) =
        compute_targets_weights(joint_ids, scaled_joints, sm_size)

    # TODO: check if this is working or not
    img_data = DataItem(img_data, scaled_img_size[1], scaled_img_size[2])

    return (img_data, arranged_img_data, scmap, scmap_weights, locref_map, locref_mask)
end

function arrange_img_data(img)
    b1 = img
    # ad-hoc solution for Mac-OS image
    macfix = convert(Array{FixedPointNumbers.Normed{UInt8,8},3}, channelview(b1))
    c1 = permutedims(macfix, (3, 2, 1))
    w, h, c = size(c1)
    d1 = convert(Array{Float32}, c1)
    f1 = reshape(d1[:, :, 1:3], (w, h, 3, 1))
    # H - W - C - N
    g1 = permutedims(f1, [2, 1, 3, 4])
    g1
end

# TODO: Check pose_dataset.py for pairwise implementation - currently skipped
function compute_targets_weights(joint_ids, coords, size)
    stride = preprocess_stride
    dist_thresh = pos_dist_thresh
    num_joints = global_num_joints
    half_stride = stride / 2
    scmap = zeros(size..., num_joints)

    locref_shape = (size..., num_joints * 2)
    locref_mask = zeros(locref_shape)
    locref_map = zeros(locref_shape)

    dist_thresh_sq = abs2(dist_thresh)

    width = size[2]
    height = size[1]

    for (k, j_id) in enumerate(joint_ids)
        joint_pt = coords[k, :]
        j_x = joint_pt[1]
        j_y = joint_pt[2]

        # don't loop over entire heatmap, but just relevant locations

        j_x_sm = round((j_x - half_stride) / stride) |> Int
        j_y_sm = round((j_y - half_stride) / stride) |> Int
        min_x = round(max(j_x_sm - dist_thresh - 1, 1)) |> Int
        max_x = round(min(j_x_sm + dist_thresh + 1, width)) |> Int
        min_y = round(max(j_y_sm - dist_thresh - 1, 1)) |> Int
        max_y = round(min(j_y_sm + dist_thresh + 1, height)) |> Int

        for j = min_y:max_y
            pt_y = j * stride + half_stride
            for i = min_x:max_x
                pt_x = i * stride + half_stride
                dx = j_x - pt_x
                dy = j_y - pt_y
                dist = abs2(dx) + abs2(dy)

                if dist <= dist_thresh_sq
                    locref_scale = 1.0 / global_locref_stdev
                    current_normalized_dist = dist * abs2(locref_scale)

                    prev_normalized_dist =
                        locref_map[j, i, j_id*2+1]^2 + locref_map[j, i, j_id*2+2]^2

                    update_scores =
                        (scmap[j, i, j_id+1] == 0) ||
                        prev_normalized_dist > current_normalized_dist

                    if update_scores
                        set_locref(
                            locref_map,
                            locref_mask,
                            locref_scale,
                            i,
                            j,
                            j_id,
                            dx,
                            dy,
                        )
                    end
                    scmap[j, i, j_id+1] = 1
                end
            end
        end
    end
    scmap_weights = compute_scmap_weights(size, joint_ids)
    return (scmap, scmap_weights, locref_map, locref_mask)
end

function set_locref(locref_map, locref_mask, locref_scale, i, j, j_id, dx, dy)
    locref_mask[j, i, j_id*2+1] = 1
    locref_mask[j, i, j_id*2+2] = 1
    locref_map[j, i, j_id*2+1] = dx * locref_scale
    locref_map[j, i, j_id*2+2] = dy * locref_scale
end

function compute_scmap_weights(scmap_size, joint_ids)
    # TODO: This can be extra feature => Check pose_dataset.py 402 from pose-tensorflow
    #     if cfg.weigh_only_present_joints:
    #             weights = np.zeros(scmap_shape)
    #             for person_joint_id in joint_id:
    #                 for j_id in person_joint_id:
    #                     weights[:, :, j_id] = 1.0
    #         else:
    weights = ones(scmap_size)
    return weights
end


# <editor-fold desc= create other elements of model"

# XXX create variables and capacity constraints for trade variables
function createTradeVarCns!(partTrd::OthPart,anyM::anyModel)
	for type in (:Buy, :Sell)
		trdPrc_sym = Symbol(:trd,type,:Prc)
		trd_sym = Symbol(:trd,type)
		if trdPrc_sym in keys(partTrd.par) && :C in namesSym(partTrd.par[trdPrc_sym].data)

			# <editor-fold desc="create trade variables"
			c_arr = unique(partTrd.par[trdPrc_sym].data[!,:C])

			# create dataframe with all potential entries for trade/sell variable
			var_df = createPotDisp(c_arr,anyM)

			# match all potential variables with defined prices
			var_df = matchSetParameter(var_df,partTrd.par[trdPrc_sym],anyM.sets)[!,Not(:val)]

			var_df = createVar(var_df,string(:trd,type),getUpBound(var_df,anyM.options.bound.disp / anyM.options.scaFac.dispTrd,anyM.supTs,anyM.sets[:Ts]),anyM.optModel,anyM.lock,anyM.sets, scaFac = anyM.options.scaFac.dispTrd)
			partTrd.var[trd_sym] = orderDf(var_df)
			produceMessage(anyM.options,anyM.report, 3," - Created variables for $(type == :Buy ? "buying" : "selling") carriers")
			# </editor-fold>

			# <editor-fold desc="create capacity constraint on variable"
			trdCap_sym = Symbol(trd_sym,:Cap)
			if trdCap_sym in keys(partTrd.par)
				cns_df = matchSetParameter(var_df,partTrd.par[trdCap_sym],anyM.sets,newCol = :cap)
				sca_arr = getResize(cns_df,anyM.sets[:Ts],anyM.supTs)
				cns_df[!,:cap] = cns_df[!,:cap] .* sca_arr

				# prepare, scale and create constraints
				cns_df[!,:cnsExpr] = map(x -> x.var - x.cap, eachrow(cns_df))
				scaleCnsExpr!(cns_df,anyM.options.coefRng,anyM.options.checkRng)
				partTrd.cns[trdCap_sym] = createCns(cnsCont(cns_df,:smaller),anyM.optModel)

				produceMessage(anyM.options,anyM.report, 3," - Created capacity restrictions for $(type == :Buy ? "buying" : "selling") carriers")
			end

			# </editor-fold>
		end
		produceMessage(anyM.options,anyM.report, 2," - Created variables and constraints for $(type == :Buy ? "buying" : "selling") carriers")
	end
	produceMessage(anyM.options,anyM.report, 1," - Created variables and constraints for trade")
end

# XXX create all energy balances (and curtailment variables if required)
function createEnergyBal!(techSym_arr::Array{Symbol,1},anyM::anyModel)

	partBal = anyM.parts.bal
	c_arr = filter(x -> x != 0,getfield.(values(anyM.sets[:C].nodes),:idx))
	allDim_df = createPotDisp(c_arr,anyM)
	bal_tup = (:C,:Ts_dis)
	agg_arr = [:Ts_dis, :R_dis, :C]

	# <editor-fold desc="create potential curtailment and loss loss load variables

	for varType in (:crt,:lss)
		# get defined entries
		var_df = DataFrame()
		for par in intersect(keys(partBal.par),vcat(varType == :crt ? :costCrt : :costLss, Symbol.(varType,[:Up,:Low,:Fix])...))
			append!(var_df,matchSetParameter(allDim_df,partBal.par[par],anyM.sets)[!,Not(:val)])
		end

		# obtain upper bound for variables and create them
		if !isempty(var_df)
			partBal.var[varType] = orderDf(createVar(var_df,string(varType),getUpBound(var_df,anyM.options.bound.disp / anyM.options.scaFac.dispTrd,anyM.supTs,anyM.sets[:Ts]),anyM.optModel,anyM.lock,anyM.sets, scaFac = anyM.options.scaFac.dispTrd))
		end
	end
	# </editor-fold>

	# <editor-fold desc="create actual balance"

	# finds all carriers that require an energy balance (not required for carriers that can only be shifted (temporal or spatial), e.g. that only have storage or exchange defined for them)
	relC_arr = Array{Int,1}()

	# if demand does not specify a carrier, it is assumed all carriers are relevant
	if !isempty(anyM.parts.bal.par[:dem].data)
		if :C in namesSym(anyM.parts.bal.par[:dem].data)
			append!(relC_arr,unique(anyM.parts.bal.par[:dem].data[!,:C]))
		else
			append!(relC_arr,c_arr)
		end
	end

	if :crt in keys(anyM.parts.bal.var) append!(relC_arr,unique(anyM.parts.bal.var[:crt][!,:C])) end
	if :lss in keys(anyM.parts.bal.var) append!(relC_arr,unique(anyM.parts.bal.var[:lss][!,:C])) end
	if :trdSell in keys(anyM.parts.trd.var) append!(relC_arr,unique(anyM.parts.trd.var[:trdSell][!,:C])) end
	if :trdBuy in keys(anyM.parts.trd.var) append!(relC_arr,unique(anyM.parts.trd.var[:trdBuy][!,:C])) end

	# add carriers beings generated or used
	append!(relC_arr,union(union(map(x -> anyM.parts.tech[x].carrier |> (y -> map(z -> getfield(y,z),intersect(keys(y),(:gen,:use)))),techSym_arr)...)...))
	relC_arr = unique(relC_arr)

	# create object to write constraint data too
	cns_arr = Array{Pair{Symbol,cnsCont}}(undef,length(relC_arr))
	itrC_arr = collect(enumerate(relC_arr))

	@threads for (idx,c) in itrC_arr

		subC_arr = unique([c,getDescendants(c,anyM.sets[:C],true)...])
		cRes_tup = anyM.cInfo[c] |> (x -> (Ts_dis = x.tsDis, R_dis = x.rDis, C = anyM.sets[:C].nodes[c].lvl))

		# XXX add demand and size it
		cns_df = matchSetParameter(filter(x -> x.C == c,allDim_df),partBal.par[:dem],anyM.sets)
		cns_df[!,:dem] = cns_df[!,:val] .* getResize(cns_df,anyM.sets[:Ts],anyM.supTs)
		select!(cns_df,Not(:val))

		# XXX get relevant variables
		src_df = cns_df[!,Not([:Ts_disSup,:dem])]

		# add tech variables
		cns_df[!,:techVar] = getTechEnerBal(c,subC_arr,src_df,anyM.parts.tech,anyM.cInfo,anyM.sets)

		# add curtailment variables
		for varType in (:crt,:lss)
			if varType in keys(partBal.var)
				cns_df[!,Symbol(varType,:Var)] = filterCarrier(partBal.var[varType],subC_arr) |> (x -> aggUniVar(x,src_df,agg_arr, cRes_tup,anyM.sets))
			else
				cns_df[!,Symbol(varType,:Var)] .= AffExpr()
			end
		end

		# add trade variables
		if !isempty(anyM.parts.trd.var)
			cns_df[!,:trdVar] = sum([filterCarrier(anyM.parts.trd.var[trd],subC_arr) |> (x -> aggUniVar(x,src_df,agg_arr,cRes_tup,anyM.sets) |> (y -> trd != :trdSell ? y : -1.0 * y)) for trd in keys(anyM.parts.trd.var)])
		else
			cns_df[!,:trdVar] .= AffExpr()
		end

		# add exchange variables
		if !isempty(anyM.parts.exc.var)
			excVarTo_df = filterCarrier(anyM.parts.exc.var[:exc],subC_arr)
			excVarFrom_df = convertExcCol(copy(excVarTo_df))

			# get loss values and apply them to variables
			excVarFrom_df = getExcLosses(excVarFrom_df,anyM.parts.exc.par,anyM.sets)
			excVarFrom_df[!,:var] = excVarFrom_df[!,:var] .* (1.0 .- excVarFrom_df[!,:loss])
			select!(excVarFrom_df,Not(:loss))

			balTo_tup, balFrom_tup = [tuple(replace(collect(bal_tup),:R_dis => x)...) for x in [:R_to, :R_from]]

			excFrom_arr = aggUniVar(convertExcCol(excVarFrom_df),rename(src_df,:R_dis => :R_to),[:Ts_dis,:R_to,:C],(Ts_dis = cRes_tup[1], R_to = cRes_tup[2], C = cRes_tup[3]),anyM.sets)
			excTo_arr  = aggUniVar(excVarTo_df,rename(src_df,:R_dis => :R_from),[:Ts_dis,:R_from,:C],(Ts_dis = cRes_tup[1], R_from = cRes_tup[2], C = cRes_tup[3]),anyM.sets)

			cns_df[!,:excVar] =  excFrom_arr .- excTo_arr
		else
			cns_df[!,:excVar] .= AffExpr()
		end

		# prepare, scale and save constraints to dictionary
		c_str = Symbol(anyM.sets[:C].nodes[c].val)
		cns_df[!,:cnsExpr] = map(x -> x.techVar + x.excVar + x.trdVar + x.lssVar - x.dem - x.crtVar, eachrow(cns_df))
		cns_df = orderDf(cns_df[!,[intCol(cns_df)...,:cnsExpr]])
		filter!(x -> x.cnsExpr != AffExpr(),cns_df)
		scaleCnsExpr!(cns_df,anyM.options.coefRng,anyM.options.checkRng)
		cns_arr[idx] = Symbol(c_str) => cnsCont(cns_df,anyM.cInfo[c].eq ? :equal : :greater)

		produceMessage(anyM.options,anyM.report, 2," - Prepared energy balance for $(c_str)")
	end

	# loops over stored constraints outside of threaded loop to create actual jump constraints
	for cns in cns_arr
		partBal.cns[cns[1]] = createCns(cns[2],anyM.optModel)
	end

	produceMessage(anyM.options,anyM.report, 1," - Created energy balances for all carriers")
	# </editor-fold>
end

# XXX aggregate all technology variables for energy balance
function getTechEnerBal(cBal_int::Int,subC_arr::Array{Int,1},src_df::DataFrame,tech_dic::Dict{Symbol,TechPart},
																				cInfo_dic::Dict{Int,NamedTuple{(:tsDis,:tsExp,:rDis,:rExp,:eq),Tuple{Int,Int,Int,Int,Bool}}},sets_dic::Dict{Symbol,Tree})
	techVar_arr = Array{Array{AffExpr,1}}(undef,length(subC_arr))

	# get temporal and spatial resolution for carrier being balanced
	cBalRes_tup = cInfo_dic[cBal_int] |> (x -> (x.tsDis, x.rDis))

	# loops over all carriers relevant for respective energy balance
	for (idx,c) in enumerate(subC_arr)

		# gets technologies relevant for respective filterCarrier
		relTech_arr = getRelTech(c,tech_dic,sets_dic[:C])

		# leaves loop for carrier, if no relevant technologies could be obtained
		if isempty(relTech_arr)
			techVar_arr[idx]  = fill(AffExpr(),size(src_df,1))
			continue
		end

		# prepare loop over tech for c by creating empty dataframe and get temporal and spatial resolution for carrier being balanced
		allVar_df = DataFrame(Ts_dis = Int[], R_dis = Int[], var = AffExpr[])
		cRes_tup = cInfo_dic[c] |> (x -> (x.tsDis, x.rDis))

		for x in relTech_arr
			# gets resolution and adjusts add_df in case of an agggregated technology
			add_df = select(filter(r -> r.C == c,tech_dic[x[1]].var[x[2]]),[:Ts_dis,:R_dis,:var])
			if isempty(add_df) continue end
			tRes_tup = tech_dic[x[1]].disAgg ? (cRes_tup[1], tech_dic[x[1]].balLvl.exp[2]) : cRes_tup
			checkTechReso!(tRes_tup,cBalRes_tup,add_df,sets_dic)

			# adds sign to variables and adds them to overall dataframe
			add_df[!,:var] = add_df[!,:var] .* (x[2] in (:use,:stExtIn) ? -1.0 : 1.0)
			append!(allVar_df, add_df)
		end

		# returns empty expression if no variales could be obtained
		if isempty(allVar_df)
			techVar_arr[idx]  = fill(AffExpr(),size(src_df,1))
		else
			grpVar_df = combine(groupby(allVar_df, [:Ts_dis, :R_dis]), :var => (x -> sum(x)) => :var)
			techVar_arr[idx] = joinMissing(src_df,grpVar_df, [:Ts_dis, :R_dis], :left, Dict(:var => AffExpr()))[!,:var]
		end
	end

	return map(x -> sum(x),eachrow(hcat(techVar_arr...)))
end

# XXX create constarints that enforce any type of limit (Up/Low/Fix) on any type of variable
function createLimitCns!(partLim::OthPart,anyM::anyModel)

	parLim_arr = String.(collectKeys(keys(partLim.par)))
	techLim_arr = filter(x ->  any(map(y -> occursin(y,x),["Up","Low","Fix"])),parLim_arr)
	limVar_arr = map(x -> map(k -> Symbol(k[1]) => Symbol(k[2][1]), filter(z -> length(z[2]) == 2,map(y -> y => split(x,y),["Up","Low","Fix"])))[1], techLim_arr)
	varToPar_dic = Dict(y => getindex.(filter(z -> z[2] == y,limVar_arr),1) for y in unique(getindex.(limVar_arr,2)))

	# loop over all variables that are subject to any type of limit (except emissions)
	allKeys_arr = collect(keys(varToPar_dic))
	cns_dic = Dict{Symbol,cnsCont}()
	signLim_dic= Dict(:Up => :smaller, :Low => :greater, :Fix => :equal)

	@threads for va in allKeys_arr

		varToPart_dic = Dict(:exc => :exc, :crt => :bal,:trdSell => :trd, :trdBuy => :trd)

		# obtain all variables relevant for limits
		allVar_df = getAllVariables(va,anyM)

		# check if acutally any variables were obtained
		if isempty(allVar_df)
			lock(anyM.lock)
			push!(anyM.report,(2,"limit",string(va),"limits for variable provided, but none of these variables are actually created"))
			unlock(anyM.lock)
			continue
		end

		allLimit_df = DataFrame(var = AffExpr[])
		# XXX loop over respective type of limits to obtain data
		for lim in varToPar_dic[va]
			par_obj = copy(partLim.par[Symbol(va,lim)])
			if va in (:capaExc,:oprCapaExc) && :R_a in namesSym(par_obj.data) && :R_b in namesSym(par_obj.data)
				par_obj.data = vcat(par_obj.data,rename(par_obj.data,:R_a => :R_b,:R_b => :R_a))
			end
			agg_tup = tuple(intCol(par_obj.data)...)

			# aggregate search variables according to dimensions in limit parameter
			if isempty(agg_tup)
				grpVar_df = allVar_df
			else
				grpVar_df = combine(groupby(convertExcCol(allVar_df),collect(agg_tup)), :var => (x -> sum(x)) => :var)
			end

			# try to aggregate variables to limits directly provided via inputs
			limit_df = copy(par_obj.data)
			if size(limit_df,2) != 1
				limit_df[!,:var] = aggDivVar(grpVar_df, limit_df[!,Not(:val)], agg_tup, anyM.sets, aggFilt = agg_tup)
			else
				limit_df[!,:var] .= sum(grpVar_df[!,:var])
			end

			# gets provided limit parameters, that no variables could assigned to so far and tests if via inheritance any could be assigned
			mtcPar_arr, noMtcPar_arr  = findall(map(x -> x != AffExpr(),limit_df[!,:var])) |>  (x -> [x, setdiff(1:size(par_obj.data,1),x)])
			# removes entries with no parameter assigned from limits
			limit_df = limit_df[mtcPar_arr,:]

			if !isempty(noMtcPar_arr)
				# tries to inherit values to existing variables only for parameters without variables aggregated so far
				aggPar_obj = copy(par_obj,par_obj.data[noMtcPar_arr,:])
				aggPar_obj.data = matchSetParameter(grpVar_df[!,Not(:var)],aggPar_obj,anyM.sets, useNew = false)
				# again performs aggregation for inherited parameter data and merges if original limits
				aggLimit_df = copy(aggPar_obj.data)
				if !isempty(aggLimit_df)
					aggLimit_df[!,:var]  = aggDivVar(grpVar_df, aggLimit_df, agg_tup, anyM.sets, aggFilt = agg_tup)
					limit_df = vcat(limit_df,aggLimit_df)
				end
			end

			# merge limit constraint to other limits for the same variables
			limit_df = convertExcCol(rename(limit_df,:val => lim))
			join_arr = [intersect(intCol(allLimit_df),intCol(limit_df))...,:var]
			miss_arr = [intCol(allLimit_df),intCol(limit_df)] |> (y -> union(setdiff(y[1],y[2]), setdiff(y[2],y[1])))
			allLimit_df = joinMissing(allLimit_df, limit_df, join_arr, :outer, merge(Dict(z => 0 for z in miss_arr),Dict(:Up => nothing, :Low => nothing, :Fix => nothing)))
		end

		# XXX check for contradicting values
		colSet_dic = Dict(x => Symbol(split(string(x),"_")[1]) for x in intCol(allLimit_df))
		limitCol_arr = intersect(namesSym(allLimit_df),(:Fix,:Up,:Low))
		entr_int = size(allLimit_df,1)
		if :Low in limitCol_arr || :Up in limitCol_arr
			# ! errors

			# upper and lower limit contradicting each other
			if :Low in limitCol_arr && :Up in limitCol_arr
				for x in findall(replace(allLimit_df[!,:Low],nothing => 0.0) .> replace(allLimit_df[!,:Up],nothing => Inf))
					dim_str = join(map(y -> allLimit_df[x,y] == 0 ?  "" : string(y,": ",join(getUniName(allLimit_df[x,y], anyM.sets[colSet_dic[y]])," < ")),intCol(allLimit_df)),"; ")
					lock(anyM.lock)
					push!(anyM.report,(3,"limit",string(va),"contradicting values for upper and lower limit detected for: " * dim_str))
					unlock(anyM.lock)
				end
			end

			# fix and upper limit contradicting each other
			if :Fix in limitCol_arr && :Up in limitCol_arr
				for x in findall(replace(allLimit_df[!,:Fix],nothing => 0.0) .> (replace(allLimit_df[!,:Up],nothing => Inf) .+ 0.0001))
					dim_str = join(map(y -> allLimit_df[x,y] == 0 ?  "" : string(y,": ",join(getUniName(allLimit_df[x,y], anyM.sets[colSet_dic[y]])," < ")),intCol(allLimit_df)),"; ")
					lock(anyM.lock)
					push!(anyM.report,(3,"limit",string(va),"fixed limit exceeds upper limit for: " * dim_str))
					unlock(anyM.lock)
				end
			end

			# fix and lower limit contradicting each other
			if :Fix in limitCol_arr && :Low in limitCol_arr
				for x in findall(replace(allLimit_df[!,:Fix],nothing => Inf) .< replace(allLimit_df[!,:Low],nothing => 0.0) .- 0.0001)
					dim_str = join(map(y -> allLimit_df[x,y] == 0 ?  "" : string(y,": ",join(getUniName(allLimit_df[x,y], anyM.sets[colSet_dic[y]])," < ")),intCol(allLimit_df)),"; ")
					lock(anyM.lock)
					push!(anyM.report,(3,"limit",string(va),"fixed limit is smaller than lower limit for: " * dim_str))
					unlock(anyM.lock)
				end
			end

			# residual values already violate limits
			resiVal_arr = getfield.(allLimit_df[!,:var],:constant)
			if :Up in limitCol_arr
				for x in findall(resiVal_arr .>  replace(allLimit_df[!,:Up],nothing => Inf))
					dim_str = join(map(y -> allLimit_df[x,y] == 0 ?  "" : string(y,": ",join(getUniName(allLimit_df[x,y], anyM.sets[colSet_dic[y]])," < ")),intCol(allLimit_df)),"; ")
					lock(anyM.lock)
					push!(anyM.report,(3,"limit",string(va),"residual values already exceed the upper limit for: " * dim_str))
					unlock(anyM.lock)
				end
			end

			# ! warnings
			# upper or lower limit of zero
			if !isempty(limitCol_arr |> (y -> filter(x -> collect(x[y]) |> (z -> any(isnothing.(z)) ? false : any(z .== 0)),allLimit_df))) && va != :emission
				lock(anyM.lock)
				push!(anyM.report,(2,"limit",string(va),"upper or lower limit of zero detected, please consider to use fix or omit limit instead"))
				unlock(anyM.lock)
				entr_int = size(allLimit_df,1)
			end

			# value is fixed, but still a upper a lower limit is provided
			if :Fix in limitCol_arr
				if !isempty(filter(x -> x != :Fix, limitCol_arr) |> (z -> filter(x -> all([!isnothing(x.Fix),any(.!isnothing.(collect(x[z])))]) ,eachrow(allLimit_df))))
					lock(anyM.lock)
					push!(anyM.report,(2,"limit",string(va),"upper and/or lower limit detected, although variable is already fixed"))
					unlock(anyM.lock)
				end
			end
		end

		# if installed capacities differ depending on the direction, because residual values were defined and at the same time fixed limits on the installed capacity were provided
		# an error will occur, because a value cannot be fixed but and the same time differ by direction, this is detected here
		if :Fix in limitCol_arr && va == :capaExc
			# filters fixed exchange capacities and extracts residual values
			fix_df = select(filter(x -> x.Fix != nothing, allLimit_df) ,intCol(allLimit_df,:var))
			fix_df[!,:resi] .= getfield.(fix_df[!,:var],:constant)
			# joins together capacities in both directions
			joinA_df = rename(select(fix_df,Not([:var])),:resi => :resiA)
			joinB_df = rename(joinA_df,:resiA => :resiB)
			comp_df = innerjoin(joinA_df,joinB_df, on = intCol(joinA_df) .=> replace(intCol(joinB_df),:R_from => :R_to,:R_to => :R_from))
			# finds cases that lead to contradiction and reports on them
			contraExc_df = filter(x -> x.resiA != x.resiB && x.R_from > x.R_to,comp_df)
			for x in eachrow(contraExc_df)
				dim_str = join(map(y -> x[y] == 0 ?  "" : string(y,": ",join(getUniName(x[y], anyM.sets[colSet_dic[y]])," < ")),intCol(contraExc_df)),"; ")
				lock(anyM.lock)
				push!(anyM.report,(3,"limit",string(va),"for the exchange capacity '" * dim_str * "' residual capacites differ by direction but at the same time the installed capacity in both directions is fixed to the same value by capaExcFix, this is a contradiction and  would lead to an infeasible model"))
				unlock(anyM.lock)
			end
		end

		# XXX check for suspicious entries for capacity where limits are provided for the sum of capacity over several years
		if occursin("capa",string(va))
			if !(:Ts_disSup in namesSym(allLimit_df))
				lock(anyM.lock)
				push!(anyM.report,(2,"limit","capacity","capacity limits were provided without specificing the superordinate dispatch timestep, this means the sum of capacity over all years was limited instead of enforcing the same limit for each year
																												(see https://leonardgoeke.github.io/AnyMOD.jl/stable/parameter_list/#Limits-on-quantities-dispatched)"))
				unlock(anyM.lock)
			elseif 0 in unique(allLimit_df[!,:Ts_disSup])
				relEntr_df = filter(x -> x.Ts_disSup == 0, allLimit_df)
				if :Te in namesSym(relEntr_df)
					allTe_arr = unique(relEntr_df[!,:Te])
					for tInt in allTe_arr
						push!(anyM.report,(2,"limit","capacity","capacity limits were provided for $(string(techSym(tInt,anyM.sets[:Te]))) without specificing the superordinate dispatch timestep, this means the sum of capacity over all years was limited instead of enforcing the same limit for each year
																														(see https://leonardgoeke.github.io/AnyMOD.jl/stable/parameter_list/#Limits-on-quantities-dispatched)"))
					end
				else
					lock(anyM.lock)
					push!(anyM.report,(2,"limit","capacity","capacity limits were provided without specificing the superordinate dispatch timestep, this means the sum of capacity over all years was limited instead of enforcing the same limit for each year
																													(see https://leonardgoeke.github.io/AnyMOD.jl/stable/parameter_list/#Limits-on-quantities-dispatched)"))
					unlock(anyM.lock)
				end
			end
		end

		# XXX write constraint containers
		for lim in limitCol_arr
			# filter respective limits (low, fix or up) out of the entire dataframe
			relLim_df = filter(x -> !isnothing(x[lim]),allLimit_df[!,Not(filter(x -> x != lim,limitCol_arr))])
			relLim_df = filter(x -> x.var != AffExpr() || x.Fix != 0.0, relLim_df)
            if isempty(relLim_df) continue end
			rename!(relLim_df,lim => :Lim)

			# prepare, scale and save constraints to dictionary
			relLim_df[!,:cnsExpr] = map(x -> x.var - x.Lim, eachrow(relLim_df))
			relLim_df = orderDf(relLim_df[!,[intCol(relLim_df)...,:cnsExpr]])
			scaleCnsExpr!(relLim_df,anyM.options.coefRng,anyM.options.checkRng)
			cns_dic[Symbol(va,lim)] = cnsCont(relLim_df,signLim_dic[lim])

			produceMessage(anyM.options,anyM.report, 3," - Created constraints for $(lim == :Up ? "upper" : (lim == :Low ? "lower" : "fixed")) limit of variable $va")
		end
		typeLim_sym = va in (:emission,) ? "term" : "variable"
		produceMessage(anyM.options,anyM.report, 2," - Prepared constraints to limit $typeLim_sym $va")
	end

	# loops over stored constraints outside of threaded loop to create actual jump constraints
	for cnsSym in keys(cns_dic)
		partLim.cns[cnsSym] = createCns(cns_dic[cnsSym],anyM.optModel)
	end

	produceMessage(anyM.options,anyM.report, 1," - Created all limiting constraints")
end

# </editor-fold>

# <editor-fold desc= utility functions"

# XXX connect capacity and expansion variables
function createCapaCns!(part::TechPart,prepTech_dic::Dict{Symbol,NamedTuple},cns_dic::Dict{Symbol,cnsCont})
    for capaVar in filter(x -> occursin("capa",string(x)),keys(prepTech_dic))
        index_arr = intCol(part.var[capaVar])
        join_arr = part.type != :mature ? index_arr : filter(x -> x != :Ts_expSup,collect(index_arr))

        # joins corresponding capacity and expansion variables together
		expVar_sym = Symbol(replace(string(capaVar),"capa" => "exp"))
		if !(expVar_sym in keys(part.var)) continue end
        expVar_df = flatten(part.var[expVar_sym],:Ts_disSup)
        cns_df = rename(innerjoin(part.var[capaVar],combine(groupby(expVar_df,join_arr), :var => (x -> sum(x)) => :exp); on = join_arr),:var => :capa)

        # creates final constraint object
		cns_df[!,:cnsExpr] = map(x -> x.capa - x.capa.constant - x.exp,eachrow(cns_df))
		cns_dic[Symbol(capaVar)] = cnsCont(select(cns_df,Not([:capa,:exp])),:equal)
    end
end

# XXX adds column with JuMP variable to dataframe
function createVar(setData_df::DataFrame,name_str::String,upBd_fl::Union{Float64,Array{Float64,1}},optModel::Model,lock_::ReentrantLock,sets::Dict{Symbol,Tree}; scaFac::Float64 = 1.0, lowBd::Float64 = 0.0)
	# adds an upper bound to all variables if provided within the options
	#if isempty(setData_df) return DataFrame(var = AffExpr[]) end
	arr_boo = typeof(upBd_fl) <: Array
	if arr_boo
		info = VariableInfo.(!isnan(lowBd), lowBd, .!isnan.(upBd_fl), upBd_fl, false, NaN, false, NaN, false, false)
		var_obj = JuMP.build_variable.(error, info)
	else
		info = VariableInfo(!isnan(lowBd), lowBd, !isnan(upBd_fl), upBd_fl, false, NaN, false, NaN, false, false)
		var_obj = JuMP.build_variable(error, info)
	end

	# writes full name for each variable
	setData_df = orderDf(setData_df)
	dim_arr = map(x -> Symbol(split(String(x),"_")[1]), filter(r -> r != :id,intCol(setData_df)))
	dim_int = length(dim_arr)
	setData_df[!,:name] = string.(name_str,"[",map(x -> join(map(y -> sets[dim_arr[y]].nodes[x[y]].val,1:dim_int),", "),eachrow(setData_df)),"]")

	lock(lock_)
	if arr_boo
		setData_df[!,:var] = [AffExpr(0,JuMP.add_variable(optModel, nameItr[1], nameItr[2]) => scaFac) for nameItr in zip(var_obj,setData_df[!,:name])]
	else
		setData_df[!,:var] = [AffExpr(0,JuMP.add_variable(optModel, var_obj, nameItr) => scaFac) for nameItr in setData_df[!,:name]]
	end
	unlock(lock_)

	return setData_df[!,Not(:name)]
end

# XXX scales expressions in the dataframe to be within the range defined within options
function scaleCnsExpr!(cnsExpr_df::DataFrame,coefRng::NamedTuple{(:mat,:rhs),Tuple{Tuple{Float64,Float64},Tuple{Float64,Float64}}},checkRng_fl::Float64)

	if isempty(cnsExpr_df) return end

	# filters variables with factor zero, because they will lead to scaling errors
	foreach(z -> foreach(y ->  delete!(z.terms,y), filter(x -> z.terms[x] == 0.0 ,collect(keys(z.terms)))), cnsExpr_df[!,:cnsExpr])

	if !all(isnan.(coefRng.rhs))
		# scale expression defining constraint so rhs coefficients are within desired range
		rhs_arr = abs.(getfield.(cnsExpr_df[!,:cnsExpr],:constant))
		findall(rhs_arr .!= 0.0) |> (x -> cnsExpr_df[x,:cnsExpr] = scaleRng(cnsExpr_df[x,:cnsExpr],rhs_arr[x],coefRng.rhs, true))
	end

	if !all(isnan.(coefRng.mat))
		# scale expression defining constraint so matrix coefficients are within desired range
		matRng_arr = map(x -> abs.(values(x.terms)) |> (y -> isempty(y) ? (0.0,0.0) : (minimum(y),maximum(y))), cnsExpr_df[!,:cnsExpr])
		findall(map(x -> x != (0.0,0.0),matRng_arr)) |> (x -> cnsExpr_df[x,:cnsExpr] = scaleRng(cnsExpr_df[x,:cnsExpr],matRng_arr[x],coefRng.mat,false))
	end

	if !isnan(checkRng_fl)
		checkExprRng(cnsExpr_df[:,:cnsExpr],checkRng_fl)
	end
end

# XXX used to perform scaling of expression array based on range of coefficients provided
function scaleRng(expr_arr::Array{AffExpr,1},rng_arr::Array,rng_tup::Tuple{Float64,Float64}, rhs_boo::Bool)
	scaRel_arr = rhs_boo ? union(findall(rng_arr .< rng_tup[1]), findall(rng_arr .> rng_tup[2])) : union(findall(getindex.(rng_arr,1) .< rng_tup[1]), findall(getindex.(rng_arr,2) .> rng_tup[2]))
	if !isempty(scaRel_arr)
		expr_arr[scaRel_arr] = map(x -> x[1] < rng_tup[1] ? rng_tup[1]/x[1] : rng_tup[2]/x[rhs_boo ? 1 : 2], rng_arr[scaRel_arr]) .* expr_arr[scaRel_arr]
	end
	return expr_arr
end

# XXX check range of coefficients in expressions within input array
function checkExprRng(expr_arr::Array{AffExpr,1},rngThres_fl::Float64)
	# obtains range of coefficients for matrix and rhs
	matRng_arr = map(x -> abs.(values(x.terms)) |> (y -> isempty(y) ? (0.0,0.0) : (minimum(y),maximum(y))), expr_arr)
	rhs_arr = abs.(getfield.(expr_arr,:constant))
	both_arr = max.(getindex.(matRng_arr,2),replace(rhs_arr,0.0 => -Inf)) ./ min.(getindex.(matRng_arr,1),replace(rhs_arr,0.0 => Inf))

	# filters rows where reange of coefficients is above threshold
	aboveThres_arr = findall(both_arr .> rngThres_fl)

	for expr in expr_arr[aboveThres_arr]
		println(expr)
	end
end

# XXX creates an actual jump constraint based on the constraint container provided
function createCns(cnsCont_obj::cnsCont,optModel::Model)
	cns_df = cnsCont_obj.data
	if cnsCont_obj.sign == :equal
		cns_df[!,:cns] = map(x -> @constraint(optModel, x.cnsExpr == 0),eachrow(cns_df))
	elseif cnsCont_obj.sign == :greater
		cns_df[!,:cns] = map(x -> @constraint(optModel, x.cnsExpr >= 0),eachrow(cns_df))
	elseif cnsCont_obj.sign == :smaller
		cns_df[!,:cns] = map(x -> @constraint(optModel, x.cnsExpr <= 0),eachrow(cns_df))
	end
	return select!(cns_df,Not(:cnsExpr))
end

# XXX adjusts resolution of var_df according to information in first two tuples
function checkTechReso!(tRes_tup::Tuple{Int,Int},cBalRes_tup::Tuple{Int,Int},var_df::DataFrame,sets_dic::Dict{Symbol,Tree})
	# if dispatch regions for technology were disaggregated, replace the disaggregated with the ones relevant for the carrier
	for (idx,dim) in enumerate([:Ts_dis,:R_dis])
		if cBalRes_tup[idx] != tRes_tup[idx]
			set_sym = Symbol(split(string(dim),"_")[1])
			dim_dic = Dict(x => getAncestors(x,sets_dic[set_sym],:int,cBalRes_tup[idx])[end] for x in unique(var_df[!,dim]))
			var_df[!,dim] = map(x -> dim_dic[x],var_df[!,dim])
		end
	end
end

# </editor-fold>


using ZMQ
include("CherokeeQCSynthesizer.jl")

context = Context()
socket = Socket(context, REP)
ZMQ.bind(socket, "tcp://*:5555")

println("[+] Cherokee ΦMQ Server listening on port 5555")
while true
  print("[+] Waiting for ΦMQ Client...")
  message = split(bytestring(ZMQ.recv(socket)), "&")
  println("Connected.")

  stages = message[1]
  nqubits = int(message[2])
  nstages = int(message[3])
  inputs = message[4]

  print("[+] Simulating...")
  unitaryMatrix = simulate(strip(stages), nqubits, nstages)
  println("DONE")
  print("[+] Mapping inputs...")
  mapped_inputs = map_inputs(inputs)
  println("DONE")
  print("[+] Computing output...")
  if unitaryMatrix != "NA"
     output = unitaryMatrix * mapped_inputs
     println("DONE")
     unitaryMatrix = string(unitaryMatrix)
     unitaryMatrix = contains(unitaryMatrix, "Complex{Float64}") ? SubString(unitaryMatrix, 17) : unitaryMatrix
     output = string(output)
     output = contains(output, "Complex{Float64}") ? SubString(output, 17) : output
     result = string(unitaryMatrix , " | " , output)
  else
    result = "NA"
  end

  print("[+] Sending results...")
  ZMQ.send(socket, result)
  println("...DONE\n")
end

ZMQ.close(socket)
ZMQ.close(context)


module Plots

using Winston
using Params

export initplots, plot_ee, displayplots, plot_buf

const colors = ["blue"; "cyan"; "green"; "red"; "magenta"; "yellow"; "black"]

type PlotFig
    l :: Array{Any, 1}
    p :: FramedPlot
    PlotFig() = new({})
end

p_ee = PlotFig()
p_ee_conv = PlotFig()
p_th = PlotFig()
p_buf = PlotFig()

function initplots()
    p_ee.p = FramedPlot(
            title="EE vs Time (Cumulative)",
            xlabel="Time (time slots)",
            ylabel="EE (\\Sigma b/\\Sigma E) (b/J)")
    p_ee_conv.p = FramedPlot(
            title="EE vs Time (501 Pt Average)",
            xlabel="Time (time slots)",
            ylabel="EE (b/E) (b/J)")
    p_th.p = FramedPlot(
            title="Throughput vs Time (501 Pt Average)",
            xlabel="Time (time slots)",
            ylabel="Throughput (b)")
    p_buf.p = FramedPlot(
            title="Buffer Occupancy (501 Pt Average)",
            xlabel="Time (time slots)",
            ylabel="Buffer Occupancy (b)")
    p_ee.l = {}
    p_ee_conv.l = {}
    p_th.l = {}
    p_buf.l = {}
end

function plot_ee(energies, bits, agent, i)
    # start from 100 in total EE plot to see the differences more easily
    ee = Curve(100:t_total, vec(cumsum(sum(bits, 1), 2) ./ cumsum(sum(energies, 1), 2))[100:end], color=colors[i])
    setattr(ee, "label", agent)
    add(p_ee.p, ee)
    push!(p_ee.l, ee)
    ee_conv = Curve(500:t_total, conv(vec(sum(bits, 1) ./ sum(energies, 1)), fill(1./501, 501))[500:end-500], color=colors[i])
    setattr(ee_conv, "label", agent)
    add(p_ee_conv.p, ee_conv)
    push!(p_ee_conv.l, ee_conv)
    th = Curve(500:t_total, conv(vec(sum(bits, 1)), fill(1./501, 501))[500:end-500], color=colors[i])
    setattr(th, "label", agent)
    add(p_th.p, th)
    push!(p_th.l, th)
    nothing
end

function plot_buf(buf_levels, agent, i)
    # start from 100 in total EE plot to see the differences more easily
    buf = Curve(500:t_total, conv(vec(mean(buf_levels, 1)) .* Params.pkt_size, fill(1./501, 501))[500:end-500], color=colors[i])
    setattr(buf, "label", agent)
    add(p_buf.p, buf)
    push!(p_buf.l, buf)
    nothing
end

function displayplots()
    for p in [p_ee, p_ee_conv, p_th, p_buf]
        f=figure()
        add(p.p, Legend(.7, .7, p.l))
        display(p.p)
        figure(f)
        savefig(string(getattr(p.p, "title"), ".png"))
        savefig(string(getattr(p.p, "title"), ".svg"))
    end
end

end # module


# This file contains the definition for the PowerPlants struct.

mutable struct PowerPlants
    unit
    node
    pmax
    pmin
    rup
    rdn
    timeup
    timedn
    mc
    varcost
    fuel
    efficiency
    emission

    function PowerPlants(pp_df::DataFrame)

        G = pp_df[!, 1]                                    # U = unit
        pp_dict = df_to_dict_with_id(pp_df)

        return new(G,
                   pp_dict[:node],
                   pp_dict[:pmax],
                   pp_dict[:pmin],
                   pp_dict[:rup],
                   pp_dict[:rdn],
                   pp_dict[:timeup],
                   pp_dict[:timedn],
                   pp_dict[:mc],
                   0,
                   0,
                   0,
                   0,
                   )
    end

    function PowerPlants(pp_df::DataFrame, fuelcost_df::DataFrame)

        G = pp_df[!, 1]                                    # U = unit
        pp_dict = df_to_dict_with_id(pp_df)

        # Marginal generation cost from fuel cost, efficiency, variable cost
        # and CO2 price

        fuelcost_dict = df_to_dict(fuelcost_df)
        mc = Dict{Int64, Array{Float64, 1}}()

        for p in keys(pp_dict[:fuel])
            f = Symbol(pp_dict[:fuel][p])
            fuelcost = fuelcost_dict[f]
            η = pp_dict[:efficiency][p]
            co_price = fuelcost_dict[:CO2]
            emission = pp_dict[:emission][p]
            vc = pp_dict[:varcost][p]

            mc[p] = fuelcost / η + co_price * emission .+ vc
        end

        return new(G,
                   pp_dict[:node],
                   pp_dict[:pmax],
                   pp_dict[:pmin],
                   pp_dict[:rup],
                   pp_dict[:rdn],
                   pp_dict[:timeup],
                   pp_dict[:timedn],
                   mc,
                   pp_dict[:varcost],
                   pp_dict[:fuel],
                   pp_dict[:efficiency],
                   pp_dict[:emission]
                   )
    end
end


#
# Summarize PRS simulation result
#
using DelimitedFiles
using Random
using DataFrames
using CSV
using Printf

# view one result
sim = 1
df = CSV.read("sim$sim/summary.txt", DataFrame)
# for sim in 1:20
#     df = CSV.read("sim$sim/summary.txt", DataFrame)
#     @show df
# end

# check cross validation result is the same for every run
# for fdr in [0.05, 0.1, 0.25, 0.5], sim in 1:100
#     f = "/scratch/users/bbchu/ukb/prs/Radj20_K50_s0/fdr$fdr/sim$sim/summary.txt"
#     g = "/scratch/users/bbchu/ukb/prs/Radj20_K50_s0/fdr$fdr/sim$sim/summary.txt"
#     if isfile(f) && isfile(g)
#         df = CSV.read(f, DataFrame)
#         dg = CSV.read(g, DataFrame)
#         if !all(df[!, 2] .≈ df[!, 2])
#             println("sim $sim fdr $fdr failed!")
#             println(df[!, 2])
#             println(dg[!, 2])
#         end
#     end
# end

# all results
df_sum = zeros(16, 5)
successes = 0
for sim in 1:100
    f = "sim$sim/summary.txt"
    if isfile(f)
        df = CSV.read(f, DataFrame)
        m = Matrix{Float64}(df[:, 2:end])
        any(isnan.(m)) && continue
        df_sum += m
        successes += 1
    else
        @warn "sim $sim failed!"
    end
end
df_mean = df_sum ./ successes
# get std
df_std = zeros(16, 5)
for sim in 1:100
    f = "sim$sim/summary.txt"
    if isfile(f)
        df = Matrix{Float64}(CSV.read(f, DataFrame)[:, 2:end])
        df_std += abs2.(df .- df_mean)
    end
end
df_std ./= successes
df_std .= sqrt.(df_std)
# save result
writedlm("prs_population_R2_mean.txt", df_mean)
writedlm("prs_population_R2_std.txt", df_std)

# print latex table
column_name = ["African", "Asian", "Bangladeshi", "British", "Caribbean", "Chinese",
    "Indian", "Irish", "Pakistani", "White-asian", "White-black", "White",
    "non-zero betas", "beta selected", "TPP", "FDP"]

for i in 1:size(df_mean, 1)
    print(column_name[i])
    for j in 1:5
        μ = @sprintf "%0.3f" df_mean[i, j]
        σ = @sprintf "%0.3f" df_std[i, j]
        print(" & \$", μ, "\\pm", σ, "\$")
        # print(" & \$", round(df_mean[i, j], digits=3), "\\pm",
        #     round(df_std[i, j], digits=3), "\$")
    end
    print("\\\\ \n")
    i == 12 && println("\\hline\n\\hline")
end
df_mean


#****************************************************************************
#                     Molecular Dynamics Potentials (MDP)
#                            CESMIX-MIT Project
#
#  Contributing authors: Ngoc-Cuong Nguyen (cuongng@mit.edu, exapde@gmail.com)
#****************************************************************************

mutable struct PotentialStruct

    style::String # single, pair, triplet, quadlet, snap
    bondtype::String # nonbonded, bonded
    bondedatomtypes::Vector{Int64} # a list of atom types for the bond
    potentialfunction::String # a string for potential function
    mu::Vector{Float64} # linear potential parameters
    rcut::Float64 # cut-off radius

end

function addpotential(style::String, bondtype::String, bondedatomtypes::Vector{Int64}, potentialfunction::String, mu::Vector{Float64}, rcut::Float64)

    potential = PotentialStruct(style, bondtype, bondedatomtypes, potentialfunction, mu, rcut)

    return potential
end

function getpotential(potential::PotentialStruct)

    style = potential.style
    bondtype = potential.bondtype
    bondedatomtypes = potential.bondedatomtypes
    potentialfunction = potential.potentialfunction
    mu = potential.mu
    rcut = potential.rcut

    return style, bondtype, bondedatomtypes, potentialfunction, mu, rcut
end


# MadNLP.jl
# Created by Sungho Shin (sungho.shin@wisc.edu)

module MadNLPPardisoMKL

import ..MadNLP:
    @kwdef, Logger, @debug, @warn, @error,
    SubVector, StrideOneVector, SparseMatrixCSC, libblas,
    SymbolicException,FactorizationException,SolveException,InertiaException,
    AbstractOptions, AbstractLinearSolver, set_options!,
    introduce, factorize!, solve!, improve!, is_inertia, inertia, blas_num_threads

const INPUT_MATRIX_TYPE = :csc

@enum(MatchingStrategy::Int,COMPLETE=1,COMPLETE2x2=2,CONSTRAINTS=3)

@kwdef mutable struct Options <: AbstractOptions
    pardisomkl_num_threads::Int = 1
    pardiso_matching_strategy::MatchingStrategy = COMPLETE2x2
    pardisomkl_max_iterative_refinement_steps::Int = 1
    pardisomkl_msglvl::Int = 0
    pardisomkl_order::Int = 2
end

mutable struct Solver <: AbstractLinearSolver
    pt::Vector{Ptr{Cvoid}}
    iparm::Vector{Int32}
    perm::Vector{Int32}
    msglvl::Ref{Int32}
    err::Ref{Int32}
    csc::SparseMatrixCSC{Float64,Int32}
    w::Vector{Float64}
    opt::Options
    logger::Logger
end

pardisomkl_pardisoinit(pt,mtype::Ref{Cint},iparm::Vector{Cint}) =
    ccall(
        (:pardisoinit,libblas),
        Cvoid,
        (Ptr{Cvoid},Ptr{Cint},Ptr{Cint}),
        pt,mtype,iparm)
pardisomkl_pardiso(pt,maxfct::Ref{Cint},mnum::Ref{Cint},mtype::Ref{Cint},
                   phase::Ref{Cint},n::Ref{Cint},a::Vector{Cdouble},ia::Vector{Cint},ja::Vector{Cint},
                   perm::Vector{Cint},nrhs::Ref{Cint},iparm::Vector{Cint},msglvl::Ref{Cint},
                   b::StrideOneVector{Cdouble},x::StrideOneVector{Cdouble},err::Ref{Cint}) =
                       ccall(
                           (:pardiso,libblas),
                           Cvoid,
                           (Ptr{Cvoid}, Ptr{Cint}, Ptr{Cint}, Ptr{Cint}, Ptr{Cint},
                            Ptr{Cint}, Ptr{Cdouble}, Ptr{Cint}, Ptr{Cint}, Ptr{Cint},
                            Ptr{Cint}, Ptr{Cint}, Ptr{Cint}, Ptr{Cdouble}, Ptr{Cdouble},Ptr{Cint}),
                           pt,maxfct,mnum,mtype,phase,n,a,ia,ja,perm,nrhs,iparm,msglvl,b,x,err)

function pardisomkl_set_num_threads!(n)
    ccall((:mkl_set_dynamic, libblas),
         Cvoid,
         (Ptr{Cint},),
          Ref{Cint}(0))
    ccall((:mkl_set_num_threads, libblas),
          Cvoid,
          (Ptr{Cint},),
          Ref{Cint}(n))
end

function Solver(csc::SparseMatrixCSC;
                opt=Options(),logger=Logger(),
                option_dict::Dict{Symbol,Any}=Dict{Symbol,Any}(),
                kwargs...)
    !isempty(kwargs) && (for (key,val) in kwargs; option_dict[key]=val; end)
    set_options!(opt,option_dict)

    w   = Vector{Float64}(undef,csc.n)

    pt = Vector{Ptr{Cvoid}}(undef,64)
    iparm = Vector{Int32}(undef,64)

    perm = Vector{Int32}(undef,csc.n)
    msglvl = Ref{Int32}(0)
    err = Ref{Int32}(0)

    pt.=0
    iparm[1]=0

    pardisomkl_pardisoinit(pt,Ref{Int32}(-2),iparm)

    iparm[1]=1
    iparm[2]=opt.pardisomkl_order # METIS
    iparm[3]=0
    iparm[6]=1
    iparm[8]=opt.pardisomkl_max_iterative_refinement_steps
    iparm[10]=12
    iparm[11]=2
    iparm[13]=1 # matching strateg
    iparm[21]=3 # bunch-kaufman pivotin
    iparm[24]=1 # parallel factorization
    iparm[25]=0 # parallel solv

    pardisomkl_set_num_threads!(opt.pardisomkl_num_threads)
    pardisomkl_pardiso(pt,Ref{Int32}(1),Ref{Int32}(1),Ref{Int32}(-2),Ref{Int32}(11),
                     Ref{Int32}(csc.n),csc.nzval,csc.colptr,csc.rowval,perm,
                       Ref{Int32}(1),iparm,msglvl,Float64[],Float64[],err)
    pardisomkl_set_num_threads!(blas_num_threads[])

    err.x < 0  && throw(SymbolicException())

    M = Solver(pt,iparm,perm,msglvl,err,csc,w,opt,logger)

    finalizer(finalize,M)

    return M
end
function factorize!(M::Solver)
    pardisomkl_set_num_threads!(M.opt.pardisomkl_num_threads)
    pardisomkl_pardiso(M.pt,Ref{Int32}(1),Ref{Int32}(1),Ref{Int32}(-2),Ref{Int32}(22),
                     Ref{Int32}(M.csc.n),M.csc.nzval,M.csc.colptr,M.csc.rowval,M.perm,
                       Ref{Int32}(1),M.iparm,M.msglvl,Float64[],Float64[],M.err)
    M.err.x < 0  && throw(FactorizationException())
    pardisomkl_set_num_threads!(blas_num_threads[])

    M.iparm[14] == 0 && return M
    pardisomkl_set_num_threads!(M.opt.pardisomkl_num_threads)
    pardisomkl_pardiso(M.pt,Ref{Int32}(1),Ref{Int32}(1),Ref{Int32}(-2),Ref{Int32}(12),
                     Ref{Int32}(M.csc.n),M.csc.nzval,M.csc.colptr,M.csc.rowval,M.perm,
                       Ref{Int32}(1),M.iparm,M.msglvl,Float64[],Float64[],M.err)
    pardisomkl_set_num_threads!(blas_num_threads[])
    M.err.x < 0  && throw(FactorizationException())
    return M
end
function solve!(M::Solver,rhs::StrideOneVector{Float64})
    pardisomkl_set_num_threads!(M.opt.pardisomkl_num_threads)
    pardisomkl_pardiso(M.pt,Ref{Int32}(1),Ref{Int32}(1),Ref{Int32}(-2),Ref{Int32}(33),
                     Ref{Int32}(M.csc.n),M.csc.nzval,M.csc.colptr,M.csc.rowval,M.perm,
                       Ref{Int32}(1),M.iparm,M.msglvl,rhs,M.w,M.err)
    pardisomkl_set_num_threads!(blas_num_threads[])
    M.err.x < 0  && throw(SolveException())
    return rhs
end

function finalize(M::Solver)
    pardisomkl_pardiso(M.pt,Ref{Int32}(1),Ref{Int32}(1),Ref{Int32}(-2),Ref{Int32}(-1),
                     Ref{Int32}(M.csc.n),M.csc.nzval,M.csc.colptr,M.csc.rowval,M.perm,
                     Ref{Int32}(1),M.iparm,M.msglvl,Float64[],M.w,M.err)
end
is_inertia(::Solver)=true
function inertia(M::Solver)
    pos = M.iparm[22]
    neg = M.iparm[23]
    return (pos,M.csc.n-pos-neg,neg)
end

function improve!(M::Solver)
    @debug(M.logger,"improve quality failed.")
    return false
end
introduce(::Solver)="pardiso-mkl"

end # module


@testset "track" begin
    @testset "basic" begin
        @system STrack(Controller) begin
            a => 1 ~ track
            b => 2 ~ track
            c(a, b) => a + b ~ track
        end
        s = instance(STrack)
        @test s.a' == 1 && s.b' == 2 && s.c' == 3
    end

    @testset "cross reference" begin
        @test_throws LoadError @eval @system STrackXRef(Controller) begin
            a(b) => b ~ track
            b(a) => a ~ track
        end
    end

    @testset "self reference without init" begin
        @eval @system STrackSRefWithoutInit(Controller) begin
            a(a) => 2a ~ track
        end
        @test_throws UndefVarError instance(STrackSRefWithoutInit)
    end

    @testset "self reference with init" begin
        @system STrackSRefWithInit(Controller) begin
            a(a) => 2a ~ track(init=1)
        end
        r = simulate(STrackSRefWithInit, stop=3)
        @test r[!, :a] == [2, 4, 8, 16]
    end

    @testset "init" begin
        @system STrackInit(Controller) begin
            a(a) => a + 1 ~ track(init=0)
            b(b) => b + 1 ~ track(init=1)
            c(c) => c + 1 ~ track(init=i)
            d => 1 ~ track(init=i)
            i(t=context.clock.tick) => t + 1 ~ track
        end
        r = simulate(STrackInit, stop=3)
        @test r[!, :a] == [1, 2, 3, 4]
        @test r[!, :b] == [2, 3, 4, 5]
        @test r[!, :c] == [2, 3, 4, 5]
        @test r[!, :d] == [1, 1, 1, 1]
    end

    @testset "minmax" begin
        @system STrackMinMax(Controller) begin
            a => 0 ~ track(min=1)
            b => 0 ~ track(max=2)
            c => 0 ~ track(min=1, max=2)
        end
        s = instance(STrackMinMax)
        @test s.a' == 1
        @test s.b' == 0
        @test s.c' == 1
    end

    @testset "minmax unit" begin
        @system STrackMinMaxUnit(Controller) begin
            a => 0 ~ track(u"m", min=1)
            b => 0 ~ track(u"m", max=2u"cm")
            c => 0 ~ track(u"m", min=1u"cm", max=2)
        end
        s = instance(STrackMinMaxUnit)
        @test s.a' == 1u"m"
        @test s.b' == 0u"m"
        @test s.c' == 1u"cm"
    end

    @testset "round" begin
        @system STrackRound(Controller) begin
            a => 1.5 ~ track(round)
            b => 1.5 ~ track(round=:round)
            c => 1.5 ~ track(round=:ceil)
            d => -1.5 ~ track(round=:floor)
            e => -1.5 ~ track(round=:trunc)
        end
        s = instance(STrackRound)
        @test s.a' === s.b'
        @test s.b' === 2.0
        @test s.c' === 2.0
        @test s.d' === -2.0
        @test s.e' === -1.0
    end

    @testset "round int" begin
        @system STrackRoundInt(Controller) begin
            a => 1.5 ~ track::int(round)
            b => 1.5 ~ track::int(round=:round)
            c => 1.5 ~ track::int(round=:ceil)
            d => -1.5 ~ track::int(round=:floor)
            e => -1.5 ~ track::int(round=:trunc)
        end
        s = instance(STrackRoundInt)
        @test s.a' === s.b'
        @test s.b' === 2
        @test s.c' === 2
        @test s.d' === -2
        @test s.e' === -1
    end

    @testset "round int unit" begin
        @system STrackRoundIntUnit(Controller) begin
            a => 1.5 ~ track::int(u"d", round)
            b => 1.5 ~ track::int(u"d", round=:round)
            c => 1.5 ~ track::int(u"d", round=:ceil)
            d => -1.5 ~ track::int(u"d", round=:floor)
            e => -1.5 ~ track::int(u"d", round=:trunc)
        end
        s = instance(STrackRoundIntUnit)
        @test s.a' === s.b'
        @test s.b' === 2u"d"
        @test s.c' === 2u"d"
        @test s.d' === -2u"d"
        @test s.e' === -1u"d"
    end

    @testset "when" begin
        @system STrackWhen(Controller) begin
            T => true ~ preserve::Bool
            F => false ~ preserve::Bool
            a => 1 ~ track
            b => 1 ~ track(when=true)
            c => 1 ~ track(when=false)
            d => 1 ~ track(when=T)
            e => 1 ~ track(when=F)
            f => 1 ~ track(when=!F)
        end
        s = instance(STrackWhen)
        @test s.a' == 1
        @test s.b' == 1
        @test s.c' == 0
        @test s.d' == 1
        @test s.e' == 0
        @test s.f' == 1
    end

    @testset "when unit" begin
        @system STrackWhenUnit(Controller) begin
            T => true ~ preserve::Bool
            F => false ~ preserve::Bool
            a => 1 ~ track(u"d")
            b => 1 ~ track(u"d", when=true)
            c => 1 ~ track(u"d", when=false)
            d => 1 ~ track(u"d", when=T)
            e => 1 ~ track(u"d", when=F)
            f => 1 ~ track(u"d", when=!F)
        end
        s = instance(STrackWhenUnit)
        @test s.a' == 1u"d"
        @test s.b' == 1u"d"
        @test s.c' == 0u"d"
        @test s.d' == 1u"d"
        @test s.e' == 0u"d"
        @test s.f' == 1u"d"
    end

    @testset "when init" begin
        @system STrackWhenInit(Controller) begin
            T => true ~ preserve::Bool
            F => false ~ preserve::Bool
            I => 1 ~ preserve
            a => 0 ~ track(init=1, when=true)
            b => 0 ~ track(init=1, when=false)
            c => 0 ~ track(init=I, when=T)
            d => 0 ~ track(init=I, when=F)
        end
        s = instance(STrackWhenInit)
        @test s.a' == 0
        @test s.b' == 1
        @test s.c' == 0
        @test s.d' == 1
    end
end


#=
This file contains the helper functions necessary for file import/export
of various file formats.
=#

"""
    determine_marker(geno_parse::T, digits::Int) where T<:AbstractDataFrame
Return either `Int8` or `Int16` depending on largest allelic value in all genotypes
in the first 10 samples of an input DataFrame (or all the samples if less than 10 samples).
If the largest allele is 20 or greater, the marker will be considered a Microsatellite
and coded in `PopData` as `Int16`, and the opposite is true for SNPs. There's no
specific reason 100 was chosen other than it being a reasonable buffer for edge
cases since SNP data <= 4, and haplotyped data could be a bit higher. Even if the
microsatellite markers are coded incorrectly, there will be zero impact to performance,
and considering how few microsatellite markers are used in typical studies, the
effect on in-memory size should be negligible (as compared to SNPs).
"""
function determine_marker(geno_parse::T, digits::Int) where T<:AbstractDataFrame
    # get the total # columns
    total_col = size(geno_parse,2)
    # find the 25% cutoff
    if total_col > 11
        num_test_loc = (total_col - 2) ÷ 8
    else
        num_test_loc = total_col - 2
    end
    # remove everything else
    test_df = @view geno_parse[:, 3:num_test_loc]

    # isolate the largest allele value
    max_allele = map(eachcol(test_df)) do i
        phase.(i, Int16, digits)  |>
        skipmissing |> Base.Iterators.flatten |> maximum
    end |> maximum

    if max_allele <= 100
        return Int8
    else
        return Int16
    end
end

"""
    find_ploidy(genotypes::T where T<:SubArray)
Used internally in the `genepop` and `delimited` file parsers to scan the genotypes
of a sample and return the ploidy of the first non-missing locus.
"""
@inline function find_ploidy(genotypes::T) where T<:GenoArray
    return Int8(length(first(skipmissing(genotypes))))
end

"""
    phase(loc::Union{String, Int}, type::DataType, digit::Int)
Takes a String of numbers or Integers and returns a typed locus appropriate for PopGen.jl as used in the
`genepop` and `csv` file parsers. Use `type` to specify output type (`Int8` or `Int16`),
and `digit` to specify the number of digits/characters used per allele in a locus.

## Examples
```
ph_locus = phase("128114", Int16, 3)
map(i -> phase(i, Int16, 3), ["112131", "211112", "001003", "516500"])
# or #
[phase(i, Int8, 2) for i in ["0101", "0103", "0202", "0103"]]
```
"""
@inline function phase(loc::T, type::DataType, digit::Int) where T<:AbstractString
    loc == "-9" || iszero(parse(Int, loc)) && return missing
    phased = map(i -> parse(type, join(i)), Iterators.partition(loc, digit))
    sort!(phased)
    Tuple(phased)
end

phase(loc::Missing, type::DataType, digit::Int) = missing

@inline function phase(loc::T, type::DataType, digit::Int) where T<:Integer
    loc == -9 || iszero(loc) && return missing
    units = 10^digit
    allele1 = loc ÷ units |> type
    allele2 = loc % units |> type
    return [allele1, allele2] |> sort |> Tuple
end

"""
    unphase(geno::T; digits::Int = 3, ploidy::Int = 2, miss::Int = 0) where T <: Genotype
Takes a `Genotype` (e.g. `(131, 94)`) and returns a string of concatenated
alleles padded with *n* number of zeroes, where *n* is given by `digits = `.
`missing` values are returned as either a string of 'digits × ploidy' zeroes (`miss = 0`)
or `"-9"` (`miss = -9`). The `ploidy` flag is only relevant for unphasing `missing` genotypes
and not used otherwise.

### Example
```
unphase((1,2,3,4), digits = 3)
"001002003004"

unphase(missing, digits = 2, ploidy = 2, miss = -9)
"-9"

unphase(missing, digits = 2, ploidy = 2, miss = 0)
"0000"
```
"""
function unphase(geno::T; digits::Int = 3, ploidy::Int = 2, miss::Int = 0) where T <: Genotype
    join(map(i -> lpad(i, digits, "0"), geno))
end

function unphase(geno::Missing; digits::Int = 3, ploidy::Int, miss::Int = 0)
    if miss == 0
        return "0"^(digits * ploidy)
    else
        return "$miss"
    end
end


# Tests for the Transaction system.
# Author: Noah C. Benson <n@nben.net>

@testset "TX" begin

    @testset "tx1" begin
        vals = Int[]
        vols = Vector{Volatile{Int}}[]
        log = Actor{typeof(stdout)}(stdout)
        worker() = begin
            global vals
            global vols
            while vols[end][1][] < 0
                # Pick a random layer and cell to calculate
                l = rand((2,3,4,5))
                layer = vols[l]
                k = rand(1:length(layer))
                # synconize for the rest.
                tx() do
                    #println("$l $k $(current_tx() === nothing) $(getfield(_current_tx, :values)))")
                    res = layer[k]
                    if res[] < 0
                        up = vols[l - 1]
                        a = up[k*2 - 1]
                        b = up[k*2]
                        if a[] >= 0 && b[] >= 0
                            res[] = a[] + b[]
                            send(log) do log
                                println("Layer $l, item $k complete $(objectid(current_task())).")
                                log
                            end
                        end
                    end
                end
            end
            send(log) do log
                println("Ending Woker $(objectid(current_task())).")
                log
            end
        end
        tx1_run(n::Int) = begin
            global vals
            global vols
            vals = rand(0:1000, 16)
            vols = Vector{Volatile{Int}}[
                [Volatile{Int}(u)   for u in vals],
                [Volatile{Int}(-1) for _ in 1:8],
                [Volatile{Int}(-1) for _ in 1:4],
                [Volatile{Int}(-1) for _ in 1:2],
                [Volatile{Int}(-1)]]
            # Spawn n threads.
            threads = [(Threads.@spawn worker()) for _ in 1:n]
            for th in threads
                Threads.wait(th)
            end
            println("  ---->  $(vols[end][1][]) == $(sum(vals))")
            return (vols[end][1][] == sum(vals))
        end
        @test tx1_run(1)
        @test tx1_run(2)
        @test tx1_run(3)
        @test tx1_run(4)
        @test tx1_run(5)
        @test tx1_run(6)
        @test tx1_run(7)
        @test tx1_run(8)
        @test tx1_run(9)
        @test tx1_run(10)
        @test tx1_run(11)
    end

end


import Base: convert, ==

# import JuLIP: energy, virial, site_energies, forces
# import JuLIP.Potentials: evaluate

import JuLIP.FIO: read_dict, write_dict
using JuLIP: JVec, JMat, chemical_symbols

export OneBody

"""
`mutable struct OneBody{T}  <: NBodyFunction{1}`

this should not normally be constructed by a user, but instead E0 should be
passed to the relevant lsq functions, which will construct it.
This structure deals with multi-species configurations.
"""
mutable struct OneBody{T <: AbstractFloat} <: AbstractCalculator
   E0::Dict{Symbol, T}
end

@pot OneBody

OneBody(args...) = OneBody(Dict(args...))

evaluate(V::OneBody, sp) = V.E0[sp]

function site_energies(V::OneBody, at::AbstractAtoms{T}) where {T}
   E = zeros(T, length(at))
   for i in 1:length(at)
      E[i] = V(chemical_symbols(at)[i])
   end
   return E
end

energy(V::OneBody, at::AbstractAtoms) = sum(site_energies(V, at))

forces(V::OneBody, at::AbstractAtoms{T}) where {T} = zeros(JVec{T}, length(at))

virial(V::OneBody, at::AbstractAtoms{T}) where {T} = zero(JMat{T})

write_dict(V::OneBody) =
         Dict("__id__" => "JuLIP_OneBody",
              "E0" => Dict([String(key) => val for (key, val) in V.E0]...))

# convert the E0 Dict{String} read from JSON into a Dict{Symbol}
OneBody(D::Dict{String}, T=Float64) =
   OneBody([Symbol(key) => T(val) for (key, val) in D]...)

read_dict(::Val{:JuLIP_OneBody}, D::Dict) = OneBody(D["E0"])

==(V1::OneBody, V2::OneBody) = (V1.E0 == V2.E0)


# To convert video to png
# run(`ffmpeg -r 1 -i $folder/$file.avi -r 1 $folder/png/$file%03d.png`)

# using FileIO

"""
    Reads a video.
    Picks the red channels since all the channels were equal.
    Returns Array{Gray{T}, 3}
"""
function readvideo(filename)
    f = openvideo(filename)
    A1 = []
    while !eof(f)
        push!(A1, read(f))
    end
    # make A1 typed
    function checkgray(x)
        red(x) == green(x) == blue(x)
        Gray(red(x))
    end
    catfun([x for x in A1], checkgray)
end
function catfun(As::AbstractArray{S}, fun) where {TE, T <: Color{TE}, S <: AbstractArray{T}}
    AA = Array{Gray{TE}, 3}(undef, size(As[1])..., length(As))
    for (i, A) in enumerate(As)
        v = view(AA, :, :, i)
        @. v = fun(A)
    end
    AA
end


@enum DirectionType begin
    Forward
    Backward
end

"""
    Track{BCFwd,BOut,DFwd,DBwd,T<:Real}

Represents a neutron trajectory across the domain, with certain azimuthal angle `ϕ`, entry
and exit points `p` and `q`, length `ℓ` and formed by `segments` coming from its
segmentation.
"""
mutable struct Track{BCFwd,BCBwd,DFwd,DBwd,T<:Real}
    uid::Int
    azim_idx::Int
    track_idx::Int

    p::Point2D{T}
    q::Point2D{T}

    ϕ::T
    ℓ::T

    ABC::SVector{3,T}
    segments::Vector{Segment{T}}

    next_track_fwd::Track
    next_track_bwd::Track

    function Track{BCFwd,BCBwd,DFwd,DBwd}(
        uid::Int,
        azim_idx::Int,
        track_idx::Int,
        p::Point2D{T},
        q::Point2D{T},
        ϕ::T,
        ℓ::T,
        ABC::SVector{3,T},
        segments::Vector{Segment{T}}
    ) where {BCFwd,BCBwd,DFwd,DBwd,T}
        track = new{BCFwd,BCBwd,DFwd,DBwd,T}(
            uid, azim_idx, track_idx, p, q, ϕ, ℓ, ABC, segments
        )
        track.next_track_fwd = track
        track.next_track_bwd = track
        return track
    end
end

universal_id(track::Track) = track.uid
azim_idx(track::Track) = track.azim_idx
track_idx(track::Track) = track.track_idx
bc_fwd(::Track{BCFwd}) where {BCFwd} = BCFwd
bc_bwd(::Track{BCFwd,BCBwd}) where {BCFwd,BCBwd} = BCBwd
dir_next_track_fwd(::Track{BCFwd,BCBwd,DFwd}) where {BCFwd,BCBwd,DFwd} = DFwd
dir_next_track_bwd(::Track{BCFwd,BCBwd,DFwd,DBwd}) where {BCFwd,BCBwd,DFwd,DBwd} = DBwd

function show(io::IO, track::Track)
    @unpack ϕ, p, q, ℓ, segments = track
    # println(io, typeof(t))
    println(io, "  Azimuthal angle: ", round(rad2deg(ϕ), digits=2))
    println(io, "  Entry point: ", p)
    println(io, "  Exit point: ", q)
    println(io, "  Length: ", ℓ)
    println(io, "  # of segments: ", length(segments)) # if it is zero, run segmentize!
    println(io, "  Boundary fwd: ", bc_fwd(track))
    print(io,   "  Boundary bwd: ", bc_bwd(track))
    # avoid printing circular references (since it is a cyclic ray tracing...)
    # println(io, "  Next track fwd: ", next_track_fwd)
    # print(io,   "  Next track fwd: ", next_track_bwd)
end

# since we are in active development of the package and there might be unconsidered cases in
# the segmentation of tracks, let's have this maximum number of iterations.
const MAX_ITER = 10_000

function _segmentize_track!(t, track::Track, k::Int, rtol::Real) # t::TrackGenerator
    @unpack mesh, tiny_step = t
    @unpack ϕ, segments = track

    # since we are going to compute them, empty!
    empty!(segments)

    # move a tiny step in ϕ direction to get inside the mesh
    xp = advance_step(track.p, tiny_step, ϕ)

    i = 0
    element = -1
    prev_element = -1
    while i < MAX_ITER

        # find the element or cell where `xp` lies
        element = find_element(mesh, xp)

        # we might be at the boundary of the mesh
        if inboundary(mesh, xp, tiny_step)
            if isempty(segments)
                # if we just started to segmentize, move a tiny step forward
                xp = advance_step(xp, tiny_step, ϕ)
                continue
            else
                # or we just finished, so leave
                break
            end
        end

        # if we are inside the domain and `find_element` did not return an element, we might
        # be dealing with a deformed mesh, so we might need to increase the knn search.
        if isequal(element, -1)
            element = find_element(mesh, xp, k)
            if isequal(element, -1)
                error("Try increasing `k`. If the problem persists, raise an issue, this " *
                      "might be a case that hasn't been presented before.")
            end
        end

        # move a step if the new element is the same as the previous one
        if isequal(prev_element, element)
            xp = advance_step(xp, tiny_step, ϕ)
            continue
        end

        # compute intersections between track and the element
        p, q = intersections(mesh, element, track)

        # we might be in a vertex, so we just continue
        if isapprox(p, q)
            xp = advance_step(xp, tiny_step, ϕ)
            continue
        end

        segment = Segment(p, q, element)
        push!(segments, segment)

        # update new starting point and previous element
        xp = advance_step(q, tiny_step, ϕ)
        prev_element = element

        i += 1 # just for safety
    end

    if !isapprox(track.ℓ, sum(ℓ.(track.segments)); rtol=rtol)
        error("Track with `uid` $(string(track.uid)) has a length that do not match the " *
              "sum of its segments lengths with the provided tolerance `rtol`. Check " *
              "whether this is an actual error or increase `rtol`.")
    end

    return nothing
end


using TypeClasses

# Monoid instances
# ================

# this is standard Applicative combine implementation
# there is no other sensible definition for combine and hence if this does fail, it fails correctly
TypeClasses.combine(a::Callable, b::Callable) = mapn(combine, a, b)

# we do not implement the same for `orelse`, as `orelse` is usually defined on the container level, but there is no sensible default implementation.

# it is not possible to implement neutral, as we do not know the element type without executing the function

# Monad instances
# ===============

# there is no definition for Base.foreach, as a callable is not runnable without knowing the arguments
TypeClasses.map(f, g::Callable) = Callable((args...; kwargs...) -> f(g(args...; kwargs...)))

TypeClasses.pure(::Type{<:Callable}, a) = (args...; kwargs...) -> a
TypeClasses.ap(f::Callable, g::Callable) = Callable((args...; kwargs...) -> f(args...; kwargs...)(g(args...; kwargs...)))

# we cannot overload this generically, because `Base.map(f, ::Vector...)` would get overwritten as well (even without warning surprisingly)
TypeClasses.map(f, a::Callable, b::Callable, more::Callable...) = mapn(f, a, b, more...)

# we don't use convert, but directly use function call,
# which should give readable errors that a something is not callable, and is a bit more flexible
TypeClasses.flatmap(f, g::Callable) = Callable((args...; kwargs...) -> f(g(args...; kwargs...))(args...; kwargs...))

# FlipTypes instance
# ==================

# there cannot be any flip_types for functions


# All constants defined in the file are exported

for name in [
    ### Matrix types from cvdef.h ###
    :CV_CN_MAX,
    :CV_CN_SHIFT,
    :CV_DEPTH_MAX,

    :CV_8U,
    :CV_8S,
    :CV_16U,
    :CV_16S,
    :CV_32S,
    :CV_32F,
    :CV_64F,
    :CV_USRTYPE1,

    :CV_MAT_DEPTH_MASK,

    :CV_8UC1,
    :CV_8UC2,
    :CV_8UC3,
    :CV_8UC4,

    :CV_16UC1,
    :CV_16UC2,
    :CV_16UC3,
    :CV_16UC4,

    :CV_16SC1,
    :CV_16SC2,
    :CV_16SC3,
    :CV_16SC4,

    :CV_32SC1,
    :CV_32SC2,
    :CV_32SC3,
    :CV_32SC4,

    :CV_32FC1,
    :CV_32FC2,
    :CV_32FC3,
    :CV_32FC4,

    :CV_64FC1,
    :CV_64FC2,
    :CV_64FC3,
    :CV_64FC4,

    :CV_MAT_CN_MASK,
    :CV_MAT_TYPE_MASK,
    :CV_MAT_CONT_FLAG_SHIFT,
    :CV_MAT_CONT_FLAG,
    :CV_SUBMAT_FLAG_SHIFT,
    :CV_SUBMAT_FLAG,
    ]
    ex = Expr(:macrocall, Symbol("@icxx_str"), string(name, ";"))
    @eval begin
        global const $name = $ex
        export $name
    end
end

const BorderTypes = Cxx.CppEnum{Symbol("cv::BorderTypes"),UInt32}
const CmpTypes = Cxx.CppEnum{Symbol("cv::CmpTypes"),UInt32}
const DecompTypes = Cxx.CppEnum{Symbol("cv::DecompTypes"),UInt32}
const DftFlags = Cxx.CppEnum{Symbol("cv::DftFlags"),UInt32}
const GemmFlags = Cxx.CppEnum{Symbol("cv::GemmFlags"),UInt32}

for name in [
    :ACCESS_READ,
    :ACCESS_WRITE,
    :ACCESS_RW,
    :ACCESS_MASK,
    :ACCESS_FAST,

    :BORDER_CONSTANT,
    :BORDER_REPLICATE,
    :BORDER_REFLECT,
    :BORDER_WRAP,
    :BORDER_REFLECT_101,
    :BORDER_TRANSPARENT,
    :BORDER_REFLECT101,
    :BORDER_DEFAULT,
    :BORDER_ISOLATED,

    :CMP_EQ,
    :CMP_GT,
    :CMP_GE,
    :CMP_LT,
    :CMP_LE,
    :CMP_NE,

    :DECOMP_LU,
    :DECOMP_SVD,
    :DECOMP_EIG,
    :DECOMP_CHOLESKY,
    :DECOMP_QR,
    :DECOMP_NORMAL,

    :DFT_INVERSE,
    :DFT_SCALE,
    :DFT_ROWS,
    :DFT_COMPLEX_OUTPUT,
    :DFT_REAL_OUTPUT,
    :DCT_INVERSE,
    :DCT_ROWS,

    :GEMM_1_T,
    :GEMM_2_T,
    :GEMM_3_T,

    :NORM_INF,
    :NORM_L1,
    :NORM_L2,
    :NORM_L2SQR,
    :NORM_HAMMING,
    :NORM_HAMMING2,
    :NORM_TYPE_MASK,
    :NORM_RELATIVE,
    :NORM_MINMAX,
    ]
    ex = Expr(:macrocall, Symbol("@icxx_str"), string("cv::", name, ";"))
    @eval global const $name = $ex
end


module dlgo

export Player, black, white, void
export other, isvoid, nbrs, zobristhash
export Point, Move, GoString, GoStringBuffer, IdType, Board, GameState,
       Territory, TerritoryStatus, GameResult
export getindex, setindex!, ==, is_on_grid
export new_game, apply_move, is_valid_move, is_over, resign, play, pass_turn, legal_moves
export grid, stringdict, stringbuffer, getpointid, setpointid!, getstring,
       setstring!, place_stone!, remove_string!, remove_point!
export tosymbol, print_board, point_from_coords, print_move
export evaluate_territory, score, winnercolor, compute_game_result
export features

include("gotypes.jl")
include("zobrist.jl")
include("goboard.jl")
include("stringrep.jl")
include("scoring.jl")
include("features.jl")

include("agent/agent.jl")

end


##############################################################################################################################
# Conversion and Promotion rules
# Small convention: Convert from type T to type S
##############################################################################################################################

"""
	mat(x::T where T<:AbstractVector, dim=LearnBase.ObsDim.Constant{1})

Function that transforms a vector into a row or column matrix.
"""
mat(x::T where T<:AbstractVector) = mat(getobs(x), LearnBase.ObsDim.Constant{1}())
mat(x::T where T<:AbstractVector, dim::S where S<:LearnBase.ObsDim.Constant{1}) = reshape(getobs(x), size(x,1), 1)
mat(x::T where T<:AbstractVector, dim::S where S<:LearnBase.ObsDim.Constant{2}) = reshape(getobs(x), 1, size(x,1))
mat(x::T where T<:AbstractMatrix, dim::S where S<:LearnBase.ObsDimension) = getobs(x)
mat(x::T where T<:Vector) = mat(x, LearnBase.ObsDim.Constant{1}())
mat(x::T where T<:Vector, dim::S where S<:LearnBase.ObsDim.Constant{1}) = reshape(x, size(x,1), 1)
mat(x::T where T<:Vector, dim::S where S<:LearnBase.ObsDim.Constant{2}) = reshape(x, 1, size(x,1))
mat(x::T where T<:Matrix, dim::S where S<:LearnBase.ObsDimension) = x

# Convert from Vectors to Matrices
convert(::Type{S}, x::T where T<:AbstractVector) where S<:AbstractMatrix = mat(x)

# Convert from CellData to AbstractArray
convert(::Type{S}, x::T where T<:DataCell) where S<:AbstractArray = convert(S, getx!(x))

# Convert to DataCell
convert(::Type{S}, x::T where T<:DataElement) where S<:DataCell = DataCell([x], nothing, nothing, 0, "DataCell (convert)", eval(oinfoglobal))
convert(::Type{S}, x::T where T<:AbstractArray) where S<:DataCell = DataCell(x, nothing, nothing, 0, "DataCell (convert)", eval(oinfoglobal))

# Convert to FunctionCell
convert(::Type{S}, x::T where T<:DataElement) where S<:FunctionCell = FunctionCell(nothing, nothing, (args...)->x, (), (), 0, "Constant output=$x", eval(oinfoglobal))
convert(::Type{S}, f::T where T<:Function) where S<:FunctionCell = FunctionCell(f, (), string(f))

# Convert to AbstractCell (default conversions to Cells-like objects)
convert(::Type{S}, x::T where T<:DataElement) where S<:AbstractCell = DataCell([x], nothing, nothing, 0, "DataCell (convert)", eval(oinfoglobal))
convert(::Type{S}, x::T where T<:AbstractArray) where S<:AbstractCell = DataCell(x, "DataCell (convert)")
convert(::Type{S}, f::T where T<:Function) where S<:AbstractCell = FunctionCell(f, (), string(f))
convert(::Type{S}, x::T where T<:Tuple) where S<:AbstractCell = S(x...)


module ADChain

using ChainRulesCore
using ChainRules
using GeneralizedGenerated
using Flux

export gradient

include("lib.jl")
include("closure_pullback.jl")
include("pass.jl")

end # module


module MittagLeffler

export mittleff
export mittlefferr
export mittleffderiv

include("mittag_leffler.jl")

end # module