module Benchmarkplotting

using BenchmarkTools
using Gadfly
using DataFrames
using Printf
using StatsBase
using Statistics

export bcompare, report

get_names(::NamedTuple{Names, T}) where {Names, T} = Names

function bcompare(criterion :: Function,
                  data :: Vector{Pair{Symbol, T}},
                  implementations :: Vector{Pair{Symbol, Function}};
                  repeat :: Int = 2,
                  quiet :: Bool = false) where T
    rows = []
    for (impl_name, impl) in implementations
        for (case_name, case) in data
            # avoid the evaluation of case(if it's of AST types)
            @inline function action()
                impl(case)
            end
            if !quiet
                @info impl_name case_name action()
            end
            fst_test = criterion(@benchmark $action())
            fields = get_names(fst_test)
            repeats = [criterion(@benchmark $action) for _ in 1:repeat]
            res = Dict([field => fst_test[field] for field in fields])
            for field in fields
                res[field] = (res[field] + sum([it[field] for it in repeats])) ./ (1 + repeat)
            end
            row = (implementation = impl_name,
                   case = case_name,
                   res...)
            push!(rows, row)
        end
    end
    DataFrame(rows)
end

function report(field :: Symbol,
                result :: DataFrame,
                layouts...)
    result = copy(result)
    impls = []
    means = []

    case_names  = unique(result.case)
    case_names_with_unit = Dict()
    for case_name in case_names
        idx = result[:case] .== case_name
        tmp = result[idx, field]
        minval = minimum(tmp)
        tmp = tmp ./ minval
        result[idx, field] = tmp
        gmean = geomean(tmp)
        push!(means, gmean)
        minval_repr = @sprintf "%.3f" minval
        case_names_with_unit[case_name] = Symbol(case_name, "(min: $minval_repr)")
    end

    casemean = DataFrame(case=case_names, geomean=means)
    benchmarks = join(result, casemean, on=:case)

    benchmarks[:, :case] = map(x -> case_names_with_unit[x], benchmarks[:, :case])
    casemean[:, :case] = map(x -> case_names_with_unit[x], casemean[:, :case])

    plot(benchmarks,
        x = :case,
        y = field,
        color=:implementation,
        Guide.ylabel(nothing),
        Guide.xlabel(nothing),
        Theme(
            guide_title_position = :left,
            colorkey_swatch_shape = :circle,
            minor_label_font = "Consolas",
            major_label_font = "Consolas"
         ),
         layouts...,
    ), casemean[:, 1:2]

end

end # module


const ParameterName=Union{Symbol,Expr}

"""
    Grid(goal=nothing, resolution=10, rng=Random.GLOBAL_RNG, shuffle=true)

Instantiate a Cartesian grid-based hyperparameter tuning strategy with
a specified number of grid points as `goal`, or using a specified
default `resolution` in each numeric dimension.

### Supported ranges:

- A single one-dimensional range (`ParamRange` object) `r`, or pair of
  the form `(r, res)` where `res` specifies a resolution to override
  the default `resolution`.

- Any vector of objects of the above form

`ParamRange` objects are constructed using the `range` method.

Example 1:

    range(model, :hyper1, lower=1, origin=2, unit=1)

Example 2:

    [(range(model, :hyper1, lower=1, upper=10), 15),
      range(model, :hyper2, lower=2, upper=4),
      range(model, :hyper3, values=[:ball, :tree]]

Note: All the `field` values of the `ParamRange` objects (`:hyper1`,
`:hyper2`, `:hyper3` in the precedng example) must refer to field
names a of single model (the `model` specified during `TunedModel`
construction).

### Algorithm

This is a standard grid search with the following specifics: In all
cases all `values` of each specified `NominalRange` are exhausted. If
`goal` is specified, then all resolutions are ignored, and a global
resolution is applied to the `NumericRange` objects that maximizes the
number of grid points, subject to the restriction that this not exceed
`goal`. Otherwise the default `resolution` and any parameter-specific
resolutions apply.

In all cases the models generated are shuffled using `rng`, unless
`shuffle=false`.

See also [TunedModel](@ref), [range](@ref).

"""
mutable struct Grid <: TuningStrategy
    goal::Union{Nothing,Int}
    resolution::Int
    shuffle::Bool
    rng::Random.AbstractRNG
end

# Constructor with keywords
Grid(; goal=nothing, resolution=10, shuffle=true,
     rng=Random.GLOBAL_RNG) =
    Grid(goal, resolution, MLJBase.shuffle_and_rng(shuffle, rng)...)

isnumeric(::Any) = false
isnumeric(::NumericRange) = true

adjusted_resolutions(::Nothing,  ranges, resolutions) = resolutions
function adjusted_resolutions(goal, ranges, resolutions)
    # get the product Π of the lengths of the NominalRanges:
    len(::NumericRange) = 1
    len(r::NominalRange) = length(r.values)
    Π = prod(len.(ranges))

    n_numeric = sum(isnumeric.(ranges))

    # compute new uniform resolution:
    goal = goal/Π
    res = round(Int, goal^(1/n_numeric))
    return  map(eachindex(resolutions)) do j
        isnumeric(ranges[j]) ? res : resolutions[j]
    end
end

function setup(tuning::Grid, model, user_range, verbosity)
    ranges, resolutions =
        process_user_range(user_range, tuning.resolution, verbosity)
    resolutions = adjusted_resolutions(tuning.goal, ranges, resolutions)

    fields = map(r -> r.field, ranges)

    parameter_scales = scale.(ranges)

    if tuning.shuffle
        models = grid(tuning.rng, model, ranges, resolutions)
    else
        models = grid(model, ranges, resolutions)
    end

    state = (models=models,
             fields=fields,
             parameter_scales=parameter_scales)

    return state

end

MLJTuning.models!(tuning::Grid, model, history::Nothing,
                  state, verbosity) = state.models
MLJTuning.models!(tuning::Grid, model, history,
                  state, verbosity) =
    state.models[length(history) + 1:end]

function tuning_report(tuning::Grid, history, state)

    plotting = plotting_report(state.fields, state.parameter_scales, history)

    # todo: remove collects?
    return (history=history, plotting=plotting)

end

function default_n(tuning::Grid, user_range)
    ranges, resolutions =
        process_user_range(user_range, tuning.resolution, -1)

    resolutions = adjusted_resolutions(tuning.goal, ranges, resolutions)
    len(t::Tuple{NumericRange,Integer}) = length(iterator(t[1], t[2]))
    len(t::Tuple{NominalRange,Integer}) = t[2]
    return prod(len.(zip(ranges, resolutions)))

end


@testset "fetch.jl" begin
    @testset "_unpack_df" begin
        test_df = DataFrame(
            :a => [1, 2],
            :b => [Dict("s" => 5, "t" => 6), Dict("s" => 2, "t" => 7)]
        )

        expected = DataFrame(
            :a => [1, 2],
            :s => [5, 2],
            :t => [6, 7]
        )

        _test_df_inequality(_unpack_df(test_df, :b), expected)
    end

    @testset "_unpack_values" begin
        @testset "dicts" begin
            ex = Dict("s" => 5, "t" => 6)

            @test issetequal(_unpack_values(ex), [5, 6])
        end
        @testset "arrays" begin
            ex1 = Dict("s" => 5, "t" => 6)
            ex2 = Dict("s" => 2, "t" => 7)
            @test _unpack_values([ex1, ex2]) == [6 7; 5 2]
        end
    end

    @testset "_parse_datetime" begin
        dt = "2021-10-01T01:01Z"
        @test _parse_datetime(dt) == DateTime(2021, 10, 1, 1, 1)
    end

    @testset "_parse_data" begin
        @testset "dataframe dispatch" begin
            ex = DataFrame(
                :from => ["2021-10-01T01:01Z", "2021-05-01T01:01Z"],
                :to => ["2021-10-02T01:01Z", "2021-05-02T01:01Z"]
            )

            expected =  DataFrame(
                :from => [DateTime(2021, 10, 1, 1, 1), DateTime(2021, 05, 1, 1, 1)],
                :to => [DateTime(2021, 10, 2, 1, 1), DateTime(2021, 05, 2, 1, 1)]
            )
            @test _parse_data(ex) == expected
        end

        @testset "array dispatch" begin
            ex = DataFrame(
                :from => ["2021-10-01T01:01Z", "2021-05-01T01:01Z"],
                :to => ["2021-10-02T01:01Z", "2021-05-02T01:01Z"]
            )

            expected = DataFrame(
                :from => [
                    DateTime(2021, 10, 1, 1, 1),
                    DateTime(2021, 05, 1, 1, 1),
                    DateTime(2021, 10, 1, 1, 1),
                    DateTime(2021, 05, 1, 1, 1),
                ],
                :to => [
                    DateTime(2021, 10, 2, 1, 1),
                    DateTime(2021, 05, 2, 1, 1),
                    DateTime(2021, 10, 2, 1, 1),
                    DateTime(2021, 05, 2, 1, 1),
                ]
            )

            data = [ex, ex]
            @test _parse_data(data) == expected
        end

        @testset "convert_mix" begin
            ex1 = Dict("fuel" => "hydro", "perc" => 0.5)
            ex2 = Dict("fuel" => "gas", "perc" => 0.1)
            data = [ex1, ex2]

            expected = Dict("hydro"=> 0.5, "gas"=> 0.1)
            @test _convert_mix(data) == expected
        end
    end

    @testset "get carbon intensity" begin
        @testset "short date range" begin
            start_date = ZonedDateTime(2021, 1, 1, tz"UTC")
            end_date = ZonedDateTime(2021, 1, 10, tz"UTC")

            data = get_carbon_intensity(start_date, end_date)

            @test maximum(data.to) <= end_date.utc_datetime
            @test minimum(data.to) >= start_date.utc_datetime

            @test data isa DataFrame
            @test !isempty(data)
            @test issetequal(names(data), ["to", "from", "forecast", "actual", "index"])
        end

        @testset "longer daterange" begin
            start_date = ZonedDateTime(2021, 1, 1, tz"UTC")
            end_date = ZonedDateTime(2021, 2, 1, tz"UTC")

            data = get_carbon_intensity(start_date, end_date)

            @test maximum(data.to) <= end_date.utc_datetime
            @test minimum(data.to) >= start_date.utc_datetime

            @test data isa DataFrame
            @test !isempty(data)
            @test issetequal(names(data), ["to", "from", "forecast", "actual", "index"])
        end
    end

    @testset "get regional data" begin
        @testset "shorter daterange" begin
            start_date = ZonedDateTime(2021, 1, 1, tz"UTC")
            end_date = ZonedDateTime(2021, 1, 10, tz"UTC")

            data = get_regional_data(start_date, end_date)

            @test data isa NamedTuple
            @test issetequal(propertynames(data), [:intensity, :generation])

            intensity = data.intensity
            generation = data.generation

            @test maximum(intensity.to) <= end_date.utc_datetime
            @test minimum(intensity.to) >= start_date.utc_datetime

            @test maximum(generation.to) <= end_date.utc_datetime
            @test minimum(generation.to) >= start_date.utc_datetime

            @test intensity isa DataFrame
            @test !isempty(intensity)
            @test issetequal(
                names(intensity),
                [
                    "to", "from", "forecast", "shortname", "dnoregion",
                    "regionid", "generationmix", "index"
                ]
            )

            @test generation isa DataFrame
            @test !isempty(generation)
            @test issetequal(
                names(generation),
                [
                    "to", "from", "intensity", "shortname", "dnoregion",
                    "regionid", "coal", "hydro", "biomass", "imports",
                    "nuclear", "other", "solar", "gas", "wind"
                ]
            )
        end

        @testset "longer daterange" begin
            start_date = ZonedDateTime(2021, 1, 1, tz"UTC")
            end_date = ZonedDateTime(2021, 2, 1, tz"UTC")

            data = get_regional_data(start_date, end_date)

            @test data isa NamedTuple
            @test issetequal(propertynames(data), [:intensity, :generation])

            intensity = data.intensity
            generation = data.generation

            @test maximum(intensity.to) <= end_date.utc_datetime
            @test minimum(intensity.to) >= start_date.utc_datetime

            @test maximum(generation.to) <= end_date.utc_datetime
            @test minimum(generation.to) >= start_date.utc_datetime

            @test intensity isa DataFrame
            @test !isempty(intensity)
            @test  issetequal(
                names(intensity),
                [
                    "to", "from", "forecast", "shortname", "dnoregion",
                    "regionid", "generationmix", "index"
                ]
            )

            @test generation isa DataFrame
            @test !isempty(generation)
            @test issetequal(
                names(generation),
                [
                    "to", "from", "intensity", "shortname", "dnoregion",
                    "regionid",  "coal", "hydro", "biomass", "imports",
                    "nuclear", "other", "solar", "gas", "wind"
                ]
            )
        end
    end

    @testset "get todays forecast" begin
        @testset "default" begin
            data = get_todays_forecast()
            @test data isa DataFrame
            @test !isempty(data)
            @test issetequal(names(data), ["to", "from", "forecast", "actual", "index"])
        end

        @testset "region kwarg" begin
            data = get_todays_forecast(regional=true)

            @test data isa DataFrame
            @test !isempty(data)
            @test issetequal(
                names(data),
                [
                    "to", "from", "forecast", "shortname", "dnoregion",
                    "regionid", "generationmix", "index"
                ]
            )
        end

        @testset "specific region" begin
            data = get_todays_forecast(regional=true, region = "North Scotland")

            @test data isa DataFrame
            @test all(data.shortname.=="North Scotland")
            @test issetequal(
                names(data),
                [
                    "to", "from", "forecast", "shortname", "dnoregion",
                    "regionid", "generationmix", "index"
                ]
            )
        end

        @testset "throw error on bad region" begin
            @test_throws DomainError get_todays_forecast(regional=true, region = "not a region")
        end
    end
end


#=
Copyright (c) 2018-2022 Chris Coey, Lea Kapelevich, and contributors

This Julia package Hypatia.jl is released under the MIT license; see LICENSE
file in the root directory or at https://github.com/chriscoey/Hypatia.jl
=#

insts = OrderedDict()
insts["minimal"] = [
    ((5, 2, 2, true, true, true),),
    ((5, 1, 2, false, true, true),),
    ((5, 1, 2, false, true, false),),
    ((5, 1, 2, true, false, false), :ExpPSD),
    ((5, 1, 2, true, false, false), :SOCExpPSD),
    ((:iris, 2, true, false, true),),
    ]
insts["fast"] = [
    ((10, 1, 10, true, false, false), :ExpPSD),
    ((10, 1, 10, true, false, false), :SOCExpPSD),
    ((100, 1, 250, true, true, true),),
    ((100, 2, 5, false, true, true),),
    ((200, 2, 20, true, true, true),),
    ((50, 3, 2, true, true, true),),
    ((50, 3, 2, false, false, true),),
    ((50, 3, 2, true, false, true), :ExpPSD),
    ((50, 3, 2, true, false, true), :SOCExpPSD),
    ((50, 3, 4, true, true, true),),
    ((500, 3, 14, true, true, true),),
    ((100, 8, 2, true, true, true),),
    ((250, 4, 4, true, true, true),),
    ((250, 4, 4, false, false, true),),
    ((200, 32, 2, true, false, true),),
    ((:iris, 4, false, false, false),),
    ((:iris, 5, true, false, false),),
    ((:iris, 6, true, true, true),),
    ((:cancer, 4, true, true, true),),
    ]
insts["various"] = [
    ((50, 2, 16, true, false, false),),
    ((50, 2, 16, true, false, false), :ExpPSD),
    ((50, 2, 16, true, false, false), :SOCExpPSD),
    ((50, 2, 16, false, false, false),),
    ((50, 2, 16, false, false, true),),
    ((50, 8, 4, true, false, false),),
    ((50, 8, 4, true, false, false), :ExpPSD),
    ((50, 8, 4, true, false, false), :SOCExpPSD),
    ((50, 8, 4, false, false, false),),
    ((50, 8, 4, false, false, true),),
    ((50, 32, 2, true, false, false),),
    ((50, 32, 2, true, false, false), :ExpPSD),
    ((50, 32, 2, true, false, false), :SOCExpPSD),
    ((:iris, 6, true, true, true),),
    ((:cancer, 4, true, true, true),),
    ]
return (DensityEstJuMP, insts)


export iauPv2p
"""
Discard velocity component of a pv-vector.

This function is part of the International Astronomical Union's
SOFA (Standards Of Fundamental Astronomy) software collection.

Status:  vector/matrix support function.

Given:
   pv      double[2][3]     pv-vector

Returned:
   p       double[3]        p-vector

Called:
   iauCp        copy p-vector

This revision:  2013 June 18

SOFA release 2018-01-30

Copyright (C) 2018 IAU SOFA Board.  See notes at end.
"""

# void iauPv2p(double pv[2][3], double p[3])

function iauPv2p(pv::AbstractMatrix{<:Real})
   # Preallocate return values
   p = zeros(Float64, 3)

   ccall((:iauPv2p, libsofa_c), Cvoid,
         (Ptr{Cdouble}, Ptr{Cdouble}),
          convert(Array{Float64, 2}, pv'), p)

   return SVector{3}(p)
end


@kwdef mutable struct DMultHandler <: AbstractHandler
    id = 0
end

@register(DMultHandler, *, [DBit, DBit] => [Number])

function (handler::DMultHandler)(netlist::Netlist,
                                 inputs::Vector{Variable},
                                 outputs::Vector{Variable})
    # check size
    if getsize(netlist, getname(inputs[1])) != (1, 1) && getsize(netlist, getname(inputs[1])) != (1, 1)
        error("Deterministic multiply does not support matrix hardware generation yet.")
    end

    outstring = """
        $stdcomment
        // BEGIN d_mult$(handler.id)
        determ_mult d_mult$(handler.id) (
                .a($(inputs[1].name)),
                .b($(inputs[2].name)),
                .y($(outputs[1].name))
            );
        // END d_mult$(handler.id)
        \n"""

    handler.id += 1

    return outstring
end

@kwdef mutable struct DMultFxpHandler <: AbstractHandler
    id = 0
end

@register(DMultFxpHandler, *, begin
    [DBit, Number] => [Number]
    [Number, DBit] => [Number]
end)

function (handler::DMultFxpHandler)(netlist::Netlist,
                                    inputs::Vector{Variable},
                                    outputs::Vector{Variable})
    # check size
    if getsize(netlist, getname(inputs[1])) != (1, 1) && getsize(netlist, getname(inputs[1])) != (1, 1)
        error("Deterministic multiply does not support matrix hardware generation yet.")
    end

    outstring = """
        $stdcomment
        // BEGIN d_mult_fxp$(handler.id)
        determ_mult_fxp #(
                .BIT_WIDTH(BIT_WIDTH)
            ) d_mult_fxp$(handler.id) (
                .a($(inputs[1].name)),
                .b($(inputs[2].name)),
                .y($(outputs[1].name))
            );
        // END d_mult_fxp$(handler.id)
        \n"""

    handler.id += 1

    return outstring
end


import Base: reducedim, sum, maximum
export argmax, argmin

function reshape3d(x::CuArray, dim::Int)
    # dim == 0 && return (1, length(x), 1)
    dim1, dim2, dim3 = 1, size(x,dim), 1
    for i = 1:dim-1
        dim1 *= size(x, i)
    end
    for i = dim+1:ndims(x)
        dim3 *= size(x, i)
    end
    reshape(x, dim1, dim2, dim3)
end

function reducedim_f(T, v0, op)
    CuFunction("""
    template<typename T>
    __inline__ __device__ void warpReduce(T &a) {
        for (int delta = warpSize/2; delta > 0; delta /= 2) {
            T b = __shfl_down(a, delta);
            a = $op;
        }
    }

    template<typename T>
    __inline__ __device__ void blockReduce(T &val) {
        static __shared__ T shared[32]; // 32*32=1024
        int laneId = threadIdx.x % warpSize;
        int warpId = threadIdx.x / warpSize;
        warpReduce<T>(val);
        if (laneId == 0) shared[warpId] = val;
        __syncthreads();

        val = shared[laneId];
        if (warpId == 0) warpReduce<T>(val);
    }

    __global__ void reduce($T *x, $T *y, int size1, int size2) {
        int idxX = blockIdx.x * blockDim.x + threadIdx.x;
        int idxY = blockIdx.y * blockDim.y + threadIdx.y;
        if (idxX >= size1) return;
        if (idxY >= size2) return;

        $T val = x[idxX + idxY * size2];
        blockReduce<$T>(val);
        if (idxX == 0) y[idxY] = val;
        //if (threadIdx.x == 0) y[blockIdx.x] = val;
    }
    """)
end

@generated function sum2(x::CuArray{T}, dim::Int) where T
    f = reducedim_f(cstring(T), "0", "a + b")
    quote
        #x3d = reshape3d(x, dim)
        #xsize = [size(x)...]
        #xsize[dim] = 1
        #y = similar(x, xsize...)

        y = similar(x, 1, size(x,2))
        #n = size(x,2)
        bx = 1024
        #gx = n <= bx ? 1 : ceil(Int, n/bx)
        gdims, bdims = (1,size(x,2),1), (bx,1,1)
        # gdims, bdims = cudims(length(x))
        culaunch($f, gdims, bdims, x.ptr, y.ptr, size(x,1), size(x,2))
        y
    end
end

function find_f{T}(::Type{T}, v0::String, op::String)
    CuFunction("""
    template<typename T>
    __inline__ __device__ void warpReduce(T &a, int &i) {
        for (int delta = warpSize/2; delta > 0; delta /= 2) {
            T b = __shfl_down(a, delta);
            int j = __shfl_down(i, delta);
            if ($op) {
                a = b;
                i = j;
            }
        }
    }
    template<typename T>
    __inline__ __device__ void blockReduce(T &value, int &index) {
        static __shared__ T values[32]; // 32*32=1024
        static __shared__ int indexs[32];
        int laneId = threadIdx.y % warpSize;
        int warpId = threadIdx.y / warpSize;
        warpReduce<T>(value, index);
        if (laneId == 0) {
            values[warpId] = value;
            indexs[warpId] = index;
        }
        __syncthreads();
        value = (laneId < 32) ? values[laneId] : $v0;
        index = (laneId < 32) ? indexs[laneId] : -1;
        warpReduce<T>(value, index);
    }
    __global__ void f(Array<$T,3> x, Array<int,3> y) {
        int idx_x = blockIdx.x * blockDim.x + threadIdx.x;
        int idx_y = blockIdx.y * blockDim.y + threadIdx.y;
        int idx_z = blockIdx.z * blockDim.z + threadIdx.z;
        $T value = (idx_y < x.dims[1]) ? x(idx_x,idx_y,idx_z) : $v0;
        int index = (idx_y < x.dims[1]) ? blockIdx.y * blockDim.y + threadIdx.y : -1;
        //for (int i = idx_y+blockDim.y*gridDim.y; i < x.dims[1]; i += blockDim.y*gridDim.y) {
        //    $T b = x(idx_x, i, idx_z);
        //    a = $op;
        //}
        blockReduce<$T>(value, index);
        if (threadIdx.y == 0) y(blockIdx.x, blockIdx.y, blockIdx.z) = index + 1;
    }
    """)
end

function reducedim_f2(T, op::String)
    blocksize = 1024
    CuFunction("""
    __global__ void reduce($T *x, $T *y) {
        static __shared__ $T temp[1024];
        int idx_x = blockIdx.x * blockDim.x + threadIdx.x;
        //int idx_y = blockIdx.y * blockDim.y + threadIdx.y;
        int idx_z = blockIdx.z * blockDim.z + threadIdx.z;
        //if (idx_y >= x.dims[1]) return;
        int tid = threadIdx.y;
        int idx_y = blockIdx.y * $blocksize * 2 + threadIdx.y;
        $T a = (idx_y < x.dims[1]) ? x(idx_x, idx_y, idx_z) : 0;

        if (idx_y+$blocksize < x.dims[1]) {
            a += x[idx_y+$blocksize];
        }
        temp[tid] = a;
        $T b;
        __syncthreads();

        if (($blocksize >= 512) && (tid < 256)) {
            b = temp[tid+256];
            a = $op;
            temp[tid] = a;
            __syncthreads();
        }
        if (($blocksize >= 256) &&(tid < 128)) {
            b = temp[tid+128];
            a = $op;
            temp[tid] = a;
            __syncthreads();
        }
        if (($blocksize >= 128) && (tid < 64)) {
            b = temp[tid+64];
            a = $op;
            temp[tid] = a;
            __syncthreads();
        }
        if (tid < 32) {
            if ($blocksize >= 64) a += temp[tid+32];
            for (int offset = warpSize/2; offset > 0; offset /= 2) {
                a += __shfl_down(a, offset);
            }
        }
        if (threadIdx.y == 0) y(blockIdx.x, blockIdx.y, blockIdx.z) = a;
    }""")
end


using Test

m = page_model()
include("../src/components/N2Ocycle.jl")

add_comp!(m, n2ocycle)

set_param!(m, :n2ocycle, :e_globalN2Oemissions, readpagedata(m,"test/validationdata/e_globalN2Oemissions.csv"))
set_param!(m, :n2ocycle, :y_year, [2009.,2010.,2020.,2030.,2040.,2050.,2075.,2100.,2150.,2200.]) #real value
set_param!(m, :n2ocycle, :y_year_0, 2008.) #real value
set_param!(m, :n2ocycle, :rtl_g_landtemperature, readpagedata(m,"test/validationdata/rtl_g_landtemperature.csv"))

##running Model
run(m)

conc=m[:n2ocycle,  :c_N2Oconcentration]
conc_compare=readpagedata(m,"test/validationdata/c_n2oconcentration.csv")

@test conc ≈ conc_compare rtol=1e-4


using Base.Iterators: product
using Dates
using Documenter: doctest
using Infinity: Infinite, InfExtendedReal, InfExtendedTime, InfMinusInfError, ∞
using Intervals
using Intervals: isfinite
using Serialization: deserialize
using Test
using TimeZones

const BOUND_PERMUTATIONS = product((Closed, Open), (Closed, Open))

include("test_utils.jl")

@testset "Intervals" begin
    include("inclusivity.jl")
    include("endpoint.jl")
    include("interval.jl")
    include("anchoredinterval.jl")
    include("comparisons.jl")
    include("plotting.jl")

    # Note: The output of the doctests currently requires a newer version of Julia
    # https://github.com/JuliaLang/julia/pull/34387
    # The doctests fail on x86, so only run them on 64-bit hardware
    if v"1.6" <= VERSION < v"1.7" && Sys.WORD_SIZE == 64
        doctest(Intervals)
    else
        @warn "Skipping doctests"
    end
end


module BLP

import LinearAlgebra as LA
using LinearAlgebra: LowerTriangular, ⋅, mul!

include("iteration.jl")
include("softmax.jl")
include("market.jl")

end


include("./produce_data.jl")
using NPZ
using HDF5

function main(;
    )
    # create data
    data = produce_data_avalanches_sandpile_illustration()
    npzwrite(@sprintf("%s/data/data_avalanche_sandpile_illustration.npz", path_project),data)
end

# call function
main()


using Test

@testset "all" begin
    include("Prerequisites/Prerequisites.jl")
    include("Mathematics/Mathematics.jl")
    include("Essentials/Essentials.jl")
end


module TriMatrices

using LinearAlgebra
using DocStringExtensions

export TriUpper, TriLower, TriSymmetric, TriMatrix

include("layout.jl")

include("Indexing/Indexing.jl")
using .Indexing

include("trimatrix.jl")
include("linearalgebra.jl")
include("Testing.jl")

end # module


## ExoplanetsSysSim/examples/run_abc.jl
## (c) 2015 Eric B. Ford

include("abc_setup.jl")

using SysSimABC

abc_plan = setup_abc(3)
@time output = SysSimABC.run_abc(abc_plan)

#@time output2 = SysSimABC.run_abc(abc_plan,output)
#@time output3 = SysSimABC.run_abc(abc_plan,output2)


"""
Save file .JSON of the boundingbox in path.
"""
function save_AABB2json(folder::String, aabb::AABB)
	@assert isdir(folder) "save_AABB2json: $path not a valid directory"
	name = splitdir(folder)[2]
	filename = name*".json"
	scale = DataStructures.OrderedDict{String,Any}("x"=>aabb.x_max-aabb.x_min, "y"=>aabb.y_max-aabb.y_min, "z"=>aabb.z_max-aabb.z_min)
	position = DataStructures.OrderedDict{String,Any}("x"=>(aabb.x_max+aabb.x_min)/2, "y"=>(aabb.y_max+aabb.y_min)/2, "z"=>(aabb.z_max+aabb.z_min)/2)
	rotation = DataStructures.OrderedDict{String,Any}("x"=>0., "y"=>0., "z"=>0.)
	data = DataStructures.OrderedDict{String,Any}("clip"=>true, "name"=>name,
			"scale"=>scale,"position"=>position,"rotation"=>rotation,
			"permitExtraction"=>true)
	open(joinpath(folder,filename),"w") do f
  		JSON.print(f, data,4)
	end
end

# """
# {
#    "object":"plane",
#    "normal":{
# 	  "x":0.000,
# 	  "y":0.000,
# 	  "z":1.000
#    },
#    "position":{
#       "x":0.0,
#       "y":0.0,
#       "z":0.0
#    },
# }
# """
# function plane2json(plane::Plane, filename::String)
# 	pos = plane.centroid
# 	dir = plane.normal
# 	position = DataStructures.OrderedDict{String,Any}("x"=>pos[1], "y"=>pos[2], "z"=>pos[3])
# 	normal = DataStructures.OrderedDict{String,Any}("x"=>dir[1], "y"=>dir[2], "z"=>dir[3])
# 	data = DataStructures.OrderedDict{String,Any}("object"=> "plane", "position"=>position,"normal"=>normal)
# 	open(filename,"w") do f
#   		JSON.print(f, data,4)
# 	end
# end

# """
# ============================================  CAMERA
# """
# function cameraparameters(path::String)
# 	dict=Dict{String,Any}[]
# 	open(path, "r") do f
# 	    dict = JSON.parse(f)  # parse and transform data
# 	end
# 	position = dict["position"]
# 	target = dict["target"]
# 	return position, target
# end

#function viewcoordinatesystem(camera)
#     position,target = camera
# 	up = [0,0,1.]
# 	dir = target-position
# 	x = -dir/Lar.norm(dir)
# 	if x != [0,0,1] && x != [0,0,-1]
# 		y = Lar.cross(x,up)
# 		z = Lar.cross(y,x)
# 	end
#     return [y';z';x']
# end
#
#
# function coordsystemcamera(file::String)
#     mat = PointClouds.cameramatrix(file)
#     return Matrix(mat[1:3,1:3]')
# end

# """
# camera parameters from JSON.
# """
# function cameramatrix(path::String)
# 	dict = Dict{String,Any}[]
# 	open(path, "r") do f
# 	    dict = JSON.parse(f)  # parse and transform data
# 	end
# 	mat = dict["object"]["matrix"]
# 	return [mat[1] mat[5] mat[9] mat[13];
# 			mat[2] mat[6] mat[10] mat[14];
# 			mat[3] mat[7] mat[11] mat[15];
# 			mat[4] mat[8] mat[12] mat[16]]
# end
#

# ====================================  read points for polygon
# """
# extract verteces from area tools.
# """
# function vertspolygonfromareaJSON(file::String)
# 	dict = Dict{String,Any}[]
# 	open(file, "r") do f
# 	    dict = JSON.parse(f)  # parse and transform data
# 	end
# 	features = dict["features"]
# 	for feature in features
# 		type = feature["geometry"]["type"]
# 		if type == "Polygon"
# 			points = feature["geometry"]["coordinates"]
# 			V = hcat(points[1][1:end-1]...)
# 			return V
# 		end
# 	end
# end
#
#
# """
# create polygon model.
# """
# function polygon(file::String)
# 	verts = vertspolygonfromareaJSON(file)
# 	EV = [[i,i+1] for i in 1:size(verts,2)-1]
# 	push!(EV,[size(verts,2),1])
# 	axis,centroid = PointClouds.planefit(verts)
# 	if Lar.dot(axis,Lar.cross(verts[:,1]-centroid,verts[:,2]-centroid))<0
# 		axis = -axis
# 	end
# 	PointClouds.projectpointson(verts,(axis,centroid),"plane")
# 	return verts,EV
# end


"""
    leftlane(roadway::Roadway, veh::Entity)
returns the lane to the left of the lane veh is currently in, returns nothing if it does not exists
"""
function leftlane(roadway::Roadway, veh::Entity)
    return leftlane(roadway, get_lane(roadway, veh))
end

"""
    rightlane(roadway::Roadway, veh::Entity)
returns the lane to the right of the lane veh is currently in, returns nothing if it does not exists
"""
function rightlane(roadway::Roadway, veh::Entity)
    return rightlane(roadway, get_lane(roadway, veh))
end

"""
    n_lanes_left(roadway::Roadway, veh::Entity)
return the numbers of lanes to the left of `veh`
"""
n_lanes_left(roadway::Roadway, veh::Entity) = n_lanes_left(roadway, get_lane(roadway, veh))

"""
    n_lanes_right(roadway::Roadway, veh::Entity)
return the numbers of lanes to the right of `veh`
"""
n_lanes_right(roadway::Roadway, veh::Entity) = n_lanes_right(roadway, get_lane(roadway, veh))

"""
    lane_width(roadway::Roadway, veh::Entity)
Returns the width of the lane where `veh` is.
"""
function lane_width(roadway::Roadway, veh::Entity)

    lane = get_lane(roadway, veh.state)

    if n_lanes_left(roadway, veh) > 0
        footpoint = get_footpoint(veh)
        lane_left = roadway[LaneTag(lane.tag.segment, lane.tag.lane + 1)]
        return -proj(footpoint, lane_left, roadway).curveproj.t
    else
        return lane.width
    end
end

"""
    markerdist_left(roadway::Roadway, veh::Entity)
distance of `veh` to the left marker of the lane
"""
function markerdist_left(roadway::Roadway, veh::Entity)
    t = posf(veh.state).t
    lw = lane_width(roadway, veh)
    return lw/2 - t
end

"""
    markerdist_left(roadway::Roadway, veh::Entity)
distance of `veh` to the right marker of the lane
"""
function markerdist_right(roadway::Roadway, veh::Entity)
    t = posf(veh.state).t
    lw = lane_width(roadway, veh)
    return lw/2 + t
end

"""
    road_edge_dist_left(roadway::Roadway, veh::Entity)
feature function, extract the lateral distance to the left edge of the road
"""
function road_edge_dist_left(roadway::Roadway, veh::Entity)
    offset = posf(veh).t
    footpoint = get_footpoint(veh)
    lane = get_lane(roadway, veh)
    roadproj = proj(footpoint, lane, roadway)
    curvept = roadway[RoadIndex(roadproj)]
    lane = roadway[roadproj.tag]
    return lane.width/2 + norm(VecE2(curvept.pos - footpoint)) - offset
end

"""
    road_edge_dist_right(roadway::Roadway, veh::Entity)
feature function, extract the lateral distance to the right edge of the road
"""
function road_edge_dist_right(roadway::Roadway, veh::Entity)
    offset = posf(veh).t
    footpoint = get_footpoint(veh)
    lane = get_lane(roadway, veh)
    roadproj = proj(footpoint, lane, roadway)
    curvept = roadway[RoadIndex(roadproj)]
    lane = roadway[roadproj.tag]
    return lane.width/2 + norm(VecE2(curvept.pos - footpoint)) + offset
end

"""
    lane_offset_left(roadway::Roadway, veh::Entity)
returns the distance to the left lane border, if there are no left lane returns `missing`
"""
function lane_offset_left(roadway::Roadway, veh::Entity)
    t = posf(veh).t
    lane = get_lane(roadway, veh)
    if n_lanes_left(roadway, lane) > 0
        lane_offset = t - lane.width/2 - leftlane(roadway, veh).width/2
        return lane_offset
    else
        missing
    end
end

"""
    lane_offset_right(roadway::Roadway, veh::Entity)
returns the distance to the right lane border, if there are no right lane returns `missing`
"""
function lane_offset_right(roadway::Roadway, veh::Entity)
    t = posf(veh).t
    lane = get_lane(roadway, veh)
    if n_lanes_right(roadway, lane) > 0
        lane_offset = t + lane.width/2 + rightlane(roadway, veh).width/2
        return lane_offset
    else
        missing
    end
end

"""
    has_lane_right(roadway::Roadway, veh::Entity)
Return true if `veh` has a lane on its right.
"""
function has_lane_right(roadway::Roadway, veh::Entity)
    return n_lanes_right(roadway, veh) > 0
end

"""
    has_lane_left(roadway::Roadway, veh::Entity)
Return true if `veh` has a lane on its left.
"""
function has_lane_left(roadway::Roadway, veh::Entity)
    return n_lanes_left(roadway, veh) > 0
end

"""
    lane_curvature(roadway::Roadway, veh::Entity)
Return the curvature of the lane at `veh`'s position.
Return missing if the curvature is `NaN`
"""
function lane_curvature(roadway::Roadway, veh::Entity)
    curvept = roadway[posf(veh.state).roadind]
    val = curvept.k
    if isnan(val)
        return missing
    end
    return val
end


export Variable

mutable struct Variable end

Base.show(io::IO, x::Variable) = print(io, "Variable(#=", objectid(x), "=#)")

Base.convert(::Type{Expr}, x::Variable) = x

vars(x::Variable) = Set([x])
matcher(x::Variable, V) = (x, push!(copy(V), x))
@inline function match!(σ, x::Variable, t::AbstractTerm)
    if haskey(σ, x)
        σ[x] == t || return nothing
    else
        σ[x] = t
    end
    return EmptySubproblem()
end
replace(x::Variable, σ) = σ[x]

>ₜ(::Variable, ::AbstractTerm) = false
>ₜ(::AbstractTerm, ::Variable) = true
>ₜ(x::Variable, y::Variable) = objectid(x) > objectid(y)

function compile(x::Variable, V)
    fn_name = gensym(:match!_var)
    empty = EmptySubproblem()
    body = if x ∈ V
        :(σ[$x] == t ? $empty : nothing)
    else
        quote
            if haskey(σ, $x)
                σ[$x] == t ? $empty : nothing
            else
                σ[$x] = t
                $empty
            end
        end
    end

    fn_name, Expr(:block, Expr(:function, :($fn_name(σ, t)), body))
end


@enum Active Initial Final

mutable struct ActiveAggregation
    active::Active

    function ActiveAggregation()
        return new(Initial)
    end
end

function shift(gap::AbstractFloat, τ::AbstractFloat)
    return gap <= τ
end

"""
    HybridAggregation

Functor object for using hybrid aggregation in an L-shaped algorithm. Create by supplying a [`HybridAggregate`](@ref) object through `aggregate ` in the `LShapedSolver` factory function and then pass to a `StochasticPrograms.jl` model.

...
# Parameters
- `initial::AbstractAggregator`: Initial aggregation scheme
- `final::AbstractAggregator`: Final aggregation scheme
- `τ::T`: The active aggregation scheme is switched from `initial` to `final` when the optimality gap decreases below `τ`
...
"""
struct HybridAggregation{T <: AbstractFloat, Agg1 <: AbstractAggregation, Agg2 <: AbstractAggregation} <: AbstractAggregation
    initial::Agg1
    final::Agg2
    τ::T
    active::ActiveAggregation

    function HybridAggregation(initial::AbstractAggregation, final::AbstractAggregation, τ::AbstractFloat)
        Agg1 = typeof(initial)
        Agg2 = typeof(final)
        T = typeof(τ)
        return new{T,Agg1,Agg2}(initial, final, τ, ActiveAggregation())
    end
end

function active(aggregation::HybridAggregation)
    if aggregation.active.active == Initial
        return aggregation.initial
    end
    return aggregation.final
end

function activate_final!(aggregation::HybridAggregation)
    aggregation.active.active = Final
    return nothing
end

function aggregate_cut!(lshaped::AbstractLShaped, aggregation::HybridAggregation, cut::HyperPlane)
    return aggregate_cut!(lshaped, active(aggregation), cut)
end

function aggregate_cut!(cutqueue::CutQueue, aggregation::HybridAggregation, metadata::MetaData, t::Integer, cut::HyperPlane, x::AbstractArray)
    return aggregate_cut!(cutqueue, active(aggregation), metadata, t, cut, x)
end

function num_thetas(num_subproblems::Integer, aggregation::HybridAggregation)
    return num_thetas(num_subproblems, active(aggregation))
end

function num_thetas(num_subproblems::Integer, aggregation::HybridAggregation, sp::DistributedScenarioProblems)
    return num_thetas(num_subproblems, active(aggregation), sp)
end

function flush!(lshaped::AbstractLShaped, aggregation::HybridAggregation)
    added = flush!(lshaped, active(aggregation))
    if shift(gap(lshaped), aggregation.τ)
        activate_final!(aggregation)
    end
    return added
end

function flush!(cutqueue::CutQueue, aggregation::HybridAggregation, metadata::MetaData, t::Integer, x::AbstractArray) where T <: AbstractFloat
    flush!(cutqueue, active(aggregation), metadata, t, x)
    if shift(fetch(metadata, t, :gap), aggregation.τ)
        activate_final!(aggregation)
    end
    return nothing
end

# API
# ------------------------------------------------------------
"""
    HybridAggregate(initial::AbstractAggregator, final::AbstractAggregator, τ::AbstractFloat)

Factory object for [`HybridAggregation`](@ref). Pass to `aggregate ` in the `LShapedSolver` factory function. See ?HybridAggregation for parameter descriptions.

"""
mutable struct HybridAggregate <: AbstractAggregator
    initial::AbstractAggregator
    final::AbstractAggregator
    τ::Float64

    function HybridAggregate(initial::AbstractAggregator, final::AbstractAggregator, τ::AbstractFloat)
        return new(initial, final, τ)
    end
end

struct InitialAggregator <: AggregationParameter end

function MOI.get(aggregator::HybridAggregate, ::InitialAggregator)
    return aggregator.initial
end

function MOI.set(aggregator::HybridAggregate, ::InitialAggregator, initial::AbstractAggregator)
    return aggregator.initial = initial
end

struct FinalAggregator <: AggregationParameter end

function MOI.get(aggregator::HybridAggregate, ::FinalAggregator)
    return aggregator.final
end

function MOI.set(aggregator::HybridAggregate, ::FinalAggregator, final::AbstractAggregator)
    return aggregator.final = final
end

function (aggregator::HybridAggregate)(num_subproblems::Integer, T::Type{<:AbstractFloat})
    initial = aggregator.initial(num_subproblems, T)
    final = aggregator.final(num_subproblems, T)
    n₁ = num_thetas(num_subproblems, initial)
    n₂ = num_thetas(num_subproblems, final)
    n₁ == n₂|| error("Inconsistent number of theta variables in hybrid aggregation: $n₁ ≠ $n₂")
    return HybridAggregation(initial, final, convert(T, aggregator.τ))
end

function remote_aggregator(aggregation::HybridAggregation, sp::AbstractScenarioProblems, w::Integer)
    return HybridAggregate(remote_aggregator(aggregation.initial, sp, w),
                           remote_aggregator(aggregation.final, sp, w),
                           aggregation.τ)
end

function str(aggregator::HybridAggregate)
    return "hybrid aggregation consisting of $(str(aggregator.initial)) and $(str(aggregator.final))"
end


# Tag model and embed data
#
export Model2Fit_TaggedModel
export get_tagged_data, get_tagged_model
export get_tagged_data_type, get_tagged_model_type

@doc raw"""
```julia
Model2Fit_TaggedModel(model,data)
```

Tag model and embed data

# Extra method

- [`get_tagged_data`](@ref)
- [`get_tagged_model`](@ref)
- [`get_tagged_data_type`](@ref)
"""
struct Model2Fit_TaggedModel{MODEL<:Abstract_Model2Fit,DATA} <: Abstract_Model2Fit
    _model::MODEL
    _data::DATA
end

# Extra methods  ================
#
@doc raw"""
```julia
get_tagged_data(m::Model2Fit_TaggedModel{MODEL,DATA)::DATA
```

Return embedded data
"""
get_tagged_data(m::Model2Fit_TaggedModel) = m._data
@doc raw"""
```julia
get_tagged_model(m::Model2Fit_TaggedModel{MODEL,DATA)::DATA
```

Return tagged model
"""
get_tagged_model(m::Model2Fit_TaggedModel) = m._model

@doc raw"""
```julia
get_tagged_data_type(m::Abstract_Model2Fit)::DataType
```

Return embedded data type, or `Nothing` if m is not a
[`Model2Fit_TaggedModel`](@ref).
"""
get_tagged_data_type(m::Abstract_Model2Fit) = Nothing
get_tagged_data_type(m::Model2Fit_TaggedModel{MODEL,DATA}) where {MODEL<:Abstract_Model2Fit,DATA} = DATA

@doc raw"""
```julia
get_tagged_model_type(m::Abstract_Model2Fit)::DataType
```

Return wrapped model type, or `Nothing` if m is not a
[`Model2Fit_TaggedModel`](@ref).
"""
get_tagged_model_type(m::Abstract_Model2Fit) = Nothing
get_tagged_model_type(m::Model2Fit_TaggedModel{MODEL,DATA}) where {MODEL<:Abstract_Model2Fit,DATA} = MODEL

# Visit  ================
#
visit_submodel_size(model::Model2Fit_TaggedModel) = 1
visit_get_submodel(model::Model2Fit_TaggedModel,submodel_idx::Int) = get_tagged_model(model)
visit_get_Y(model::Model2Fit_TaggedModel,submodel_idx::Int,Y::AbstractVector,X::AbstractVector,θ::AbstractVector) = Y
visit_get_X(model::Model2Fit_TaggedModel,submodel_idx::Int,Y::AbstractVector,X::AbstractVector,θ::AbstractVector) = X
visit_get_θ(model::Model2Fit_TaggedModel,submodel_idx::Int,Y::AbstractVector,X::AbstractVector,θ::AbstractVector) = θ

# Interface  ================
#
parameter_size(m::Model2Fit_TaggedModel) = parameter_size(visit_get_submodel(m,1))

function accumulate_y!(m::Model2Fit_TaggedModel,Y::AbstractVector,X::AbstractVector,θ::AbstractVector)
    @assert length(θ) == parameter_size(m)
    @assert length(X) == length(Y)

    accumulate_y!(get_tagged_model(m),Y,X,θ)
end


# regularize/z-list.jl

include("Aodwt.jl")
include("diffl.jl")
include("diffs.jl")


"""
    A PETSc Mat wrapper.
"""
mutable struct PetscMat <: AbstractMatrix{PetscScalar}

    " The pointer to the PETSc Mat object "
    mat::Ref{Mat}

    " Whether or not a size has been set by calling setSize() "
    sized::Bool

    " Whether or not preallocation has been done by calling setPreallocation() "
    preallocated::Bool

    " Whether or not the matrix has ever been assembled (Note: the matrix might currently NOT be assembled) "
    assembled::Bool

    function PetscMat()
        mat = Ref{Mat}()
        ccall((:MatCreate, library), PetscErrorCode, (comm_type, Ref{Mat}), MPI.COMM_WORLD, mat)
        ccall((:MatSetType, library), PetscErrorCode, (Mat, MatType), mat[], MATAIJ)
        new(mat, false, false, false)
    end
end

"""
    Set up the size of a PetscMat

    Note: This MUST be called before setting Pre-allocation and _definitely_ must be called before setting/getting values!
"""
function setSize!(mat::PetscMat; m_local::PetscInt=PETSC_DECIDE, n_local::PetscInt=PETSC_DECIDE, m_global::PetscInt=PETSC_DETERMINE, n_global::PetscInt=PETSC_DETERMINE)
    @assert !mat.sized

    # Must provide _some_ size!
    @assert (m_local != PETSC_DECIDE && n_local != PETSC_DECIDE) || (m_global != PETSC_DETERMINE && n_global != PETSC_DETERMINE)

    ccall((:MatSetSizes, library), PetscErrorCode, (Mat, PetscInt, PetscInt, PetscInt, PetscInt), mat.mat[], m_local, n_local, m_global, n_global)

    mat.sized = true
end

"""
    Set the preallocation for storage in each row.

    Note: Should be called _after_ setSize!
"""
function setPreallocation!(mat::PetscMat, local_nonzeros_per_row::Array{PetscInt}, off_processor_nonzeros_per_row::Array{PetscInt})
    @assert mat.sized
    @assert !mat.preallocated

    ccall((:MatSeqAIJSetPreallocation, library), PetscErrorCode, (Mat, PetscInt, Ptr{PetscInt}),
          mat.mat[], PETSC_DEFAULT, local_nonzeros_per_row)

    ccall((:MatMPIAIJSetPreallocation, library), PetscErrorCode, (Mat, PetscInt, Ptr{PetscInt}, PetscInt, Ptr{PetscInt}),
          mat.mat[], PETSC_DEFAULT, local_nonzeros_per_row, PETSC_DEFAULT, off_processor_nonzeros_per_row)

    mat.preallocated = true
end

"""
    Must be called after setting/adding values to construct the matrix
"""
function assemble!(mat::PetscMat)
    @assert mat.sized
    @assert mat.preallocated

    ccall((:MatAssemblyBegin, library), PetscErrorCode, (Mat, MatAssemblyType), mat.mat[], MAT_FINAL_ASSEMBLY)
    ccall((:MatAssemblyEnd, library), PetscErrorCode, (Mat, MatAssemblyType), mat.mat[], MAT_FINAL_ASSEMBLY)

    mat.assembled = true
end

"""
    Use PETSc viewer to print out the matrix
"""
function viewMat(mat::PetscMat)
    @assert mat.sized
    @assert mat.preallocated
    @assert mat.assembled

    viewer = ccall((:PETSC_VIEWER_STDOUT_, library), PetscViewer, (comm_type,), MPI.COMM_WORLD)
    ccall((:MatView, library), PetscErrorCode, (Mat, PetscViewer), mat.mat[], viewer)
end

"""
    mat[i,j] += v

    If the values coming in aren't Floats then we need to make them so
"""
function plusEquals!(mat::PetscMat, v::Matrix{T}, i, j) where T
    plusEquals!(mat, float(v), i, j)
end

"""
    mat[i,j] += v
"""
function plusEquals!(mat::PetscMat, v::Matrix{Float64}, i, j)
    # TODO: do the transpose faster (with loops so there are less copies
    # The transpose is to go from column-major to row-major
    v_T = Vector{Float64}(reshape(v', length(v)))

    # Convert the indices to 0-based indexing
    i_ind = (PetscInt)[i_val-1 for i_val in i]
    j_ind = (PetscInt)[j_val-1 for j_val in j]

    @assert length(v_T) == length(i_ind) * length(j_ind)

    ccall((:MatSetValues, library), PetscErrorCode, (Mat, PetscInt, Ptr{PetscInt}, PetscInt, Ptr{PetscInt}, Ptr{PetscScalar}, InsertMode), mat.mat[], length(i_ind), i_ind, length(j_ind), j_ind, v_T, ADD_VALUES)
end

"""
    mat = 0
"""
function zero!(mat::PetscMat)
    ccall((:MatZeroEntries, library), PetscErrorCode, (Mat,), mat.mat[])
end

"""
    Zeros rows
"""
function zeroRows!(mat::PetscMat, i)
    rows = (PetscInt)[i_ind-1 for i_ind in i]
    ccall((:MatSetOption, library), PetscErrorCode, (Mat, MatOption, PetscBool), mat.mat[], MAT_KEEP_NONZERO_PATTERN, PETSC_TRUE)
    ccall((:MatZeroRows, library), PetscErrorCode, (Mat, PetscInt, Ptr{PetscInt}, PetscScalar, Vec, Vec), mat.mat[], length(rows), rows, 0., C_NULL, C_NULL)
end

#### AbstractArray Interface Definitions ###

"""
    PETSc Matrices are inherently 2D
"""
function linearindexing(mat::PetscMat)
    return Base.LinearSlow()
end

import Base.size

"""
    Returns the _global_ size of the matrix
"""
function size(mat::PetscMat)
    M = Ref{PetscInt}(0)
    N = Ref{PetscInt}(0)

    ccall((:MatGetSize, library), PetscErrorCode, (Mat, Ref{PetscInt}, Ref{PetscInt}), mat.mat[], M, N)

    return (M[], N[])
end

import Base.setindex!

"""
    Sets the value at i,j
"""
function setindex!(mat::PetscMat, v, i, j)
    # Copy out the values
    val = (Float64)[v_val for v_val in v]

    # Call the specialization below
    setindex!(mat, val, i, j)
end

"""
    Sets a matrix into the larger matrix
"""
function setindex!(mat::PetscMat, v::Matrix{Float64}, i, j)
    # TODO: do the transpose faster (with loops so there are less copies
    # The transpose is to go from column-major to row-major
    v_T = reshape(v', length(v))

    # Call the specialization below
    setindex!(mat, v_T, i, j)
end

"""
    Sets the value at i,j.

    Specialization for when v is already an array of Float64 (faster because we don't need to copy it)

    HUGE NOTE!!!  This assumes that `v` is already laid out _row-major_.  If that's not true call one of the above functions...
"""
function setindex!(mat::PetscMat, v::Array{Float64}, i, j)
    # Convert the indices to 0-based indexing and flip them for row-major
    i_ind = (PetscInt)[i_val-1 for i_val in i]
    j_ind = (PetscInt)[j_val-1 for j_val in j]

    @assert length(v) == length(i_ind) * length(j_ind)

    ccall((:MatSetValues, library), PetscErrorCode, (Mat, PetscInt, Ptr{PetscInt}, PetscInt, Ptr{PetscInt}, Ptr{PetscScalar}, InsertMode), mat.mat[], length(i_ind), i_ind, length(j_ind), j_ind, v, INSERT_VALUES)
end

"""
    Don't do this
"""
function setindex!(mat::PetscMat, v, I)
    error("Attempt to index PetscMatrix using a single dimension!")
end

import Base.getindex

"""
    Don't do this either
"""
function getindex(mat::PetscMat, I)
    error("Attempt to index PetscMatrix using a single dimension!")
end

"""
    Proper getter for entries from the matrix for integer indices
"""
function getindex(mat::PetscMat, i::T, j::T) where T<:Integer
    # Don't forget about 1-based indexing...
    i_ind = (PetscInt)[i-1]
    j_ind = (PetscInt)[j-1]

    get_vals = Vector{Float64}(undef, 1)

    ccall((:MatGetValues, library), PetscErrorCode, (Mat, PetscInt, Ptr{PetscInt}, PetscInt, Ptr{PetscInt}, Ref{PetscScalar}), mat.mat[], 1, i_ind, 1, j_ind, get_vals)

    return get_vals[1]
end

"""
    Proper getter for entries from the matrix
"""
function getindex(mat::PetscMat, i, j)
    i_ind = (PetscInt)[i_val-1 for i_val in i]
    j_ind = (PetscInt)[j_val-1 for j_val in j]

    get_vals = Vector{Float64}(undef, length(i_ind) * length(j_ind))

    ccall((:MatGetValues, library), PetscErrorCode, (Mat, PetscInt, Ptr{PetscInt}, PetscInt, Ptr{PetscInt}, Ref{PetscScalar}), mat.mat[], length(i_ind), i_ind, length(j_ind), j_ind, get_vals)

    return reshape(get_vals, length(j_ind), length(i_ind))'
end


### Julia OpenStreetMap Package ###
### MIT License                 ###
### Copyright 2014              ###

### Functions for finding highway intersections ###

### Generate a list of intersections ###
function findIntersections(highways::Dict{Int,Highway})
    seen = Set{Int}()
    intersections = Dict{Int,Intersection}()

    for hwy in values(highways)
        n_nodes = length(hwy.nodes)

        for i in 1:n_nodes
            node = hwy.nodes[i]

            if i == 1 || i == n_nodes || in(node, seen)
                get!(Intersection, intersections, node)
            else
                push!(seen, node)
            end
        end
    end

    for (hwy_key, hwy) in highways
        n_nodes = length(hwy.nodes)

        for i in 1:n_nodes
            node = hwy.nodes[i]

            if i == 1 || i == n_nodes || haskey(intersections, node)
                push!(intersections[node].highways, hwy_key)
            end
        end
    end

    return intersections
end

### Generate a new list of highways divided up by intersections
function segmentHighways(nodes, highways::Dict{Int,Highway}, intersections, classes, levels=Set(1:10))
    segments = Segment[]
    inters = Set(keys(intersections))

    for (i, class) in classes
        if in(class, levels) && haskey(highways,i)
            highway = highways[i]
            first = 1
            for j = 2:length(highway.nodes)
                if highway.nodes[first] != highway.nodes[j] && (in(highway.nodes[j], inters) || j == length(highway.nodes))
                    node0 = highway.nodes[first]
                    node1 = highway.nodes[j]
                    route_nodes = highway.nodes[first:j]
                    dist = distance(nodes, route_nodes)
                    s = Segment(node0, node1, route_nodes, dist, class, i, true)
                    push!(segments, s)

                    if !highway.oneway
                        s = Segment(node1, node0, reverse(route_nodes), dist, class, i, true)
                        push!(segments, s)
                    end
                    first = j
                end
            end
        end
    end

    return segments
end

### Generate a list of highways from segments, for plotting purposes
function highwaySegments( segments::Vector{Segment} )
    highways = Dict{Int,Highway}()

    for k = 1:length(segments)
        highways[k] = Highway("", 1, true, "", "", "", "$(segments[k].parent)", segments[k].nodes)
    end

    return highways
end

### Cluster highway intersections into higher-level intersections ###
# Note that there may be multiple intersection clusters containing the same
# streets, due to curved streets. Parameter max_dist controls how far apart an
# intersection must be from an existing cluster to create a new cluster.
function findIntersectionClusters( nodes::Dict{Int,ENU},
                                   intersections_in::Dict{Int,Intersection},
                                   highway_clusters::Vector{HighwaySet};
                                   max_dist=15.0 )
    hwy_cluster_mapping = Dict{Int,Int}()
    for k = 1:length(highway_clusters)
        hwys = [highway_clusters[k].highways...]
        for kk = 1:length(hwys)
            hwy_cluster_mapping[hwys[kk]] = k
        end
    end

    # Deep copy intersections dictionary and replace highways with highway
    # sets where available
    intersections = deepcopy(intersections_in)
    for (node,inter) in intersections
        hwys = [inter.highways...]
        for k = 1:length(hwys)
            if haskey(hwy_cluster_mapping,hwys[k])
                hwys[k] = hwy_cluster_mapping[hwys[k]]
            end
        end
        inter.highways = Set(hwys)
    end

    # Group intersections by number of streets contained
    hwy_counts = Vector{Int}[]
    for (node,inter) in intersections
        hwy_cnt = length(inter.highways)
        if hwy_cnt > length(hwy_counts)
            for k = (length(hwy_counts)+1):hwy_cnt
                push!(hwy_counts,Int[])
            end
        end
        push!(hwy_counts[hwy_cnt], node)
    end

    clusters = Set{Int}[]                   # Array of sets of highway IDs in each cluster
    clusters_nodes = Set{Int}[]             # Array of sets of node IDs in each cluster
    intersection_mapping = Dict{Int,Int}()  # [intersection id => index in `clusters`]

    for kk = 1:(length(hwy_counts)-1)
        # Start with intersections with most highways, as they are the best
        # "seeds" for new clusters because all intersection nodes added to the cluster
        # must have their highways be a subset of the highways already in the cluster.
        # Skip checking intersections with only 1 highway (road ends)
        k = length(hwy_counts)+1-kk

        for inter in hwy_counts[k]
            found = false
            for index = 1:length(clusters)
                if issubset(intersections[inter].highways,clusters[index])
                    # Check distance to cluster centroid
                    c = centroid(nodes,[clusters_nodes[index]...])
                    c_dist = distance(c,nodes[inter])
                    if c_dist < max_dist
                        intersection_mapping[inter] = index
                        clusters_nodes[index] = Set([clusters_nodes[index]...,inter])
                        found = true
                        break
                    end
                end
            end
            if !found
                push!(clusters,intersections[inter].highways)
                push!(clusters_nodes,Set(inter))
                intersection_mapping[inter] = length(clusters)
            end
        end
    end

    # Create new node at centroid of each intersection cluster
    cluster_map = Dict{Int,Int}()   # [Intersection Node ID => Cluster Node ID]
    for k = 1:length(clusters_nodes)
        if length(clusters_nodes[k]) > 1
            n = [clusters_nodes[k]...]
            c = centroid(nodes,n)
            cluster_node_id = addNewNode!(nodes,c)

            for j = 1:length(n)
                cluster_map[n[j]] = cluster_node_id
            end
        end
    end

    return cluster_map
end

### Replace Nodes in Highways Using Node Remapping
function replaceHighwayNodes!( highways::Dict{Int,Highway}, node_map::Dict{Int,Int} )
    for (key,hwy) in highways
        all_equal = true
        for k = 1:length(hwy.nodes)
            if haskey(node_map,hwy.nodes[k])
                hwy.nodes[k] = node_map[hwy.nodes[k]]
            end

            if k > 1 && hwy.nodes[k] != hwy.nodes[k-1]
                all_equal = false
            end
        end

        # If all nodes in hwy are now equal, delete it.
        if all_equal
            delete!(highways,key)
        end
    end
    return nothing
end


#=
An IP implementation of the Router for comparison against the Pathfinder
algorithm implemented in Mapper2.

I have no plans of making a full fledged router for full models like the one in
Mapper2. Instead, this model is going to be specialized for the network style
of KiloCore/KiloCore2 in that it will be an equivalent abstraction of the
circuit switched network, but not contain all of the details for the routing
muxes.

This is because a lot of these details are unnecessary to get a valid route, and
the fewer unnecessary variables we introduce to the IP forumlation, the better
it will perform (given the usual caveats that implies to IPs)

The basic idea is that the layered network of KC will be represented as two
connected mesh grids like the diagram below

                Circuit Layer 0

              o ----- o ----- o
             /       /       /|
            /       /       / |
           o ----- o ----- o  |
          /       /       /   |
         /       /       /    |
        o ----- o ----- o     o Sink Node
        |                     |
        | Source nodes will   |
        | connect both layers |
 Source |                     |
 Node   |                     |
        o     o ----- o ----- o
        |    /       /       /
        |   /       /       /
        |  o ----- o ----- o
        | /       /       /     Circuit Layer 1
        |/       /       /
        o ----- o ----- o

Both layers will not be connected to eachother except at:

1. Source nodes, to model processor's ability to write to any circuit net.
2. Destination nodes if the "preserve_dest" metadata is set to "false".
   Otherwise, only the destination node on the appropriate layer will be marked
   as a suitable destination.

-------------------
Problem Formulation
-------------------

The problem formulation is that of a constrained multi-commodity maximum network
flow. Communication channels from one processor to another are equivalend to
flows through a network, with the start of the flow coming from the vertex
representing the communicating processor and the end of the flow stopping at the
destination processor.

The primary objective is to minimize the total sum of links used to satisfy all
communication channels.

Constraints are:

1. Flow conservation constraints. The flow into a vertex must be equal to the
   flow out of a vertex. This ensures a continuous, non-branching path from
   source to sink and ensures we can't have spontaneous generation of flows.

   The equivalent to this constraint for terminal nodes is that flow out of a
   source must be 1, and flow into a destination must be 1.

2. Capacity constraints. We must ensure that a flow going through any node does
   not exceed 1 to model the physical constraints of the Kilocore communication
   architecture.

------------------------
Objective Interpretation
------------------------
The final objective is the number of global links used in the final circuit
switched network. We will just use this number and not bother trying to route
the actual Map data structure.

-------------
Optimizations
-------------
To reduce model size, mesh generation for source-sink pairs will be limited to
3 or 4 addresses around the bounding box generated by the source-sink pair.
=#

using JuMP
using Gurobi
using DataStructures
using LightGraphs
using ProgressMeter

import Mapper2.Routing
using Mapper2.Helper

# This is not necessarily scalable to an arbitrary number of layers, or
# interconnect topology.
@enum Network layer_0 layer_1 memory_request memory_return

struct AddressCircuit
    address :: Address{2}
    network :: Network
end

address(a::AddressCircuit) = a.address
getnetwork(a::AddressCircuit) = a.network

function iproute(m::Map{A,2}) where {A}
    # Constuct model
    ip_model, struct_time, struct_bytes, _, _ = @timed build_ip_model(m)
    # Solve the model
    status, solve_time, solve_bytes, _, _ = @timed solve(ip_model)

    m.metadata["ip_route_struct_time"] = struct_time
    m.metadata["ip_route_struct_bytes"] = struct_bytes
    m.metadata["ip_route_total_time"] = solve_time
    m.metadata["ip_route_solve_bytes"] = solve_bytes
    m.metadata["ip_route_objective"] = getobjectivevalue(ip_model)
    m.metadata["ip_route_solve_time"] = getsolvetime(ip_model)

    if status == :Optimal
        m.metadata["ip_optimal"] = true
    else
        m.metadata["ip_opcimal"] = false
    end

    m.metadata["ip_status"] = status

    return m
end

function build_ip_model(m::Map{2})
    # Build the network data type.
    network = build_network(m)

    # Instantiate an empty IP model
    ip_model = Model(solver = GurobiSolver(Threads=1))

    # To create variables for JuMP, we need to do it all in one shot. So, we
    # must iterate over the network to get all the variables
    # (nodes in the network) that we need.
    vertices_for_taskgraph_edge = Vector{Int}[]

    # Amount to grow the bounding box around the extrema of the source-sink
    # pair. This helps keep the number of generated variables down without
    # hurting routing. Keep somewhere between 2 and 4.
    box_growth = 3

    # Collection of vertices used for each edge in the taskgraph.
    vertex_collection = Vector{Int}[]
    # Indices of start and stop nodes for each edge of the graph.
    source_collection = Vector{Int}[]
    sink_collection   = Vector{Int}[]

    num_edges = Mapper2.num_edges(m.taskgraph)

    for (edge_index, taskgraph_edge) in enumerate(getedges(m.taskgraph))
        # Get the source and destination tasks, as well as the addresses
        # for these tasks.
        #
        # Since there are no taskgraphs with fanout, just get the first
        # of the sources/sinks as there should only be 1.
        source_name = first(getsources(taskgraph_edge))
        sink_name = first(getsinks(taskgraph_edge))

        source_address = getaddress(m, source_name)
        sink_address = getaddress(m, sink_name)

        source_task = getnode(m.taskgraph, source_name)
        sink_task = getnode(m.taskgraph, sink_name)

        # Create a bounding box for this address
        box = BoundingBox(source_address, sink_address, box_growth)

        # Get the network for this edge.

        # If the sink is a memory, this uses the "memory_request" link
        if ismemory(sink_task)
            links = [memory_request]

        # Otherwise if the source is a memory, it is a "memory_return" link.
        # Note that there should be no links where both the sources and the
        # sinks are memories.
        elseif ismemory(source_task)
            links = [memory_return]

        # Default - use the circuit links.
        else
            links = [layer_0, layer_1]
        end

        # Get the source and sink vertices for this layer.
        local_source = getvertices(network, source_address, links)
        local_sink = getvertices(network, sink_address, links)
        push!(source_collection, local_source)
        push!(sink_collection, local_sink)

        # Now, collect all vertices that are within the bounding box
        local_vertices = Int[]
        for addr in addresses(box)
            append!(local_vertices, getvertices(network, addr, links))
        end
        push!(vertex_collection, local_vertices)
    end

    # We are now ready to generate the IP model.
    @variable(ip_model,
        x[i = 1:num_edges,
          j = vertex_collection[i],
          k = outneighbors(network, j);
          in(k, vertex_collection[i])
        ], Bin)

    @objective(ip_model, Min, sum(x))

    # Add flow constraints. For start and stop edges, make sure the
    # outgoing/incoming sum is 1.
    #
    # Must also ensure that flow out of a sink node and into a source node are
    # zero to avoid out->in loops.
    #
    # For all other edges, incoming flow must equal outgoing flow.

    for i in 1:num_edges
        valid_vertices = vertex_collection[i]
        @constraint(ip_model, sum(x[i, j, k] for j in source_collection[i], k in outneighbors(network, j) if k in valid_vertices) == 1)
        @constraint(ip_model, sum(x[i, k, j] for j in source_collection[i], k in inneighbors(network, j) if k in valid_vertices) == 0)

        @constraint(ip_model, sum(x[i, j, k] for j in sink_collection[i], k in outneighbors(network, j) if k in valid_vertices) == 0)
        @constraint(ip_model, sum(x[i, k, j] for j in sink_collection[i], k in inneighbors(network, j) if k in valid_vertices) == 1)

        for vertex in valid_vertices
            # Skip source and sink vertices
            (vertex in source_collection[i] || vertex in sink_collection[i]) && continue

            # Flow in and flow out must be equal.
            @constraint(ip_model,
                sum(x[i,vertex,k] for k in outneighbors(network, vertex) if k in valid_vertices) -
                sum(x[i,k,vertex] for k in  inneighbors(network, vertex) if k in valid_vertices) == 0)
        end
    end

    # Add capacity constraints.
    for edge in edges(network)
        source = src(edge)
        dest   = dst(edge)

        # Collect all channels that have this edge.
        indices = Int[]
        for i in 1:num_edges
            if source in vertex_collection[i] && dest in vertex_collection[i]
                push!(indices, i)
            end
        end

        # Add capacity constraint.
        @constraint(ip_model, sum(x[i, source, dest] for i in indices) <= 1)
    end

    return ip_model
end

function build_network(m::Map{2})
    # First, build a light graph for the network given above.
    network = SimpleNetwork()

    # Iterate through all addresses. Add circuit and memory nodes as necessary.
    for addr in addresses(m.architecture)
        # Get the component at this address, and then get the mappable component
        # child of this component.
        component = m.architecture[addr]
        mappables = collect(Iterators.filter(ismappable, component[p] for p in walk_children(component)))
        @assert length(mappables) == 1

        mappable_component = first(mappables)

        # Create vertices for this component
        if isinput(mappable_component) || isoutput(mappable_component)
            # Add just one vertex because the input/output handlers only connect
            # to circuit net 0
            #
            # Since we just added a vertex, the index of that vertex is simply
            # the number of vertices.
            add_vertex!(network, addr, layer_0)
        elseif ismemoryproc(mappable_component)
            # Add both layers for memory processor as well as memory request and
            # and memory return nodes.
            add_vertex!(network, addr, layer_0)
            add_vertex!(network, addr, layer_1)
            add_vertex!(network, addr, memory_return)
            add_vertex!(network, addr, memory_request)
        elseif isproc(mappable_component)
            # Just add the two circuit layers
            add_vertex!(network, addr, layer_0)
            add_vertex!(network, addr, layer_1)
        elseif ismemory(mappable_component)
            # Add the two memory networks
            add_vertex!(network, addr, memory_return)
            add_vertex!(network, addr, memory_request)
        else
            error("Unrecognized component type")
        end
    end

    # Now, extract adjacency information to start connecting nodes together
    connected_components = MapperCore.connected_components(m.architecture)
    for (source,destinations) in connected_components
        for dest in destinations
            add_edge!(network, source, dest)
        end
    end

    return network
end

################################################################################
# Abstraction of the network for convenience purposes.

struct SimpleNetwork
    graph               :: LightGraphs.SimpleGraphs.SimpleDiGraph{Int64}
    vertex_data         :: Dict{Int, AddressCircuit}
    address_to_vertex   :: Dict{Address{2}, Vector{Int}}
end

SimpleNetwork() = SimpleNetwork(
    DiGraph(),
    Dict{Int, AddressCircuit}(),
    Dict{Address{2},Vector{Int}}()
)

getdata(n::SimpleNetwork, i::Int) = n.vertex_data[i]
getvertices(n::SimpleNetwork, a::Address{2}) = get(n.address_to_vertex, a, Int[])

function getvertices(n::SimpleNetwork, a::Address{2}, networks)
    return collect(Iterators.filter(
        x -> getdata(n, x).network in networks,
        i for i in getvertices(n, a)
    ))
end

function LightGraphs.add_vertex!(n::SimpleNetwork, address::Address{2}, network)
    add_vertex!(n.graph)
    n.vertex_data[nv(n.graph)] = AddressCircuit(address, network)

    # Register a vertex at this address.
    if haskey(n.address_to_vertex, address)
        push!(n.address_to_vertex[address], nv(n.graph))
    else
        n.address_to_vertex[address] = [nv(n.graph)]
    end
    return nothing
end

function LightGraphs.add_edge!(n::SimpleNetwork, source::Address, dest::Address)
    # Get the lists of nodes at the source and destination vertices.
    source_vertices = n.address_to_vertex[source]
    dest_vertices   = n.address_to_vertex[dest]

    # Add edges from source to destination by matching circuit types.
    # Note that for things like memory links, this will generate a few extra
    # variables that aren't strictly needed, but this is minimal and won't
    # affect the final routing.
    for sv in source_vertices
        source_data = n.vertex_data[sv]
        for dv in dest_vertices
            dest_data = n.vertex_data[dv]
            if getnetwork(source_data) == getnetwork(dest_data)
                add_edge!(n.graph, sv, dv)
            end
        end
    end
end

# Simple forwards.
LightGraphs.nv(n::SimpleNetwork) = nv(n.graph)
LightGraphs.ne(n::SimpleNetwork) = ne(n.graph)
LightGraphs.outneighbors(n::SimpleNetwork, i) = outneighbors(n.graph, i)
LightGraphs.inneighbors(n::SimpleNetwork, i) = inneighbors(n.graph, i)
LightGraphs.edges(n::SimpleNetwork) = edges(n.graph)

################################################################################
struct BoundingBox{D}
    bounds::NTuple{D,UnitRange{Int}}
end

function BoundingBox(a::Address{D}, b::Address{D}, growth = 0) where D
    return BoundingBox(Tuple(map(1:D) do i
        minval, maxval = minmax(a.I[i], b.I[i])
        return (minval - growth):(maxval + growth)
    end))
end

Mapper2.addresses(b::BoundingBox) = (CartesianIndex(i) for i in Iterators.product(b.bounds...))

function BoundingBox(min::NTuple{D}, max::NTuple{D}) where D
    return BoundingBox(Tuple(min[i]:max[i] for i in 1:D))
end

function Base.in(addr::Address{D}, box::BoundingBox{D}) where D
    for i in 1:D
        in(addr.I[i], box.bounds[i]) || return false
    end
    return true
end