#########################################################################
#                                                                       #
# This file provides functions for interfacing with Matpower data files #
#                                                                       #
#########################################################################

"Parses the matpwer data from either a filename or an IO object"
function parse_matpower(file::Union{IO, String}; validate=true)
    mp_data = parse_matpower_file(file)
    pm_data = matpower_to_powermodels(mp_data)
    if validate
        check_network_data(pm_data)
    end
    return pm_data
end

### Data and functions specific to Matpower format ###

mp_data_names = ["mpc.version", "mpc.baseMVA", "mpc.bus", "mpc.gen",
    "mpc.branch", "mpc.dcline", "mpc.gencost", "mpc.dclinecost",
    "mpc.bus_name", "mpc.storage"
]

mp_bus_columns = [
    ("bus_i", Int),
    ("bus_type", Int),
    ("pd", Float64), ("qd", Float64),
    ("gs", Float64), ("bs", Float64),
    ("area", Int),
    ("vm", Float64), ("va", Float64),
    ("base_kv", Float64),
    ("zone", Int),
    ("vmax", Float64), ("vmin", Float64),
    ("lam_p", Float64), ("lam_q", Float64),
    ("mu_vmax", Float64), ("mu_vmin", Float64)
]

mp_bus_name_columns = [
    ("bus_name", Union{String,SubString{String}})
]

mp_gen_columns = [
    ("gen_bus", Int),
    ("pg", Float64), ("qg", Float64),
    ("qmax", Float64), ("qmin", Float64),
    ("vg", Float64),
    ("mbase", Float64),
    ("gen_status", Int),
    ("pmax", Float64), ("pmin", Float64),
    ("pc1", Float64),
    ("pc2", Float64),
    ("qc1min", Float64), ("qc1max", Float64),
    ("qc2min", Float64), ("qc2max", Float64),
    ("ramp_agc", Float64),
    ("ramp_10", Float64),
    ("ramp_30", Float64),
    ("ramp_q", Float64),
    ("apf", Float64),
    ("mu_pmax", Float64), ("mu_pmin", Float64),
    ("mu_qmax", Float64), ("mu_qmin", Float64)
]

mp_branch_columns = [
    ("f_bus", Int),
    ("t_bus", Int),
    ("br_r", Float64), ("br_x", Float64),
    ("br_b", Float64),
    ("rate_a", Float64),
    ("rate_b", Float64),
    ("rate_c", Float64),
    ("tap", Float64), ("shift", Float64),
    ("br_status", Int),
    ("angmin", Float64), ("angmax", Float64),
    ("pf", Float64), ("qf", Float64),
    ("pt", Float64), ("qt", Float64),
    ("mu_sf", Float64), ("mu_st", Float64),
    ("mu_angmin", Float64), ("mu_angmax", Float64)
]

mp_dcline_columns = [
    ("f_bus", Int),
    ("t_bus", Int),
    ("br_status", Int),
    ("pf", Float64), ("pt", Float64),
    ("qf", Float64), ("qt", Float64),
    ("vf", Float64), ("vt", Float64),
    ("pmin", Float64), ("pmax", Float64),
    ("qminf", Float64), ("qmaxf", Float64),
    ("qmint", Float64), ("qmaxt", Float64),
    ("loss0", Float64),
    ("loss1", Float64),
    ("mu_pmin", Float64), ("mu_pmax", Float64),
    ("mu_qminf", Float64), ("mu_qmaxf", Float64),
    ("mu_qmint", Float64), ("mu_qmaxt", Float64)
]

mp_storage_columns = [
    ("storage_bus", Int),
    ("energy", Float64), ("energy_rating", Float64),
    ("charge_rating", Float64), ("discharge_rating", Float64),
    ("charge_efficiency", Float64), ("discharge_efficiency", Float64),
    ("thermal_rating", Float64),
    ("qmin", Float64), ("qmax", Float64),
    ("r", Float64), ("x", Float64),
    ("standby_loss", Float64),
    ("status", Int)
]

""
function parse_matpower_file(file_string::String)
    mp_data = open(file_string) do io
        parse_matpower_file(io)
    end

    return mp_data
end

""
function parse_matpower_file(io::IO)
    lines = readlines(io)

    return parse_matpower_string(lines)
end

""
function parse_matpower_string(data_lines::Array{String})
    matlab_data, func_name, colnames = parse_matlab_string(data_lines, extended=true)

    case = Dict{String,Any}()

    if func_name != nothing
        case["name"] = func_name
    else
        @info(string("no case name found in matpower file.  The file seems to be missing \"function mpc = ...\""))
        case["name"] = "no_name_found"
    end

    case["source_type"] = "matpower"
    if haskey(matlab_data, "mpc.version")
        case["source_version"] = VersionNumber(matlab_data["mpc.version"])
    else
        @info(string("no case version found in matpower file.  The file seems to be missing \"mpc.version = ...\""))
        case["source_version"] = "0.0.0+"
    end

    if haskey(matlab_data, "mpc.baseMVA")
        case["baseMVA"] = matlab_data["mpc.baseMVA"]
    else
        @info(string("no baseMVA found in matpower file.  The file seems to be missing \"mpc.baseMVA = ...\""))
        case["baseMVA"] = 1.0
    end

    if haskey(matlab_data, "mpc.bus")
        buses = []
        for bus_row in matlab_data["mpc.bus"]
            bus_data = row_to_typed_dict(bus_row, mp_bus_columns)
            bus_data["index"] = check_type(Int, bus_row[1])
            push!(buses, bus_data)
        end
        case["bus"] = buses
    else
        error(string("no bus table found in matpower file.  The file seems to be missing \"mpc.bus = [...];\""))
    end

    if haskey(matlab_data, "mpc.gen")
        gens = []
        for (i, gen_row) in enumerate(matlab_data["mpc.gen"])
            gen_data = row_to_typed_dict(gen_row, mp_gen_columns)
            gen_data["index"] = i
            push!(gens, gen_data)
        end
        case["gen"] = gens
    else
        error(string("no gen table found in matpower file.  The file seems to be missing \"mpc.gen = [...];\""))
    end

    if haskey(matlab_data, "mpc.branch")
        branches = []
        for (i, branch_row) in enumerate(matlab_data["mpc.branch"])
            branch_data = row_to_typed_dict(branch_row, mp_branch_columns)
            branch_data["index"] = i
            push!(branches, branch_data)
        end
        case["branch"] = branches
    else
        error(string("no branch table found in matpower file.  The file seems to be missing \"mpc.branch = [...];\""))
    end

    if haskey(matlab_data, "mpc.dcline")
        dclines = []
        for (i, dcline_row) in enumerate(matlab_data["mpc.dcline"])
            dcline_data = row_to_typed_dict(dcline_row, mp_dcline_columns)
            dcline_data["index"] = i
            push!(dclines, dcline_data)
        end
        case["dcline"] = dclines
    end

    if haskey(matlab_data, "mpc.storage")
        storage = []
        for (i, storage_row) in enumerate(matlab_data["mpc.storage"])
            storage_data = row_to_typed_dict(storage_row, mp_storage_columns)
            storage_data["index"] = i
            push!(storage, storage_data)
        end
        case["storage"] = storage
    end

    if haskey(matlab_data, "mpc.bus_name")
        bus_names = []
        for (i, bus_name_row) in enumerate(matlab_data["mpc.bus_name"])
            bus_name_data = row_to_typed_dict(bus_name_row, mp_bus_name_columns)
            bus_name_data["index"] = i
            push!(bus_names, bus_name_data)
        end
        case["bus_name"] = bus_names

        if length(case["bus_name"]) != length(case["bus"])
            error("incorrect Matpower file, the number of bus names ($(length(case["bus_name"]))) is inconsistent with the number of buses ($(length(case["bus"]))).\n")
        end
    end

    if haskey(matlab_data, "mpc.gencost")
        gencost = []
        for (i, gencost_row) in enumerate(matlab_data["mpc.gencost"])
            gencost_data = mp_cost_data(gencost_row)
            gencost_data["index"] = i
            push!(gencost, gencost_data)
        end
        case["gencost"] = gencost

        if length(case["gencost"]) != length(case["gen"]) && length(case["gencost"]) != 2*length(case["gen"])
            error("incorrect Matpower file, the number of generator cost functions ($(length(case["gencost"]))) is inconsistent with the number of generators ($(length(case["gen"]))).\n")
        end
    end

    if haskey(matlab_data, "mpc.dclinecost")
        dclinecosts = []
        for (i, dclinecost_row) in enumerate(matlab_data["mpc.dclinecost"])
            dclinecost_data = mp_cost_data(dclinecost_row)
            dclinecost_data["index"] = i
            push!(dclinecosts, dclinecost_data)
        end
        case["dclinecost"] = dclinecosts

        if length(case["dclinecost"]) != length(case["dcline"])
            error("incorrect Matpower file, the number of dcline cost functions ($(length(case["dclinecost"]))) is inconsistent with the number of dclines ($(length(case["dcline"]))).\n")
        end
    end

    for k in keys(matlab_data)
        if !in(k, mp_data_names) && startswith(k, "mpc.")
            case_name = k[5:length(k)]
            value = matlab_data[k]
            if isa(value, Array)
                column_names = []
                if haskey(colnames, k)
                    column_names = colnames[k]
                end
                tbl = []
                for (i, row) in enumerate(matlab_data[k])
                    row_data = row_to_dict(row, column_names)
                    row_data["index"] = i
                    push!(tbl, row_data)
                end
                case[case_name] = tbl
                @info("extending matpower format with data: $(case_name) $(length(tbl))x$(length(tbl[1])-1)")
            else
                case[case_name] = value
                @info("extending matpower format with constant data: $(case_name)")
            end
        end
    end

    return case
end

""
function mp_cost_data(cost_row)
    ncost = cost_row[4]
    model = cost_row[1]
    if model == 1
        nr_parameters = ncost*2
    elseif model == 2
       nr_parameters = ncost
    end
    cost_data = Dict{String,Any}(
        "model" => check_type(Int, cost_row[1]),
        "startup" => check_type(Float64, cost_row[2]),
        "shutdown" => check_type(Float64, cost_row[3]),
        "ncost" => check_type(Int, cost_row[4]),
        "cost" => [check_type(Float64, x) for x in cost_row[5:5+nr_parameters-1]]
    )

    #=
    # skip this literal interpretation, as its hard to invert
    cost_values = [check_type(Float64, x) for x in cost_row[5:length(cost_row)]]
    if cost_data["model"] == 1:
        if length(cost_values)%2 != 0
            error("incorrect matpower file, odd number of pwl cost function values")
        end
        for i in 0:(length(cost_values)/2-1)
            p_idx = 1+2*i
            f_idx = 2+2*i
            cost_data["p_$(i)"] = cost_values[p_idx]
            cost_data["f_$(i)"] = cost_values[f_idx]
        end
    else:
        for (i,v) in enumerate(cost_values)
            cost_data["c_$(length(cost_values)+1-i)"] = v
        end
    =#
    return cost_data
end

### Data and functions specific to PowerModels format ###

"""
Converts a Matpower dict into a PowerModels dict
"""
function matpower_to_powermodels(mp_data::Dict{String,Any})
    pm_data = deepcopy(mp_data)

    # required default values
    if !haskey(pm_data, "dcline")
        pm_data["dcline"] = []
    end
    if !haskey(pm_data, "gencost")
        pm_data["gencost"] = []
    end
    if !haskey(pm_data, "dclinecost")
        pm_data["dclinecost"] = []
    end
    if !haskey(pm_data, "storage")
        pm_data["storage"] = []
    end

    # translate component models
    mp2pm_branch(pm_data)
    mp2pm_dcline(pm_data)

    # translate cost models
    add_dcline_costs(pm_data)

    # merge data tables
    merge_bus_name_data(pm_data)
    merge_generator_cost_data(pm_data)
    merge_generic_data(pm_data)

    # split loads and shunts from buses
    split_loads_shunts(pm_data)

    # use once available
    arrays_to_dicts!(pm_data)

    for optional in ["dcline", "load", "shunt", "storage"]
        if length(pm_data[optional]) == 0
            pm_data[optional] = Dict{String,Any}()
        end
    end

    return pm_data
end

"""
    split_loads_shunts(data)

Seperates Loads and Shunts in `data` under separate "load" and "shunt" keys in the
PowerModels data format. Includes references to originating bus via "load_bus"
and "shunt_bus" keys, respectively.
"""
function split_loads_shunts(data::Dict{String,Any})
    data["load"] = []
    data["shunt"] = []

    load_num = 1
    shunt_num = 1
    for (i,bus) in enumerate(data["bus"])
        if bus["pd"] != 0.0 || bus["qd"] != 0.0
            append!(data["load"], [Dict{String,Any}("pd" => bus["pd"],
                                                    "qd" => bus["qd"],
                                                    "load_bus" => bus["bus_i"],
                                                    "status" => convert(Int8, bus["bus_type"] != 4),
                                                    "index" => load_num)])
            load_num += 1
        end

        if bus["gs"] != 0.0 || bus["bs"] != 0.0
            append!(data["shunt"], [Dict{String,Any}("gs" => bus["gs"],
                                                     "bs" => bus["bs"],
                                                     "shunt_bus" => bus["bus_i"],
                                                     "status" => convert(Int8, bus["bus_type"] != 4),
                                                     "index" => shunt_num)])
            shunt_num += 1
        end

        for key in ["pd", "qd", "gs", "bs"]
            delete!(bus, key)
        end
    end
end

"sets all branch transformer taps to 1.0, to simplify branch models"
function mp2pm_branch(data::Dict{String,Any})
    branches = [branch for branch in data["branch"]]
    if haskey(data, "ne_branch")
        append!(branches, data["ne_branch"])
    end
    for branch in branches
        if branch["tap"] == 0.0
            branch["transformer"] = false
            branch["tap"] = 1.0
        else
            branch["transformer"] = true
        end

        branch["g_fr"] = 0.0
        branch["g_to"] = 0.0

        branch["b_fr"] = branch["br_b"] / 2.0
        branch["b_to"] = branch["br_b"] / 2.0

        delete!(branch, "br_b")

        if branch["rate_a"] == 0.0
            delete!(branch, "rate_a")
        end
        if branch["rate_b"] == 0.0
            delete!(branch, "rate_b")
        end
        if branch["rate_c"] == 0.0
            delete!(branch, "rate_c")
        end
    end
end

"adds pmin and pmax values at to and from buses"
function mp2pm_dcline(data::Dict{String,Any})
    for dcline in data["dcline"]
        pmin = dcline["pmin"]
        pmax = dcline["pmax"]
        loss0 = dcline["loss0"]
        loss1 = dcline["loss1"]

        delete!(dcline, "pmin")
        delete!(dcline, "pmax")

        if pmin >= 0 && pmax >=0
            pminf = pmin
            pmaxf = pmax
            pmint = loss0 - pmaxf * (1 - loss1)
            pmaxt = loss0 - pminf * (1 - loss1)
        end
        if pmin >= 0 && pmax < 0
            pminf = pmin
            pmint = pmax
            pmaxf = (-pmint + loss0) / (1-loss1)
            pmaxt = loss0 - pminf * (1 - loss1)
        end
        if pmin < 0 && pmax >= 0
            pmaxt = -pmin
            pmaxf = pmax
            pminf = (-pmaxt + loss0) / (1-loss1)
            pmint = loss0 - pmaxf * (1 - loss1)
        end
        if pmin < 0 && pmax < 0
            pmaxt = -pmin
            pmint = pmax
            pmaxf = (-pmint + loss0) / (1-loss1)
            pminf = (-pmaxt + loss0) / (1-loss1)
        end

        dcline["pmaxt"] = pmaxt
        dcline["pmint"] = pmint
        dcline["pmaxf"] = pmaxf
        dcline["pminf"] = pminf

        # preserve the old pmin and pmax values
        dcline["mp_pmin"] = pmin
        dcline["mp_pmax"] = pmax

        dcline["pt"] = -dcline["pt"] # matpower has opposite convention
        dcline["qf"] = -dcline["qf"] # matpower has opposite convention
        dcline["qt"] = -dcline["qt"] # matpower has opposite convention
    end
end

"adds dcline costs, if gen costs exist"
function add_dcline_costs(data::Dict{String,Any})
    if length(data["gencost"]) > 0 && length(data["dclinecost"]) <= 0 && length(data["dcline"]) > 0
        @info("added zero cost function data for dclines")
        model = data["gencost"][1]["model"]
        if model == 1
            for (i, dcline) in enumerate(data["dcline"])
                dclinecost = Dict(
                    "index" => i,
                    "model" => 1,
                    "startup" => 0.0,
                    "shutdown" => 0.0,
                    "ncost" => 2,
                    "cost" => [dcline["pminf"], 0.0, dcline["pmaxf"], 0.0]
                )
                push!(data["dclinecost"], dclinecost)
            end
        else
            for (i, dcline) in enumerate(data["dcline"])
                dclinecost = Dict(
                    "index" => i,
                    "model" => 2,
                    "startup" => 0.0,
                    "shutdown" => 0.0,
                    "ncost" => 3,
                    "cost" => [0.0, 0.0, 0.0]
                )
                push!(data["dclinecost"], dclinecost)
            end
        end
    end
end

"merges generator cost functions into generator data, if costs exist"
function merge_generator_cost_data(data::Dict{String,Any})
    if haskey(data, "gencost")
        for (i, gencost) in enumerate(data["gencost"])
            gen = data["gen"][i]
            @assert(gen["index"] == gencost["index"])
            delete!(gencost, "index")

            check_keys(gen, keys(gencost))
            merge!(gen, gencost)
        end
        delete!(data, "gencost")
    end

    if haskey(data, "dclinecost")
        for (i, dclinecost) in enumerate(data["dclinecost"])
            dcline = data["dcline"][i]
            @assert(dcline["index"] == dclinecost["index"])
            delete!(dclinecost, "index")

            check_keys(dcline, keys(dclinecost))
            merge!(dcline, dclinecost)
        end
        delete!(data, "dclinecost")
    end
end

"merges bus name data into buses, if names exist"
function merge_bus_name_data(data::Dict{String,Any})
    if haskey(data, "bus_name")
        # can assume same length is same as bus
        # this is validated during matpower parsing
        for (i, bus_name) in enumerate(data["bus_name"])
            bus = data["bus"][i]
            delete!(bus_name, "index")

            check_keys(bus, keys(bus_name))
            merge!(bus, bus_name)
        end
        delete!(data, "bus_name")
    end
end

"merges Matpower tables based on the table extension syntax"
function merge_generic_data(data::Dict{String,Any})
    mp_matrix_names = [name[5:length(name)] for name in mp_data_names]

    key_to_delete = []
    for (k,v) in data
        if isa(v, Array)
            for mp_name in mp_matrix_names
                if startswith(k, "$(mp_name)_")
                    mp_matrix = data[mp_name]
                    push!(key_to_delete, k)

                    if length(mp_matrix) != length(v)
                        error("failed to extend the matpower matrix \"$(mp_name)\" with the matrix \"$(k)\" because they do not have the same number of rows, $(length(mp_matrix)) and $(length(v)) respectively.")
                    end

                    @info("extending matpower format by appending matrix \"$(k)\" in to \"$(mp_name)\"")

                    for (i, row) in enumerate(mp_matrix)
                        merge_row = v[i]
                        #@assert(row["index"] == merge_row["index"]) # note this does not hold for the bus table
                        delete!(merge_row, "index")
                        for key in keys(merge_row)
                            if haskey(row, key)
                                error("failed to extend the matpower matrix \"$(mp_name)\" with the matrix \"$(k)\" because they both share \"$(key)\" as a column name.")
                            end
                            row[key] = merge_row[key]
                        end
                    end

                    break # out of mp_matrix_names loop
                end
            end

        end
    end

    for key in key_to_delete
        delete!(data, key)
    end
end

"Export power network data in the matpower format"
function export_matpower(data::Dict{String,Any})
    return sprint(export_matpower, data)
end

" Get a default value for dict entry "
function get_default(dict, key, default=0.0)
    if haskey(dict, key) && dict[key] != NaN
        return dict[key]
    end
    return default
end

"Export power network data in the matpower format"
function export_matpower(io::IO, data::Dict{String,Any})
    data = deepcopy(data)

    #convert data to mixed unit
    if data["per_unit"]
       make_mixed_units(data)
    end

    # make all costs have the name number of items (to prepare for table export)
    standardize_cost_terms(data)

    # create some useful maps and data structures
    buses = Dict{Int, Dict}()
    for (idx,bus) in data["bus"]
        buses[bus["index"]] = bus
    end
    generators = Dict{Int, Dict}()
    for (idx,gen) in data["gen"]
        generators[gen["index"]] = gen
    end
    branches = Dict{Int, Dict}()
    for (idx,branch) in data["branch"]
       branches[branch["index"]] = branch
    end
    dclines = Dict{Int, Dict}()
    for (idx,dcline) in data["dcline"]
       dclines[dcline["index"]] = dcline
    end
    ne_branches = Dict{Int, Dict}()
    if haskey(data, "ne_branch")
        for (idx,branch) in data["ne_branch"]
            ne_branches[branch["index"]] = branch
        end
    end

    pd = Dict{Int, Float64}()
    qd = Dict{Int, Float64}()
    gs = Dict{Int, Float64}()
    bs = Dict{Int, Float64}()

    # collect all the loads
    for (idx,bus) in sort(data["bus"])
        pd[bus["index"]] = 0
        qd[bus["index"]] = 0
    end
    for (idx,load) in sort(data["load"])
        bus = buses[load["load_bus"]]
        pd[bus["index"]] = pd[bus["index"]] + load["pd"]
        qd[bus["index"]] = qd[bus["index"]] + load["qd"]
    end

    # collect all the shunts
    for (idx,bus) in sort(data["bus"])
        gs[bus["index"]] = 0
        bs[bus["index"]] = 0
    end
    for (idx, shunt) in data["shunt"]
        bus = buses[shunt["shunt_bus"]]
        bs[bus["index"]] = bs[bus["index"]] + shunt["bs"]
        gs[bus["index"]] = gs[bus["index"]] + shunt["gs"]
    end

    mvaBase = data["baseMVA"]

    # Print the header information
    println(io, "%% MATPOWER Case Format : Version 2")
    println(io, "function mpc = ", data["name"])
    println(io, "mpc.version = '2';")
    println(io)
    println(io, "%%-----  Power Flow Data  -----%%")
    println(io, "%% system MVA base")
    println(io, "mpc.baseMVA = ", mvaBase, ";")

    # Print the bus data
    println(io, "%% bus data")
    println(io, "%    bus_i    type    Pd    Qd    Gs    Bs    area    Vm    Va    baseKV    zone    Vmax    Vmin")
    println(io, "mpc.bus = [")
    for (idx,bus) in sort(buses)
    println(io, "\t", bus["index"],
                "\t", bus["bus_type"],
                "\t", get_default(pd, bus["index"]),
                "\t", get_default(qd, bus["index"]),
                "\t", get_default(gs, bus["index"]),
                "\t", get_default(bs, bus["index"]),
                "\t", get_default(bus, "area"),
                "\t", get_default(bus, "vm"),
                "\t", get_default(bus, "va"),
                "\t", get_default(bus, "base_kv"),
                "\t", get_default(bus, "zone"),
                "\t", get_default(bus, "vmax"),
                "\t", get_default(bus, "vmin"),
                "\t", get_default(bus, "lam_p", ""),
                "\t", get_default(bus, "lam_q", ""),
                "\t", get_default(bus, "mu_vmax", ""),
                "\t", get_default(bus, "mu_vmin",""),
                )
    end
    println(io, "];")
    println(io)

    # Print the bus names
    if haskey(collect(values(buses))[1], "bus_name")
        println(io, "%% bus names")
        println(io, "mpc.bus_name = {")
        for (idx,bus) in sort(buses)
            println(io, "\t'", bus["bus_name"], "'")
        end
        println(io, "};")
        println(io)
    end

    # Print the generator data
    println(io, "%% generator data")
    println(io, "%    bus    Pg    Qg    Qmax    Qmin    Vg    mBase    status    Pmax    Pmin    Pc1    Pc2    Qc1min    Qc1max    Qc2min    Qc2max    ramp_agc    ramp_10    ramp_30    ramp_q    apf")
    println(io, "mpc.gen = [")
    i = 1
    for (idx,gen) in sort(generators)
        if idx != gen["index"]
            @info("The index of the generator does not match the matpower assigned index. Any data that uses generator indexes for reference is corrupted.");
        end
    println(io, "\t", gen["gen_bus"],
                "\t", get_default(gen, "pg"),
                "\t", get_default(gen, "qg"),
                "\t", get_default(gen, "qmax"),
                "\t", get_default(gen, "qmin"),
                "\t", get_default(gen, "vg"),
                "\t", get_default(gen, "mbase"),
                "\t", get_default(gen, "gen_status"),
                "\t", get_default(gen, "pmax"),
                "\t", get_default(gen, "pmin"),
                "\t", get_default(gen, "pc1", ""),
                "\t", get_default(gen, "pc2", ""),
                "\t", get_default(gen, "qc1min", ""),
                "\t", get_default(gen, "qc1max", ""),
                "\t", get_default(gen, "qc2min", ""),
                "\t", get_default(gen, "qc2max", ""),
                "\t", get_default(gen, "ramp_agc", ""),
                "\t", (haskey(gen, "ramp_10") ? gen["ramp_10"] : haskey(gen, "ramp_30") ? 0 : ""),
                "\t", get_default(gen, "ramp_30", ""),
                "\t", get_default(gen, "ramp_q", ""),
                "\t", get_default(gen, "apf", ""),
                "\t", get_default(gen, "mu_pmax", ""),
                "\t", get_default(gen, "mu_pmin", ""),
                "\t", get_default(gen, "mu_qmax", ""),
                "\t", get_default(gen, "mu_qmin", ""),
                )

        i = i+1
    end
    println(io,"];")
    println(io)

    # Print the branch data
    println(io, "%% branch data")
    println(io, "%    fbus    tbus    r    x    b    rateA    rateB    rateC    ratio    angle    status    angmin    angmax")
    println(io, "mpc.branch = [")
    i = 1
    for (idx,branch) in sort(branches)
        if idx != branch["index"]
            @info("The index of the branch does not match the matpower assigned index. Any data that uses branch indexes for reference is corrupted.");
        end
        println(io,
            "\t", get_default(branch, "f_bus"),
            "\t", get_default(branch, "t_bus"),
            "\t", get_default(branch, "br_r"),
            "\t", get_default(branch, "br_x"),
            "\t", (branch["b_to"] + branch["b_fr"]) != NaN ? (branch["b_to"] + branch["b_fr"]) : 0,
            "\t", get_default(branch, "rate_a"),
            "\t", get_default(branch, "rate_b"),
            "\t", get_default(branch, "rate_c"),

            "\t", (branch["transformer"] ? branch["tap"] : 0),
            "\t", (branch["transformer"] ? branch["shift"] : 0),
            "\t", get_default(branch, "br_status"),
            "\t", get_default(branch, "angmin"),
            "\t", get_default(branch, "angmax"),
            "\t", get_default(branch, "pf", ""),
            "\t", get_default(branch, "qf", ""),
            "\t", get_default(branch, "pt", ""),
            "\t", get_default(branch, "qt", ""),
            "\t", get_default(branch, "mu_sf", ""),
            "\t", get_default(branch, "mu_st", ""),
            "\t", get_default(branch, "mu_angmin", ""),
            "\t", get_default(branch, "mu_angmax", ""),
        )
        i = i+1
    end
    println(io, "];")
    println(io)

    # print the dcline data
    println(io, "%% dcline data")
    println(io, "%	fbus	tbus	status	Pf	Pt	Qf	Qt	Vf	Vt	Pmin	Pmax	QminF	QmaxF	QminT	QmaxT	loss0	loss1")
    println(io, "mpc.dcline = [")
    for (idx, dcline) in sort(dclines)
        println(io,
            "\t", get_default(dcline, "f_bus"),
            "\t", get_default(dcline, "t_bus"),
            "\t", get_default(dcline, "br_status"),
            "\t", get_default(dcline, "pf"),
            "\t", -get_default(dcline, "pt"),  # opposite convention
            "\t", -get_default(dcline, "qf"),  # opposite convention
            "\t", -get_default(dcline, "qt"),  # opposite convention
            "\t", get_default(dcline, "vf"),
            "\t", get_default(dcline, "vt"),
            "\t", (haskey(dcline, "mp_pmin") ? dcline["mp_pmin"] : min(dcline["pmaxt"], dcline["pmint"], dcline["pmaxf"], dcline["pminf"])),
            "\t", (haskey(dcline, "mp_pmax") ? dcline["mp_pmax"] : max(dcline["pmaxt"], dcline["pmint"], dcline["pmaxf"], dcline["pminf"])),
            "\t", get_default(dcline, "qminf"),
            "\t", get_default(dcline, "qmaxf"),
            "\t", get_default(dcline, "qmint"),
            "\t", get_default(dcline, "qmaxt"),
            "\t", get_default(dcline, "loss0"),
            "\t", get_default(dcline, "loss1"),
            "\t", get_default(dcline, "mu_pmin", ""),
            "\t", get_default(dcline, "mu_pmax", ""),
            "\t", get_default(dcline, "mu_qminf", ""),
            "\t", get_default(dcline, "mu_qmaxf", ""),
            "\t", get_default(dcline, "mu_qmint", ""),
            "\t", get_default(dcline, "mu_qmaxt", ""),
        )
    end
    println(io, "];")
    println(io)

    # Print the gen cost data
    export_cost_data(io, generators, "mpc.gencost")

    # Print the dcline cost data
    export_cost_data(io, dclines, "mpc.dclinecost")

    # ne branch is not part of the matpower specs. However, it is treated as a special case by the matpower parser
    # for example, br_b is converted into b_to and b_fr
    if haskey(data, "ne_branch")
        println(io, "%column_names%	f_bus	t_bus	br_r	br_x	br_b	rate_a	rate_b	rate_c	tap	shift	br_status	angmin	angmax	construction_cost")
        println(io, "mpc.ne_branch = [")
        i = 1
        for (idx,branch) in sort(ne_branches)
            if idx != branch["index"]
                @info("The index of the ne_branch does not match the matpower assigned index. Any data that uses branch indexes for reference is corrupted.");
            end
            println(io,
                "\t", branch["f_bus"],
                "\t", branch["t_bus"],
                "\t", get_default(branch, "br_r"),
                "\t", get_default(branch, "br_x"),
                "\t", (haskey(branch,"b_to") ? branch["b_to"] + branch["b_fr"]  : 0),
                "\t", get_default(branch, "rate_a"),
                "\t", get_default(branch, "rate_b"),
                "\t", get_default(branch, "rate_c"),
                "\t", (branch["transformer"] ? branch["tap"] : 0),
                "\t", (branch["transformer"] ? branch["shift"] : 0),
                "\t", get_default(branch, "br_status"),
                "\t", get_default(branch, "angmin"),
                "\t", get_default(branch, "angmax"),
                "\t", get_default(branch, "construction_cost"),
            )
            i = i+1
        end
        println(io, "];");
        println(io)
    end

    # Print the extra bus data
    export_extra_data(io, data, "bus", Set(["index", "gs", "bs", "zone", "bus_i", "bus_type", "qd",  "vmax", "area", "vmin", "va", "vm", "base_kv", "pd", "bus_name", "lam_p", "lam_q", "mu_vmax", "mu_vmin"]); postfix="_data")

    # Print the extra generator data
    export_extra_data(io, data, "gen", Set(["index", "gen_bus", "pg", "qg", "qmax", "qmin", "vg", "mbase", "gen_status", "pmax", "pmin", "pc1", "pc2", "qc1min", "qc1max", "qc2min", "qc2max", "ramp_agc", "ramp_10", "ramp_30", "ramp_q", "apf", "ncost", "model", "shutdown", "startup", "cost", "mu_pmax", "mu_pmin", "mu_qmax", "mu_qmin"]); postfix="_data")

    # Print the extra branch data
    export_extra_data(io, data, "branch", Set(["index", "f_bus", "t_bus", "br_r", "br_x", "br_b", "b_to", "b_fr", "rate_a", "rate_b", "rate_c", "tap", "shift", "br_status", "angmin", "angmax", "transformer", "g_to", "g_fr", "pf", "qf", "pt", "qt", "mu_sf", "mu_st", "mu_angmin", "mu_angmax"]); postfix="_data")

    # Print the extra dcline data
    export_extra_data(io, data, "dcline", Set(["index", "mu_qmaxt", "mu_qmint", "mu_qmaxf", "mu_qminf", "mu_pmax", "mu_pmin", "loss0", "loss1", "qmint", "qmaxt", "pmin", "pmax", "qminf", "qmaxf", "f_bus", "t_bus", "br_status", "pf", "pt", "qf", "qt", "vf", "vt", "ncost", "model", "shutdown", "pmaxt", "startup", "pmint", "cost", "pminf", "pmaxf", "mp_pmax", "mp_pmin"]); postfix="_data")

    # Print the extra ne_branch data
    if haskey(data, "ne_branch")
        export_extra_data(io, data, "ne_branch", Set(["index", "f_bus", "t_bus", "br_r", "br_x", "br_b", "b_to", "b_fr", "rate_a", "rate_b", "rate_c", "tap", "shift", "br_status", "angmin", "angmax", "transformer", "construction_cost", "g_to", "g_fr"]); postfix="_data")
    end

    # print the extra load data
    export_extra_data(io, data, "load", Set(["index", "load_bus", "status", "qd", "pd"]); postfix="_data")

    # print the extra shunt data
    export_extra_data(io, data, "shunt", Set(["index", "shunt_bus", "status", "gs", "bs"]); postfix="_data")

    # print the extra component data
    for (key, value) in data
        if key != "bus" && key != "gen" && key != "branch" && key != "load" && key != "shunt" && key != "dcline" && key != "ne_branch" && key != "version" && key != "baseMVA" && key != "per_unit" && key != "name" && key != "source_type" && key != "source_version"
            export_extra_data(io, data, key)
        end
    end
end

"Export fields of a component type"
function export_extra_data(io::IO, data::Dict{String,Any}, component, excluded_fields=Set(["index"]); postfix="")
    if isa(data[component], Int) || isa(data[component], Int64) || isa(data[component], Float64)
        println(io, "mpc.", component, " = ", data[component], ";")
        println(io)
        return
    end

    if isa(data[component], String) || isa(data[component], SubString{String})
        println(io, "mpc.", component, " = '", data[component], "';")
        println(io)
        return
    end

    if !isa(data[component], Dict)
        return
    end

    if length(data[component]) == 0
        return
    end

    # Gather the fields
    included_fields = []
    c = collect(values(data[component]))[1]
    for (key, value) in c
        if !in(key, excluded_fields)
            push!(included_fields, key)
        end
    end

    if length(included_fields) == 0
        return
    end

    # Print the header
    print(io, "%column_names% ")
    for field in included_fields
        print(io, field)
        print(io, " ")
    end
    println(io)
    println(io, "mpc.", component, postfix, " = {")

    # sort the data
    components = Dict{Int, Dict}()
    for (idx,c) in data[component]
        components[c["index"]] = c
    end

    # print the data
    i = 1
    for (idx,c) in sort(components)
        if idx != c["index"]
            @info("The index of a component does not match the matpower assigned index. Any data that uses component indexes for reference is corrupted.");
        end

        for field in included_fields
            print(io,"\t")
            if isa(c[field], Union{String,SubString})
                print(io, "'")
            end
            print(io,c[field])
            if isa(c[field], Union{String,SubString})
                print(io, "'")
            end
        end
        println(io)
        i = i+1
    end
    println(io, "};")
    println(io)
end

"Export cost data"
function export_cost_data(io::IO, components::Dict{Int,Dict}, prefix::String)
    if length(components) <= 0
        return
    end

    a_comp = collect(values(components))[1]
    if haskey(a_comp, "cost")
        ncost = length(a_comp["cost"])
        model = a_comp["model"]

        for (i,comp) in components
            if length(comp["cost"]) != ncost || comp["model"] != model
                @info("heterogeneous cost functions will be ommited in Matpower data")
                return
            end
        end

        println(io, "%%-----  OPF Data  -----%%")
        println(io, "%% cost data")
        println(io, "%    1    startup    shutdown    n    x1    y1    ...    xn    yn")
        println(io, "%    2    startup    shutdown    n    c(n-1)    ...    c0")
        println(io, prefix, " = [")

        for (idx,gen) in (sort(components))
            if gen["model"] == 1
                print(io, "\t1\t", gen["startup"], "\t", gen["shutdown"], "\t", (length(gen["cost"])/2) ),
                for l=1:length(gen["cost"])
                    print(io, "\t", gen["cost"][l])
                end
            else
                print(io, "\t2\t", gen["startup"], "\t", gen["shutdown"], "\t", length(gen["cost"])),
                for l=1:length(gen["cost"])
                    print(io, "\t", gen["cost"][l])
                end
            end
            println(io)
        end
        println(io, "];");
        println(io)
    end
end


#!/usr/bin/env julia

# For the files given in ARGS, create all the needed continuation jobs
# for the second phase. That includes all the accessible T2 and all
# the tw₁ starts.

const T2iters = 1e8
const availableT2 = [0.9, 0.8, 0.7, 0.6, 0.5]
const tw1list = map(Int64,[1e4,1e5,1e6,1e7,1e8])

const param = "-m 33 -t 1e4 -T 1e8 -n 10 -L"

function get_fields(file)
    regex = r"quench_T_([0-9.]+)_L_([0-9]+)_([a-z]+)_R([AB]).net"
    fields = match(regex, basename(file)).captures
    length(fields) == 0 && error("Couldn't get metadata from $file")
    T = parse(Float64, fields[1])
    L = parse(Int64, fields[2])
    id = fields[3] * "-R" * fields[4]
    return T, L, id
end

# Return strings containing the continuation jobs for .net file
function get_job_list(file)
    T1, L, id = get_fields(file)
    jobs = String[]
    T2list = [t for t ∈ availableT2 if t<T1]
    for T2 in T2list
        for tw1 in tw1list
            outname = "continue_at_T2_$(T2)_tw1_$(tw1)_for_T1_$(T1)_L_$(L)_id_$(id).net"
            cmd = "./exe/annealer -c $tw1  $param -p -l $L -i $T2iters $T2"
            job = "cat $file | $cmd \"$(outname)\""
            cleanup = "lzma $(outname)"
            push!(jobs, job * "\n" * cleanup * "\n" * "echo 'All done! ($outname)'")
        end
    end
    return jobs
end

jobs = String[]
for file in ARGS
    for j in get_job_list(file)
        push!(jobs,j)
    end
end
info("$(length(jobs)) jobs processed.")

n = 1
for job in jobs
    header=
    """
    #!/bin/sh
    #\$ -o \$HOME/glass_continue_$(n).out -j y
    #\$ -N glass_$(n)

    cd tfg/

    """
    open("continueT2_$n.sh", "w") do io
        println(io,header)
        println(io,job)
    end
    n+=1
end


# make a configuration broadcastable
Broadcast.broadcastable( sm::SurrogateModel ) = Ref(sm);
Broadcast.broadcastable( sc::SurrogateConfig ) = Ref(sc);

# Methods to be implemented by each type inheriting from SurrogateConfig
max_evals( :: SurrogateConfig ) = typemax(Int) :: Int;

# return data that is stored in iter data in each iteration
# get_saveable(::SurrogateMeta) = nothing :: Union{Nothing, <:SurrogateMeta};

fully_linear( :: SurrogateModel ) = false :: Bool;

# can objective functions with same configuration types be combined
# to a new vector objective?
combinable( :: SurrogateConfig ) = false :: Bool

# TODO make combinable bi-variate to check for to concrete configs if they are combinable

## TODO: make `prepare_init_model` and `_init_model` have a `ensure_fully_linear` kwarg too
function prepare_init_model( ::SurrogateConfig, :: AbstractObjective, :: AbstractMOP,
    :: AbstractIterData, ::AbstractDB, :: AbstractConfig; kwargs... ) :: SurrogateMeta
    nothing
end

function _init_model( ::SurrogateConfig, :: AbstractObjective, :: AbstractMOP,
    :: AbstractIterData, ::AbstractDB, :: AbstractConfig, :: SurrogateMeta; kwargs... ) :: Tuple{<:SurrogateModel,<:SurrogateMeta}
    nothing
end

## TODO: Allow to pass a SurrogateConfig here as well. (ATM use `model_cfg(objf)`) #src
## In general, the function signatures are somewhat messy. We should unify them a bit. #src
function update_model( :: SurrogateModel, :: AbstractObjective, :: SurrogateMeta, :: AbstractMOP,
    :: AbstractIterData, :: AbstractDB, :: AbstractConfig; kwargs... ) :: Tuple{<:SurrogateModel,<:SurrogateMeta}
    nothing
end

eval_models( :: SurrogateModel, ::Vec ) ::Vec = nothing
get_gradient( :: SurrogateModel, ::Vec, :: Int ) :: Vec = nothing
get_jacobian( :: SurrogateModel, :: Vec ) :: Mat = nothing

# DEFAULTS

get_saveable_type( :: SurrogateMeta ) = Nothing
get_saveable( :: SurrogateMeta ) = nothing

prepare_update_model( mod, objf, meta, mop, iter_data, db, algo_config; kwargs...) = meta
prepare_improve_model( mod, objf, meta, mop, iter_data, db, algo_config; kwargs...) = meta

# overwrite if possible, this is inefficient:
eval_models( sm :: SurrogateModel, x̂ :: Vec, ℓ :: Int) = eval_models(sm, x̂)[ℓ]

function improve_model( mod :: SurrogateModel, objf:: AbstractObjective, meta :: SurrogateMeta,
    mop :: AbstractMOP, id :: AbstractIterData, db :: AbstractDB, ac :: AbstractConfig;
    ensure_fully_linear = false)
    return mod, meta
end

# check if surrogate configurations are equal (only really needed if combinable)
function Base.:(==)( cfg1 :: T, cfg2 :: T ) where T <: SurrogateConfig
    all( getfield(cfg1, fname) == getfield(cfg2, fname) for fname ∈ fieldnames(T) )
end

function Base.:(==)( cfg1 :: T, cfg2 :: F ) where {T <: SurrogateConfig, F<:SurrogateConfig}
    false
end

# only needed if combinable
combine( cfg :: SurrogateConfig, :: SurrogateConfig  ) = cfg

## derived

@doc """
Return a function handle to be used with `NLopt` for output `ℓ` of `model`.
That is, if `model` is a surrogate for two scalar objectives, then `ℓ` must
be either 1 or 2.
"""
function _get_optim_handle( model :: SurrogateModel, ℓ :: Int )
    # Return an anonymous function that modifies the gradient if present
    function (x :: Vec, g :: Vec)
        if !isempty(g)
            g[:] = get_gradient( model, x, ℓ)
        end
        return eval_models( model, x, ℓ)
    end
end


@safetestset "ishash" begin
    using Validator

    # length = 32
    algorithms = ["md5", "md4", "ripemd128", "tiger128"]
    for algorithm in algorithms
        @test Validator.ishash("d94f3f016ae679c3008de268209132f2", algorithm) === true
        @test Validator.ishash("751adbc511ccbe8edf23d486fa4581cd", algorithm) === true
        @test Validator.ishash("88dae00e614d8f24cfd5a8b3f8002e93", algorithm) === true
        @test Validator.ishash("0bf1c35032a71a14c2f719e5a14c1e96", algorithm) === true
        @test Validator.ishash("d94f3F016Ae679C3008de268209132F2", algorithm) === true
        @test Validator.ishash("88DAE00e614d8f24cfd5a8b3f8002E93", algorithm) === true

        @test Validator.ishash("q94375dj93458w34", algorithm) === false
        @test Validator.ishash("39485729348", algorithm) === false
        @test Validator.ishash("%&FHKJFvk", algorithm) === false
        @test Validator.ishash("KYT0bf1c35032a71a14c2f719e5a1", algorithm) === false
        @test Validator.ishash("", algorithm) === false
        @test Validator.ishash("d94f3f016ae679c3008de268209132f&", algorithm) === false
    end

    algorithms = ["crc32", "crc32b"]
    for algorithm in algorithms
        @test Validator.ishash("d94f3f01", algorithm) === true
        @test Validator.ishash("751adbc5", algorithm) === true
        @test Validator.ishash("88dae00e", algorithm) === true
        @test Validator.ishash("0bf1c350", algorithm) === true
        @test Validator.ishash("88DAE00e", algorithm) === true
        @test Validator.ishash("751aDBc5", algorithm) === true

        @test Validator.ishash("KYT0bf1c35032a71a14c2f719e5a14c1", algorithm) === false
        @test Validator.ishash("q94375dj93458w34", algorithm) === false
        @test Validator.ishash("q943", algorithm) === false
        @test Validator.ishash("39485729348", algorithm) === false
        @test Validator.ishash("%&FHKJFvk", algorithm) === false
    end

    algorithms = ["sha1", "tiger160", "ripemd160"]
    for algorithm in algorithms
        @test Validator.ishash("3ca25ae354e192b26879f651a51d92aa8a34d8d3", algorithm) === true
        @test Validator.ishash("aaf4c61ddcc5e8a2dabede0f3b482cd9aea9434d", algorithm) === true
        @test Validator.ishash("beb8c3f30da46be179b8df5f5ecb5e4b10508230", algorithm) === true
        @test Validator.ishash("efd5d3b190e893ed317f38da2420d63b7ae0d5ed", algorithm) === true
        @test Validator.ishash("AAF4c61ddCC5e8a2dabede0f3b482cd9AEA9434D", algorithm) === true
        @test Validator.ishash("3ca25AE354e192b26879f651A51d92aa8a34d8D3", algorithm) === true

        @test Validator.ishash("KYT0bf1c35032a71a14c2f719e5a14c1", algorithm) === false
        @test Validator.ishash("KYT0bf1c35032a71a14c2f719e5a14c1dsjkjkjkjkkjk", algorithm) === false
        @test Validator.ishash("q94375dj93458w34", algorithm) === false
        @test Validator.ishash("39485729348", algorithm) === false
        @test Validator.ishash("%&FHKJFvk", algorithm) === false
    end

    @test Validator.ishash("2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824", "sha256") === true
    @test Validator.ishash("1d996e033d612d9af2b44b70061ee0e868bfd14c2dd90b129e1edeb7953e7985", "sha256") === true
    @test Validator.ishash("80f70bfeaed5886e33536bcfa8c05c60afef5a0e48f699a7912d5e399cdcc441", "sha256") === true
    @test Validator.ishash("579282cfb65ca1f109b78536effaf621b853c9f7079664a3fbe2b519f435898c", "sha256") === true
    @test Validator.ishash("2CF24dba5FB0a30e26E83b2AC5b9E29E1b161e5C1fa7425E73043362938b9824", "sha256") === true
    @test Validator.ishash("80F70bFEAed5886e33536bcfa8c05c60aFEF5a0e48f699a7912d5e399cdCC441", "sha256") === true

    @test Validator.ishash("KYT0bf1c35032a71a14c2f719e5a14c1", "sha256") === false
    @test Validator.ishash("KYT0bf1c35032a71a14c2f719e5a14c1dsjkjkjkjkkjk", "sha256") === false
    @test Validator.ishash("q94375dj93458w34", "sha256") === false
    @test Validator.ishash("39485729348", "sha256") === false
    @test Validator.ishash("%&FHKJFvk", "sha256") === false

    @test Validator.ishash("3fed1f814d28dc5d63e313f8a601ecc4836d1662a19365cbdcf6870f6b56388850b58043f7ebf2418abb8f39c3a42e31", "sha384") === true
    @test Validator.ishash("b330f4e575db6e73500bd3b805db1a84b5a034e5d21f0041d91eec85af1dfcb13e40bb1c4d36a72487e048ac6af74b58", "sha384") === true
    @test Validator.ishash("bf547c3fc5841a377eb1519c2890344dbab15c40ae4150b4b34443d2212e5b04aa9d58865bf03d8ae27840fef430b891", "sha384") === true
    @test Validator.ishash("fc09a3d11368386530f985dacddd026ae1e44e0e297c805c3429d50744e6237eb4417c20ffca8807b071823af13a3f65", "sha384") === true
    @test Validator.ishash("3fed1f814d28dc5d63e313f8A601ecc4836d1662a19365CBDCf6870f6b56388850b58043f7ebf2418abb8f39c3a42e31", "sha384") === true
    @test Validator.ishash("b330f4E575db6e73500bd3b805db1a84b5a034e5d21f0041d91EEC85af1dfcb13e40bb1c4d36a72487e048ac6af74b58", "sha384") === true

    @test Validator.ishash("KYT0bf1c35032a71a14c2f719e5a14c1", "sha384") === false
    @test Validator.ishash("KYT0bf1c35032a71a14c2f719e5a14c1dsjkjkjkjkkjk", "sha384") === false
    @test Validator.ishash("q94375dj93458w34", "sha384") === false
    @test Validator.ishash("39485729348", "sha384") === false
    @test Validator.ishash("%&FHKJFvk", "sha384") === false

    @test Validator.ishash("9b71d224bd62f3785d96d46ad3ea3d73319bfbc2890caadae2dff72519673ca72323c3d99ba5c11d7c7acc6e14b8c5da0c4663475c2e5c3adef46f73bcdec043", "sha512") === true
    @test Validator.ishash("83c586381bf5ba94c8d9ba8b6b92beb0997d76c257708742a6c26d1b7cbb9269af92d527419d5b8475f2bb6686d2f92a6649b7f174c1d8306eb335e585ab5049", "sha512") === true
    @test Validator.ishash("45bc5fa8cb45ee408c04b6269e9f1e1c17090c5ce26ffeeda2af097735b29953ce547e40ff3ad0d120e5361cc5f9cee35ea91ecd4077f3f589b4d439168f91b9", "sha512") === true
    @test Validator.ishash("432ac3d29e4f18c7f604f7c3c96369a6c5c61fc09bf77880548239baffd61636d42ed374f41c261e424d20d98e320e812a6d52865be059745fdb2cb20acff0ab", "sha512") === true
    @test Validator.ishash("9B71D224bd62f3785D96d46ad3ea3d73319bFBC2890CAAdae2dff72519673CA72323C3d99ba5c11d7c7ACC6e14b8c5DA0c4663475c2E5c3adef46f73bcDEC043", "sha512") === true
    @test Validator.ishash("432AC3d29E4f18c7F604f7c3c96369A6C5c61fC09Bf77880548239baffd61636d42ed374f41c261e424d20d98e320e812a6d52865be059745fdb2cb20acff0ab", "sha512") === true

    @test Validator.ishash("KYT0bf1c35032a71a14c2f719e5a14c1", "sha512") === false
    @test Validator.ishash("KYT0bf1c35032a71a14c2f719e5a14c1dsjkjkjkjkkjk", "sha512") === false
    @test Validator.ishash("q94375dj93458w34", "sha512") === false
    @test Validator.ishash("39485729348", "sha512") === false
    @test Validator.ishash("%&FHKJFvk", "sha512") === false

    @test Validator.ishash("6281a1f098c5e7290927ed09150d43ff3990a0fe1a48267c", "tiger192") === true
    @test Validator.ishash("56268f7bc269cf1bc83d3ce42e07a85632394737918f4760", "tiger192") === true
    @test Validator.ishash("46fc0125a148788a3ac1d649566fc04eb84a746f1a6e4fa7", "tiger192") === true
    @test Validator.ishash("7731ea1621ae99ea3197b94583d034fdbaa4dce31a67404a", "tiger192") === true
    @test Validator.ishash("6281A1f098c5e7290927ed09150d43ff3990a0fe1a48267C", "tiger192") === true
    @test Validator.ishash("46FC0125a148788a3AC1d649566fc04eb84A746f1a6E4fa7", "tiger192") === true

    @test Validator.ishash("KYT0bf1c35032a71a14c2f719e5a14c1", "tiger192") === false
    @test Validator.ishash("KYT0bf1c35032a71a14c2f719e5a14c1dsjkjkjkjkkjk", "tiger192") === false
    @test Validator.ishash("q94375dj93458w34", "tiger192") === false
    @test Validator.ishash("39485729348", "tiger192") === false
    @test Validator.ishash("%&FHKJFvk", "tiger192") === false
end


using CounterfactualExplanations
using CounterfactualExplanations.Data
import CounterfactualExplanations.Data: toy_data_linear, toy_data_multi, toy_data_non_linear

@testset "Artifacts" begin

    @testset "cats_dogs" begin
        @test !isnothing(CounterfactualExplanations.Data.cats_dogs_data())
        @test !isnothing(CounterfactualExplanations.Data.cats_dogs_model())
        # @test !isnothing(CounterfactualExplanations.Data.cats_dogs_laplace())
    end

    @testset "MNIST" begin
        @test !isnothing(CounterfactualExplanations.Data.mnist_data())
        @test !isnothing(CounterfactualExplanations.Data.mnist_model())
        @test !isnothing(CounterfactualExplanations.Data.mnist_ensemble())
    end
end

@testset "Toy data" begin
    @test length(toy_data_linear()) == 2
    @test length(toy_data_non_linear()) == 2
    @test length(toy_data_multi()) == 2
end


using ReactionEngine
using Test

@testset "ReactionEngine.jl" begin
    # Write your tests here.

    @testset "Validate Inspect" begin
        retcode = inspect("inspect/inspect.xml","lib/")
        @test retcode == Symbol("Success")
    end

    @testset "Validate Thermoprobe" begin
        retcode = thermoprobe("thermo_probe/thermo_probe.xml","lib/")
        @test retcode == Symbol("Success")
    end

    @testset "Validate Transport properties" begin
        retcode = transport_properties("transport/transport.xml","lib/")
        @test retcode == Symbol("Success")
    end

    @testset "Validate CSTR" begin
        retcode = cstr("cstr/cstr.xml","lib/")
        @test retcode == Symbol("Success")
    end

    @testset "Validate Plug" begin
        retcode = plug("plug/plug.xml","lib/")
        @test retcode == Symbol("Success")
    end

    @testset "Validate Batch" begin
        retcode = batch("batch/batch.xml","lib/")
        @test retcode == Symbol("Success")
    end

    @testset "Validate Equilibrium" begin
        retcode = equilibrate("equil/ch4/equil.xml","lib/")
        @test retcode == Symbol("Success")
    end

end


import Random

function generate_deck()
    cards::Vector{Card} = Card[]
    for i in 1:4
        for j in 2:14
            push!(cards, Card(Ranks.Rank(j), Suits.Suit(i)))
        end
    end
    return Deck(cards, 1)
end

function predeal_shuffle!(deck::Deck)
    Random.shuffle!(deck.cards)
end

function reshuffle!(deck::Deck)
    Random.shuffle!(deck.cards)
    deck.top = 1
end

function deal!(deck::Deck, players::Player...)
    i = 1
    while i <= length(players)
        players[i].hole.card1 = _pop_deck!(deck)
        i += 1
    end
    i = 1
    while i <= length(players)
        players[i].hole.card2 = _pop_deck!(deck)
        i += 1
    end
end

function flop!(deck::Deck, board::Board)
    _burn!(deck)
    board.flop1 = _pop_deck!(deck)
    board.flop2 = _pop_deck!(deck)
    board.flop3 = _pop_deck!(deck)
end

function turn!(deck::Deck, board::Board)
    _burn!(deck)
    board.turn = _pop_deck!(deck)
end

function river!(deck::Deck, board::Board)
    _burn!(deck)
    board.river = _pop_deck!(deck)
end

function _burn!(deck::Deck)
    _pop_deck!(deck)
end

function _pop_deck!(deck::Deck)
    card = deck.cards[deck.top]
    deck.top += 1
    return card
end


function FieldVector2D(arg0::Field, arg1::Vector2D)
    return FieldVector2D((Field, Vector2D), arg0, arg1)
end

function FieldVector2D(arg0::RealFieldElement, arg1::FieldVector2D)
    return FieldVector2D((RealFieldElement, FieldVector2D), arg0, arg1)
end

function FieldVector2D(arg0::RealFieldElement, arg1::FieldVector2D, arg2::RealFieldElement, arg3::FieldVector2D)
    return FieldVector2D((RealFieldElement, FieldVector2D, RealFieldElement, FieldVector2D), arg0, arg1, arg2, arg3)
end

function FieldVector2D(arg0::RealFieldElement, arg1::FieldVector2D, arg2::RealFieldElement, arg3::FieldVector2D, arg4::RealFieldElement, arg5::FieldVector2D)
    return FieldVector2D((RealFieldElement, FieldVector2D, RealFieldElement, FieldVector2D, RealFieldElement, FieldVector2D), arg0, arg1, arg2, arg3, arg4, arg5)
end

function FieldVector2D(arg0::RealFieldElement, arg1::FieldVector2D, arg2::RealFieldElement, arg3::FieldVector2D, arg4::RealFieldElement, arg5::FieldVector2D, arg6::RealFieldElement, arg7::FieldVector2D)
    return FieldVector2D((RealFieldElement, FieldVector2D, RealFieldElement, FieldVector2D, RealFieldElement, FieldVector2D, RealFieldElement, FieldVector2D), arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7)
end

function FieldVector2D(arg0::RealFieldElement, arg1::RealFieldElement)
    return FieldVector2D((RealFieldElement, RealFieldElement), arg0, arg1)
end

function FieldVector2D(arg0::RealFieldElement, arg1::Vector2D)
    return FieldVector2D((RealFieldElement, Vector2D), arg0, arg1)
end

function FieldVector2D(arg0::RealFieldElement, arg1::Vector2D, arg2::RealFieldElement, arg3::Vector2D)
    return FieldVector2D((RealFieldElement, Vector2D, RealFieldElement, Vector2D), arg0, arg1, arg2, arg3)
end

function FieldVector2D(arg0::RealFieldElement, arg1::Vector2D, arg2::RealFieldElement, arg3::Vector2D, arg4::RealFieldElement, arg5::Vector2D)
    return FieldVector2D((RealFieldElement, Vector2D, RealFieldElement, Vector2D, RealFieldElement, Vector2D), arg0, arg1, arg2, arg3, arg4, arg5)
end

function FieldVector2D(arg0::RealFieldElement, arg1::Vector2D, arg2::RealFieldElement, arg3::Vector2D, arg4::RealFieldElement, arg5::Vector2D, arg6::RealFieldElement, arg7::Vector2D)
    return FieldVector2D((RealFieldElement, Vector2D, RealFieldElement, Vector2D, RealFieldElement, Vector2D, RealFieldElement, Vector2D), arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7)
end

function FieldVector2D(arg0::Vector{RealFieldElement})
    return FieldVector2D((Vector{RealFieldElement},), arg0)
end

function FieldVector2D(arg0::jdouble, arg1::FieldVector2D)
    return FieldVector2D((jdouble, FieldVector2D), arg0, arg1)
end

function FieldVector2D(arg0::jdouble, arg1::FieldVector2D, arg2::jdouble, arg3::FieldVector2D)
    return FieldVector2D((jdouble, FieldVector2D, jdouble, FieldVector2D), arg0, arg1, arg2, arg3)
end

function FieldVector2D(arg0::jdouble, arg1::FieldVector2D, arg2::jdouble, arg3::FieldVector2D, arg4::jdouble, arg5::FieldVector2D)
    return FieldVector2D((jdouble, FieldVector2D, jdouble, FieldVector2D, jdouble, FieldVector2D), arg0, arg1, arg2, arg3, arg4, arg5)
end

function FieldVector2D(arg0::jdouble, arg1::FieldVector2D, arg2::jdouble, arg3::FieldVector2D, arg4::jdouble, arg5::FieldVector2D, arg6::jdouble, arg7::FieldVector2D)
    return FieldVector2D((jdouble, FieldVector2D, jdouble, FieldVector2D, jdouble, FieldVector2D, jdouble, FieldVector2D), arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7)
end

function add(obj::FieldVector2D, arg0::FieldVector2D)
    return jcall(obj, "add", FieldVector2D, (FieldVector2D,), arg0)
end

function add(obj::FieldVector2D, arg0::RealFieldElement, arg1::FieldVector2D)
    return jcall(obj, "add", FieldVector2D, (RealFieldElement, FieldVector2D), arg0, arg1)
end

function add(obj::FieldVector2D, arg0::RealFieldElement, arg1::Vector2D)
    return jcall(obj, "add", FieldVector2D, (RealFieldElement, Vector2D), arg0, arg1)
end

function add(obj::FieldVector2D, arg0::Vector2D)
    return jcall(obj, "add", FieldVector2D, (Vector2D,), arg0)
end

function add(obj::FieldVector2D, arg0::jdouble, arg1::FieldVector2D)
    return jcall(obj, "add", FieldVector2D, (jdouble, FieldVector2D), arg0, arg1)
end

function add(obj::FieldVector2D, arg0::jdouble, arg1::Vector2D)
    return jcall(obj, "add", FieldVector2D, (jdouble, Vector2D), arg0, arg1)
end

function angle(::Type{FieldVector2D}, arg0::FieldVector2D, arg1::FieldVector2D)
    return jcall(FieldVector2D, "angle", RealFieldElement, (FieldVector2D, FieldVector2D), arg0, arg1)
end

function angle(::Type{FieldVector2D}, arg0::FieldVector2D, arg1::Vector2D)
    return jcall(FieldVector2D, "angle", RealFieldElement, (FieldVector2D, Vector2D), arg0, arg1)
end

function angle(::Type{FieldVector2D}, arg0::Vector2D, arg1::FieldVector2D)
    return jcall(FieldVector2D, "angle", RealFieldElement, (Vector2D, FieldVector2D), arg0, arg1)
end

function cross_product(obj::FieldVector2D, arg0::FieldVector2D, arg1::FieldVector2D)
    return jcall(obj, "crossProduct", RealFieldElement, (FieldVector2D, FieldVector2D), arg0, arg1)
end

function cross_product(obj::FieldVector2D, arg0::Vector2D, arg1::Vector2D)
    return jcall(obj, "crossProduct", RealFieldElement, (Vector2D, Vector2D), arg0, arg1)
end

function distance(::Type{FieldVector2D}, arg0::FieldVector2D, arg1::FieldVector2D)
    return jcall(FieldVector2D, "distance", RealFieldElement, (FieldVector2D, FieldVector2D), arg0, arg1)
end

function distance(::Type{FieldVector2D}, arg0::FieldVector2D, arg1::Vector2D)
    return jcall(FieldVector2D, "distance", RealFieldElement, (FieldVector2D, Vector2D), arg0, arg1)
end

function distance(::Type{FieldVector2D}, arg0::Vector2D, arg1::FieldVector2D)
    return jcall(FieldVector2D, "distance", RealFieldElement, (Vector2D, FieldVector2D), arg0, arg1)
end

function distance(obj::FieldVector2D, arg0::FieldVector2D)
    return jcall(obj, "distance", RealFieldElement, (FieldVector2D,), arg0)
end

function distance(obj::FieldVector2D, arg0::Vector2D)
    return jcall(obj, "distance", RealFieldElement, (Vector2D,), arg0)
end

function distance1(::Type{FieldVector2D}, arg0::FieldVector2D, arg1::FieldVector2D)
    return jcall(FieldVector2D, "distance1", RealFieldElement, (FieldVector2D, FieldVector2D), arg0, arg1)
end

function distance1(::Type{FieldVector2D}, arg0::FieldVector2D, arg1::Vector2D)
    return jcall(FieldVector2D, "distance1", RealFieldElement, (FieldVector2D, Vector2D), arg0, arg1)
end

function distance1(::Type{FieldVector2D}, arg0::Vector2D, arg1::FieldVector2D)
    return jcall(FieldVector2D, "distance1", RealFieldElement, (Vector2D, FieldVector2D), arg0, arg1)
end

function distance1(obj::FieldVector2D, arg0::FieldVector2D)
    return jcall(obj, "distance1", RealFieldElement, (FieldVector2D,), arg0)
end

function distance1(obj::FieldVector2D, arg0::Vector2D)
    return jcall(obj, "distance1", RealFieldElement, (Vector2D,), arg0)
end

function distance_inf(::Type{FieldVector2D}, arg0::FieldVector2D, arg1::FieldVector2D)
    return jcall(FieldVector2D, "distanceInf", RealFieldElement, (FieldVector2D, FieldVector2D), arg0, arg1)
end

function distance_inf(::Type{FieldVector2D}, arg0::FieldVector2D, arg1::Vector2D)
    return jcall(FieldVector2D, "distanceInf", RealFieldElement, (FieldVector2D, Vector2D), arg0, arg1)
end

function distance_inf(::Type{FieldVector2D}, arg0::Vector2D, arg1::FieldVector2D)
    return jcall(FieldVector2D, "distanceInf", RealFieldElement, (Vector2D, FieldVector2D), arg0, arg1)
end

function distance_inf(obj::FieldVector2D, arg0::FieldVector2D)
    return jcall(obj, "distanceInf", RealFieldElement, (FieldVector2D,), arg0)
end

function distance_inf(obj::FieldVector2D, arg0::Vector2D)
    return jcall(obj, "distanceInf", RealFieldElement, (Vector2D,), arg0)
end

function distance_sq(::Type{FieldVector2D}, arg0::FieldVector2D, arg1::FieldVector2D)
    return jcall(FieldVector2D, "distanceSq", RealFieldElement, (FieldVector2D, FieldVector2D), arg0, arg1)
end

function distance_sq(::Type{FieldVector2D}, arg0::FieldVector2D, arg1::Vector2D)
    return jcall(FieldVector2D, "distanceSq", RealFieldElement, (FieldVector2D, Vector2D), arg0, arg1)
end

function distance_sq(::Type{FieldVector2D}, arg0::Vector2D, arg1::FieldVector2D)
    return jcall(FieldVector2D, "distanceSq", RealFieldElement, (Vector2D, FieldVector2D), arg0, arg1)
end

function distance_sq(obj::FieldVector2D, arg0::FieldVector2D)
    return jcall(obj, "distanceSq", RealFieldElement, (FieldVector2D,), arg0)
end

function distance_sq(obj::FieldVector2D, arg0::Vector2D)
    return jcall(obj, "distanceSq", RealFieldElement, (Vector2D,), arg0)
end

function dot_product(obj::FieldVector2D, arg0::FieldVector2D)
    return jcall(obj, "dotProduct", RealFieldElement, (FieldVector2D,), arg0)
end

function dot_product(obj::FieldVector2D, arg0::Vector2D)
    return jcall(obj, "dotProduct", RealFieldElement, (Vector2D,), arg0)
end

function equals(obj::FieldVector2D, arg0::Object)
    return jcall(obj, "equals", jboolean, (Object,), arg0)
end

function get_minus_i(::Type{FieldVector2D}, arg0::Field)
    return jcall(FieldVector2D, "getMinusI", FieldVector2D, (Field,), arg0)
end

function get_minus_j(::Type{FieldVector2D}, arg0::Field)
    return jcall(FieldVector2D, "getMinusJ", FieldVector2D, (Field,), arg0)
end

function get_na_n(::Type{FieldVector2D}, arg0::Field)
    return jcall(FieldVector2D, "getNaN", FieldVector2D, (Field,), arg0)
end

function get_negative_infinity(::Type{FieldVector2D}, arg0::Field)
    return jcall(FieldVector2D, "getNegativeInfinity", FieldVector2D, (Field,), arg0)
end

function get_norm(obj::FieldVector2D)
    return jcall(obj, "getNorm", RealFieldElement, ())
end

function get_norm1(obj::FieldVector2D)
    return jcall(obj, "getNorm1", RealFieldElement, ())
end

function get_norm_inf(obj::FieldVector2D)
    return jcall(obj, "getNormInf", RealFieldElement, ())
end

function get_norm_sq(obj::FieldVector2D)
    return jcall(obj, "getNormSq", RealFieldElement, ())
end

function get_plus_i(::Type{FieldVector2D}, arg0::Field)
    return jcall(FieldVector2D, "getPlusI", FieldVector2D, (Field,), arg0)
end

function get_plus_j(::Type{FieldVector2D}, arg0::Field)
    return jcall(FieldVector2D, "getPlusJ", FieldVector2D, (Field,), arg0)
end

function get_positive_infinity(::Type{FieldVector2D}, arg0::Field)
    return jcall(FieldVector2D, "getPositiveInfinity", FieldVector2D, (Field,), arg0)
end

function get_x(obj::FieldVector2D)
    return jcall(obj, "getX", RealFieldElement, ())
end

function get_y(obj::FieldVector2D)
    return jcall(obj, "getY", RealFieldElement, ())
end

function get_zero(::Type{FieldVector2D}, arg0::Field)
    return jcall(FieldVector2D, "getZero", FieldVector2D, (Field,), arg0)
end

function hash_code(obj::FieldVector2D)
    return jcall(obj, "hashCode", jint, ())
end

function is_infinite(obj::FieldVector2D)
    return jcall(obj, "isInfinite", jboolean, ())
end

function is_na_n(obj::FieldVector2D)
    return jcall(obj, "isNaN", jboolean, ())
end

function negate(obj::FieldVector2D)
    return jcall(obj, "negate", FieldVector2D, ())
end

function normalize(obj::FieldVector2D)
    return jcall(obj, "normalize", FieldVector2D, ())
end

function orientation(::Type{FieldVector2D}, arg0::FieldVector2D, arg1::FieldVector2D, arg2::FieldVector2D)
    return jcall(FieldVector2D, "orientation", RealFieldElement, (FieldVector2D, FieldVector2D, FieldVector2D), arg0, arg1, arg2)
end

function scalar_multiply(obj::FieldVector2D, arg0::RealFieldElement)
    return jcall(obj, "scalarMultiply", FieldVector2D, (RealFieldElement,), arg0)
end

function scalar_multiply(obj::FieldVector2D, arg0::jdouble)
    return jcall(obj, "scalarMultiply", FieldVector2D, (jdouble,), arg0)
end

function subtract(obj::FieldVector2D, arg0::FieldVector2D)
    return jcall(obj, "subtract", FieldVector2D, (FieldVector2D,), arg0)
end

function subtract(obj::FieldVector2D, arg0::RealFieldElement, arg1::FieldVector2D)
    return jcall(obj, "subtract", FieldVector2D, (RealFieldElement, FieldVector2D), arg0, arg1)
end

function subtract(obj::FieldVector2D, arg0::RealFieldElement, arg1::Vector2D)
    return jcall(obj, "subtract", FieldVector2D, (RealFieldElement, Vector2D), arg0, arg1)
end

function subtract(obj::FieldVector2D, arg0::Vector2D)
    return jcall(obj, "subtract", FieldVector2D, (Vector2D,), arg0)
end

function subtract(obj::FieldVector2D, arg0::jdouble, arg1::FieldVector2D)
    return jcall(obj, "subtract", FieldVector2D, (jdouble, FieldVector2D), arg0, arg1)
end

function subtract(obj::FieldVector2D, arg0::jdouble, arg1::Vector2D)
    return jcall(obj, "subtract", FieldVector2D, (jdouble, Vector2D), arg0, arg1)
end

function to_array(obj::FieldVector2D)
    return jcall(obj, "toArray", Vector{RealFieldElement}, ())
end

function to_string(obj::FieldVector2D)
    return jcall(obj, "toString", JString, ())
end

function to_string(obj::FieldVector2D, arg0::NumberFormat)
    return jcall(obj, "toString", JString, (NumberFormat,), arg0)
end

function to_vector2d(obj::FieldVector2D)
    return jcall(obj, "toVector2D", Vector2D, ())
end


using .Networks

using Printf
using Dates
using Parameters
using DataStructures

@enum GeneratorState begin
    ON
    OFF
end

function Base.float(generator_state::GeneratorState)
    if generator_state == ON
        return 1.
    elseif generator_state == OFF
        return 0.
    else
        throw( error("Unknown GeneratorState value : ", generator_state) )
    end
end

function Base.parse(::Type{GeneratorState}, str::String)
    if lowercase(str) == "on"
        return ON
    elseif  lowercase(str) == "off"
        return OFF
    else
        throw( error("Unable to convert `", str, "` to a GeneratorState") )
    end
end

function Base.parse(::Type{GeneratorState}, val::Float64)
    if val > 1e-09
        return ON
    else
        return OFF
    end
end

##########################################
## Firmness
##########################################

@enum DecisionFirmness begin
    DECIDED # a firm decision was already taken
    TO_DECIDE # a firm decision needs to be taken
    FREE # by scenario decisions
end

@with_kw_noshow struct Firmness
    #gen,ts
    commitment::SortedDict{String, SortedDict{Dates.DateTime, DecisionFirmness} } =
        SortedDict{String, SortedDict{Dates.DateTime, DecisionFirmness} }()
    power_level::SortedDict{String, SortedDict{Dates.DateTime, DecisionFirmness} } =
        SortedDict{String, SortedDict{Dates.DateTime, DecisionFirmness} }()
end

Base.:(==)(a::Firmness, b::Firmness) = a.commitment==b.commitment && a.power_level==b.power_level

function get_commitment_firmness(firmness::Firmness)
    return firmness.commitment
end

function get_power_level_firmness(firmness::Firmness)
    return firmness.power_level
end

function set_commitment_firmness!(firmness::Firmness, gen_id::String, ts, decision_firmness::DecisionFirmness)
    get!(firmness.commitment, gen_id, SortedDict{Dates.DateTime, DecisionFirmness}())
    firmness.commitment[gen_id][ts] = decision_firmness
end

function set_power_level_firmness!(firmness::Firmness, gen_id::String, ts, decision_firmness::DecisionFirmness)
    get!(firmness.power_level, gen_id, SortedDict{Dates.DateTime, DecisionFirmness}())
    firmness.power_level[gen_id][ts] = decision_firmness
end

function get_commitment_firmness(firmness::Firmness, gen_id::String)
    if haskey(firmness.commitment,gen_id)
        return firmness.commitment[gen_id]
    else
        return missing
    end
end

function get_commitment_firmness(firmness::Firmness, gen_id::String, ts::Dates.DateTime)
    if haskey(firmness.commitment,gen_id)
        if haskey(firmness.commitment[gen_id], ts)
            return firmness.commitment[gen_id][ts]
        else
            return missing
        end
    else
        return missing
    end
end

function get_power_level_firmness(firmness::Firmness, gen_id::String)
    if haskey(firmness.power_level,gen_id)
        return firmness.power_level[gen_id]
    else
        return missing
    end
end

function get_power_level_firmness(firmness::Firmness, gen_id::String, ts::Dates.DateTime)
    if haskey(firmness.power_level,gen_id)
        if haskey(firmness.power_level[gen_id], ts)
            return firmness.power_level[gen_id][ts]
        else
            return missing
        end
    else
        return missing
    end
end

function Base.show(io::IO, firmness::Firmness)
    println(io, "commitment :")
    pretty_print(io, firmness.commitment)
    println(io, "power_level :")
    pretty_print(io, firmness.power_level)
end

##########################################
## Schedule
##########################################

struct GeneratorSchedule
    gen_id::String
    commitment::SortedDict{Dates.DateTime, UncertainValue{GeneratorState}}
    production::SortedDict{Dates.DateTime, UncertainValue{Float64}}
end
function GeneratorSchedule(gen_id::String)
    return  GeneratorSchedule(gen_id,
                            SortedDict{Dates.DateTime, UncertainValue{GeneratorState}}(),
                            SortedDict{Dates.DateTime, UncertainValue{Float64}}())
end

function get_commitment(generator_schedule::GeneratorSchedule)
    return generator_schedule.commitment
end

function get_production(generator_schedule::GeneratorSchedule)
    return generator_schedule.production
end

mutable struct Schedule <: AbstractSchedule
    decider_type::DeciderType
    decision_time::Dates.DateTime
    #gen_id => ts => uncertainValue(s)
    generator_schedules::SortedDict{String, GeneratorSchedule }

    #bus,ts,s
    loss_of_load_by_bus::SortedDict{Tuple{String,DateTime,String}, Float64 }
    #limitable_gen_id,ts,s
    capping::SortedDict{Tuple{String,DateTime,String}, Float64 }
end
function Schedule(decider_type, ech)
    return Schedule(decider_type, ech,
                    SortedDict{String, GeneratorSchedule}(),
                    SortedDict{Tuple{String,DateTime,String}, Float64 }(),
                    SortedDict{Tuple{String,DateTime,String}, Float64 }()
                    )
end
function Schedule(decider_type, ech, generator_schedules)
    return Schedule(decider_type, ech, generator_schedules,
                    SortedDict{Tuple{String,DateTime,String}, Float64 }(),
                    SortedDict{Tuple{String,DateTime,String}, Float64 }()
                    )
end

function init!(schedule::Schedule, network::Networks.Network,
            target_timepoints::Vector{Dates.DateTime}, scenarios::Vector{String})
    empty!(schedule.generator_schedules)
    for generator_l in Networks.get_generators(network)
        gen_id = Networks.get_id(generator_l)
        schedule.generator_schedules[gen_id] = GeneratorSchedule(gen_id)
        for ts in target_timepoints

            schedule.generator_schedules[gen_id].production[ts] = UncertainValue{Float64}(scenarios)

            #commitment values are only defined for generators that have a pmin > 0
            if (Networks.get_p_min(generator_l) > 1e-09)
                schedule.generator_schedules[gen_id].commitment[ts] = UncertainValue{GeneratorState}(scenarios)
            end
        end
    end
end

function get_sub_schedule(schedule::Schedule, gen_id::String)::Union{GeneratorSchedule,Missing}
    if haskey(schedule.generator_schedules, gen_id)
        return schedule.generator_schedules[gen_id]
    else
        throw( error("no generator schedule was initialized for generator ", gen_id))
        #return missing
    end
end

function get_prod_uncertain_value(sub_schedule::GeneratorSchedule, ts::Dates.DateTime)::Union{UncertainValue{Float64},Missing}
    if haskey(sub_schedule.production, ts)
        return sub_schedule.production[ts]
    else
        @warn("power level schedule is not defined for generator ", sub_schedule.gen_id)
        return missing
    end
end
function get_prod_uncertain_value(schedule::Schedule, gen_id::String, ts::Dates.DateTime)::Union{UncertainValue{Float64},Missing}
    return get_prod_uncertain_value(schedule.generator_schedules[gen_id], ts)
end

function get_commitment_uncertain_value(sub_schedule::GeneratorSchedule, ts::Dates.DateTime)::Union{UncertainValue{GeneratorState},Missing}
    if haskey(sub_schedule.commitment, ts)
        return sub_schedule.commitment[ts]
    else
        # @warn("commitment schedule is not defined for generator ", sub_schedule.gen_id)
        return missing
    end
end
function get_commitment_uncertain_value(schedule::Schedule, gen_id::String, ts::Dates.DateTime)::Union{UncertainValue{GeneratorState},Missing}
    return get_commitment_uncertain_value(schedule.generator_schedules[gen_id], ts)
end

function get_prod_value(sub_schedule::GeneratorSchedule, ts::Dates.DateTime)::Union{Float64, Missing}
    uncertain_value = get_prod_uncertain_value(sub_schedule, ts)
    if ismissing(uncertain_value)
        return missing
    end
    return get_value(uncertain_value)
end
function get_prod_value(schedule::Schedule, gen_id::String, ts::Dates.DateTime)::Union{Float64, Missing}
    return get_prod_value(schedule.generator_schedules[gen_id], ts)
end

function safeget_prod_value(schedule::Schedule, gen_id::String, ts::Dates.DateTime)::Float64
    return safeget_prod_value(schedule.generator_schedules[gen_id], ts)
end
function safeget_prod_value(sub_schedule::GeneratorSchedule, ts::Dates.DateTime)::Float64
    prod = get_prod_value(sub_schedule, ts)
    if ismissing(prod)
        msg = @sprintf("Missing production value for ts=%s in schedule %s",
                        ts, sub_schedule.gen_id)
        throw( error(msg) )
    else
        return prod
    end
end

function get_commitment_value(sub_schedule::GeneratorSchedule, ts::Dates.DateTime)::Union{GeneratorState, Missing}
    uncertain_value = get_commitment_uncertain_value(sub_schedule, ts)
    if ismissing(uncertain_value)
        return missing
    end
    return get_value(uncertain_value)
end
function get_commitment_value(schedule::Schedule, gen_id::String, ts::Dates.DateTime)::Union{GeneratorState, Missing}
    return get_commitment_value(schedule.generator_schedules[gen_id], ts)
end

function safeget_commitment_value(schedule::Schedule, gen_id::String, ts::Dates.DateTime)::GeneratorState
    return safeget_commitment_value(schedule.generator_schedules[gen_id], ts)
end
function safeget_commitment_value(sub_schedule::GeneratorSchedule, ts::Dates.DateTime)::GeneratorState
    commitment = get_commitment_value(sub_schedule, ts)
    if ismissing(commitment)
        msg = @sprintf("Missing commitment value for ts=%s in schedule %s",
                        ts, sub_schedule.gen_id)
        throw( error(msg) )
    else
        return commitment
    end
end

function get_prod_value(sub_schedule::GeneratorSchedule, ts::Dates.DateTime, scenario::String)::Union{Float64, Missing}
    uncertain_value = get_prod_uncertain_value(sub_schedule, ts)
    if ismissing(uncertain_value)
        return missing
    end
    return get_value(uncertain_value, scenario)
end
function get_prod_value(schedule::Schedule, gen_id::String, ts::Dates.DateTime, scenario::String)::Union{Float64, Missing}
    return get_prod_value(schedule.generator_schedules[gen_id], ts, scenario)
end
function safeget_prod_value(schedule::Schedule, gen_id::String, ts::Dates.DateTime, scenario::String)::Union{Float64, Missing}
    prod = get_prod_value(schedule.generator_schedules[gen_id], ts, scenario)
    if ismissing(prod)
        msg = @sprintf("Missing production value in schedule for (gen_id=%s,ts=%s,s=%s)",
                        gen_id,ts,scenario)
        throw( error(msg) )
    else
        return prod
    end
end
function safeget_prod_value(sub_schedule::GeneratorSchedule, ts::Dates.DateTime, scenario::String)::Float64
    prod = get_prod_value(sub_schedule, ts, scenario)
    if ismissing(prod)
        msg = @sprintf("Missing production value for (ts=%s,s=%s) in schedule %s",
                        ts, scenario, sub_schedule.gen_id)
        throw( error(msg) )
    else
        return prod
    end
end

function get_commitment_value(sub_schedule::GeneratorSchedule, ts::Dates.DateTime, scenario::String)::Union{GeneratorState, Missing}
    uncertain_value = get_commitment_uncertain_value(sub_schedule, ts)
    if ismissing(uncertain_value)
        return missing
    end
    return get_value(uncertain_value, scenario)
end
function get_commitment_value(schedule::Schedule, gen_id::String, ts::Dates.DateTime, scenario::String)::Union{GeneratorState, Missing}
    return get_commitment_value(schedule.generator_schedules[gen_id], ts, scenario)
end

function get_capping(schedule::Schedule, gen_id::String, ts::Dates.DateTime, scenario)::Union{Float64, Missing}
    key_l = (gen_id, ts, scenario)
    if !haskey(schedule.capping, key_l)
        return missing
    else
        return schedule.capping[key_l]
    end
end
function safeget_capping(schedule::Schedule, gen_id::String, ts::Dates.DateTime, scenario)::Float64
    capping = get_capping(schedule, gen_id, ts, scenario)
    if ismissing(capping)
        msg = @sprintf("Missing capping value for (gen_id=%s,ts=%s,s=%s) in schedule",
                        gen_id, ts, scenario)
        throw( error(msg) )
    else
        return capping
    end
end

function get_loss_of_load(schedule::Schedule, bus_id, ts::Dates.DateTime, scenario)::Union{Float64, Missing}
    key_l = (bus_id, ts, scenario)
    if !haskey(schedule.loss_of_load_by_bus, key_l)
        return missing
    else
        return schedule.loss_of_load_by_bus[key_l]
    end
end
function safeget_loss_of_load(schedule::Schedule, bus_id, ts::Dates.DateTime, scenario)::Float64
    loss_of_load = get_loss_of_load(schedule, bus_id, ts, scenario)
    if ismissing(loss_of_load)
        msg = @sprintf("Missing loss_of_load value for (gen_id=%s,ts=%s,s=%s) in schedule",
                        bus_id, ts, scenario)
        throw( error(msg) )
    else
        return loss_of_load
    end
end

function set_prod_value!(sub_schedule::GeneratorSchedule, ts::Dates.DateTime, scenario::String, value::Float64)
    uncertain_value = get_prod_uncertain_value(sub_schedule, ts)
    if ismissing(uncertain_value)
        msg = @sprintf("cannot set uninitialized schedule production value for generator `%s` at timestep %s and scenario %s",
                        sub_schedule.gen_id, ts, scenario)
        throw( error(msg) )
    end
    set_value!(uncertain_value, scenario, value)
end
function set_prod_value!(schedule, gen_id::String, ts::Dates.DateTime, scenario::String, value::Float64)
    return set_prod_value!(schedule.generator_schedules[gen_id], ts, scenario, value)
end

function set_commitment_value!(sub_schedule::GeneratorSchedule, ts::Dates.DateTime, scenario::String, value::GeneratorState)
    uncertain_value = get_commitment_uncertain_value(sub_schedule, ts)
    if ismissing(uncertain_value)
        msg = @sprintf("cannot set uninitialized schedule commitment value for generator `%s` at timestep %s and scenario %s",
                        sub_schedule.gen_id, ts, scenario)
        throw( error(msg) )
    end
    set_value!(uncertain_value, scenario, value)
end
function set_commitment_value!(schedule, gen_id::String, ts::Dates.DateTime, scenario::String, value::GeneratorState)
    return set_commitment_value!(schedule.generator_schedules[gen_id], ts, scenario, value)
end

function set_prod_definitive_value!(sub_schedule::GeneratorSchedule, ts::Dates.DateTime, value::Float64)
    uncertain_value = get_prod_uncertain_value(sub_schedule, ts)
    if ismissing(uncertain_value)
        msg = @sprintf("cannot set uninitialized schedule production value for generator `%s` at timestep %s",
                        sub_schedule.gen_id, ts)
        throw( error(msg) )
    end
    safeset_definitive_value!(uncertain_value, value)
end
function set_prod_definitive_value!(schedule, gen_id::String, ts::Dates.DateTime, value::Float64)
    return set_prod_definitive_value!(schedule.generator_schedules[gen_id], ts, value)
end

function set_commitment_definitive_value!(sub_schedule::GeneratorSchedule, ts::Dates.DateTime, value::GeneratorState)
    uncertain_value = get_commitment_uncertain_value(sub_schedule, ts)
    if ismissing(uncertain_value)
        msg = @sprintf("cannot set uninitialized schedule commitment value for generator `%s` at timestep %s",
                        sub_schedule.gen_id, ts)
        throw( error(msg) )
    end
    set_definitive_value!(uncertain_value, value)
end
function set_commitment_definitive_value!(schedule, gen_id::String, ts::Dates.DateTime, value::GeneratorState)
    return set_commitment_definitive_value!(schedule.generator_schedules[gen_id], ts, value)
end

function set_capping_value!(schedule::Schedule, gen_id::String, ts::Dates.DateTime, scenario::String, value::Float64)
    schedule.capping[gen_id, ts, scenario] = value
end
function set_loss_of_load_value!(schedule::Schedule, bus_id::String, ts::Dates.DateTime, scenario::String, value::Float64)
    schedule.loss_of_load_by_bus[bus_id, ts, scenario] = value
end

function reset_loss_of_load_by_bus!(schedule::Schedule)
    empty!(schedule.loss_of_load_by_bus)
end
function reset_capping!(schedule::Schedule)
    empty!(schedule.capping)
end

function get_commitment_sub_schedule(schedule::Schedule, gen_id::String)
    return get_commitment(get_sub_schedule(schedule, gen_id))
end

function get_production_sub_schedule(schedule::Schedule, gen_id::String)
    return get_production(get_sub_schedule(schedule, gen_id))
end

function Base.show(io::IO, gen_schedule::GeneratorSchedule)
    @printf("generator %s:\n", gen_schedule.gen_id)
    println("commitment:")
    pretty_print(io, gen_schedule.commitment)
    println("production:")
    pretty_print(io, gen_schedule.production)
end
function Base.show(io::IO, schedule::Schedule)
    @printf("schedule decided by %s at %s:\n", schedule.decider_type, schedule.decision_time)
    pretty_print(io, schedule.generator_schedules)
    println("capping:")
    pretty_print(io, schedule.capping)
    println("loss_of_load_by_bus:")
    pretty_print(io, schedule.loss_of_load_by_bus)
end

#######################
# Helpers
########################

function get_target_timepoints(gen_schedule::GeneratorSchedule)
    return union(keys(gen_schedule.commitment), keys(gen_schedule.production))
end

function get_start_value(preceding_state, current_state)
    if preceding_state==OFF && current_state==ON
        return 1
    else
        return 0
    end
end

function get_start_value(generator_reference_schedule, ts, preceding_ts, generator_initial_state)
    current_state = safeget_commitment_value(generator_reference_schedule, ts)
    preceding_state = ( isnothing(preceding_ts) ? generator_initial_state :
                            safeget_commitment_value(generator_reference_schedule, preceding_ts) )
    return get_start_value(preceding_state, current_state)
end


module RBM

#using PyPlot

export RBM_t
export serialize
export rbm_copy
export rbm_create
export rbm_create_full
export rbm_init_weights_random!
export rbm_init_visible_bias!
export rbm_init_output_bias_random!, rbm_init_hidden_bias_random!
export rbm_rescale_weights!
#= export rbm_calculate_L1, rbm_calculate_L2 =#
export rbm_write, rbm_read, rbm_read_old
export sigm
export rbm_visualise

type RBM_t
  bins::Int64
  n::Int64            # number of output nodes
  m::Int64            # number of hidden nodes
  k::Int64            # number of input nodes
  uditer::Int64       # number of up-down passes when sampling
  alpha::Float64      # learning rate
  momentum::Float64   # contribution of previous gradient
  weightcost::Float64 # weight cost for L2
  numepochs::Int64    # number of training epochs
  batchsize::Int64    # size of training data batches
  W::Matrix{Float64}  # interaction weights between hidden and outputs
  V::Matrix{Float64}  # interaction weights between hidden and inputs
  b::Vector{Float64}  # bias weights for outputs
  c::Vector{Float64}  # bias weights for hiddens
  vW::Matrix{Float64} # for the momentum
  vV::Matrix{Float64} # for the momentum
  vb::Vector{Float64} # for the momentum
  vc::Vector{Float64} # for the momentum
  dropout::Float64
  perturbation::Float64
end

#= serialize(s, rbm::RBM_t) = serialize(f, rbm) =#

function rbm_create_full(n::Int64, m::Int64, k::Int64, uditer::Int64,
  alpha::Float64, momentum::Float64, weightcost::Float64,
  numepochs::Int64, batchsize::Int64, bins::Int64, dropout::Float64,
  perturbation::Float64)

  W = zeros((m, n))
  V = zeros((m, k))
  b = zeros(n)
  c = zeros(m)

  vW = zeros((m, n))
  vV = zeros((m, k))
  vb = zeros(n)
  vc = zeros(m)

  return RBM_t(bins, n, m, k, uditer, alpha,
               momentum, weightcost, numepochs, batchsize,
               W, V, b, c, vW, vV, vb, vc, dropout, perturbation)
end

rbm_create(n::Int64, m::Int64, k::Int64) = rbm_create_full(n, m, k, 15, 0.01, 0.5, 0.001, 10000, 100, 16, 0.0, 0.1)

function rbm_copy(src::RBM_t)
  copy = rbm_create(src.n, src.m, src.k, src.uditer, src.alpha, src.momentum, src.weightcost, src.numepochs, src.batchsize, src.bins, src.dropout)
  copy.W  = deepcopy(src.W)
  copy.V  = deepcopy(src.V)
  copy.b  = deepcopy(src.b)
  copy.c  = deepcopy(src.c)
  copy.vW = deepcopy(src.vW)
  copy.vV = deepcopy(src.vV)
  copy.vb = deepcopy(src.vb)
  copy.vc = deepcopy(src.vc)
  return copy
end

function rbm_init_weights_random!(rbm::RBM_t)
  rbm.W = randn((rbm.m,rbm.n)) .* 0.01
  rbm.V = randn((rbm.m,rbm.k)) .* 0.01
end

function rbm_init_hidden_bias_random!(rbm::RBM_t)
  rbm.b = rand(rbm.n) .* 0.01
end

function rbm_init_output_bias_random!(rbm::RBM_t)
  rbm.c = rand(rbm.m) .* 0.01
end

function rbm_init_visible_bias!(rbm::RBM_t, data::Array{Int64,2})
  # each row of the data must contain be of type {0,1}^k
  n = size(data)[2]
  N = rbm.n
  @assert (size(data)[2] == rbm.n) "rbm_init_visible_bias!: Each row of the data array must be of type {0,1}^n. It has $n and not $N"

  l = size(data)[1]
  for i=1:rbm.n
    s = sum(data[:,i])
    if s == l # only 1 in data
      p =  (l-1) / l
    elseif s == 0 # no 1 in data
      p =  1 / l
    else
      p = s / l
    end
    rbm.b[i] = log2(p/(1.0 - p))
  end
end

function rbm_rescale_weights!(rbm::RBM_t, abs_maximum::Float64)
  @assert (abs_maximum > 0.0) "rbm_rescale_weights!: abs_maximum must be > 0.0"
  mw = maximum(abs(rbm.W))
  mv = maximum(abs(rbm.V))
  if mw > mv
    m = mw
  else
    m = mv
  end
  if m > abs_maximum
    factor = abs_maximum / m
    rbm.W = rbm.W .* factor
    rbm.V = rbm.V .* factor
  end
end

#= function rbm_calculate_L2(rbm::RBM_t, last::Float64) =#
  #= current = sum(rbm.W^2) + sum(rbm.V^2) =#
  #= diff    = rbm.weightcost * (current - last) * rbm.alpha =#
  #= return diff =#
#= end =#

#= function rbm_calculate_L1(rbm::RBM_t, last::Float64) =#
  #= current = sum(abs(rbm.W)) + sum(abs(rbm.V)) =#
  #diff    = rbm.weightcost * (current - last) * rbm.alpha
  #= return diff =#
#= end =#

function rbm_write(filename, rbm)
  fd = open(filename, "w")
  serialize(fd, rbm)
  close(fd)
end

function rbm_read(filename)
  fd = open(filename, "r")
  rbm = deserialize(fd)
  close(fd)
  return rbm
end

function rbm_read_old(filename)
  f        = open(filename, "r")
  s        = split(strip(readline(f)))

  bins     = int(eval(parse(s[1])))
  n        = int(eval(parse(s[2])))
  m        = int(eval(parse(s[3])))
  k        = int(eval(parse(s[4])))
  uditer   = int(eval(parse(s[5])))
  alpha    = float64(eval(parse(s[6])))
  momentum = float64(eval(parse(s[7])))
  epochs   = int(eval(parse(s[8])))
  batch    = int(eval(parse(s[9])))
  memory   = int(eval(parse(s[10])))

  println("n: ", n, " bins: ", bins, " m: ", m, " k: ", k, " uditer: ", uditer)
  rbm = rbm_create(n, m, k, uditer, alpha, momentum, 0.0, epochs, batch, bins, 0.0, 0.0)

  # reading W
  for i=1:m
    l = split(strip(readline(f)))
    for j=1:n
      rbm.W[i,j] = eval(parse(l[j]))
    end
  end
  # reading V
  for i=1:m
    l = split(strip(readline(f)))
    for j=1:k
      rbm.V[i,j]=eval(parse(l[j]))
    end
  end
  # reading b
  bs = split(strip(readline(f)))
  for i=1:length(bs)
    rbm.b[i] = eval(parse(bs[i]))
  end
  cs = split(strip(readline(f)))
  for i=1:length(cs)
    rbm.c[i] = eval(parse(cs[i]))
  end
  close(f)
  return rbm
end

function rbm_visualise(rbm::RBM_t)

end

end # module


"""
    Magic

`DataType` used for storing magic numbers, rook or bishop move masks, and respective shift and offset values for the computation of sliding moves.
"""
struct Magic
    mask::Bitboard
    num::Bitboard
    shift::UInt8
    #offset::Int32
end
Magic(mask::UInt64, num::UInt64, shift::Integer, offset::Integer) = Magic(Bitboard(mask), Bitboard(num), shift)#, offset)

const ROOK_MAGICS = @SVector [Magic(0x000101010101017e, 0x008000400020801a, 52, 0), Magic(0x000202020202027c, 0x0840004020001003, 53, 4096),
    Magic(0x000404040404047a, 0x8880200010018108, 53, 6144), Magic(0x0008080808080876, 0x0480040800801001, 53, 8192),
    Magic(0x001010101010106e, 0x0900080002050010, 53, 10240), Magic(0x002020202020205e, 0x0200100804010200, 53, 12288),
    Magic(0x004040404040403e, 0x0200010084080200, 53, 14336), Magic(0x008080808080807e, 0x0200024100821224, 52, 16384),
    Magic(0x0001010101017e00, 0x08a0802080004000, 53, 20480), Magic(0x0002020202027c00, 0x0006401004402000, 54, 22528),
    Magic(0x0004040404047a00, 0x0800802000100080, 54, 23552), Magic(0x0008080808087600, 0x4004800803100081, 54, 24576),
    Magic(0x0010101010106e00, 0x0106000810200600, 54, 25600), Magic(0x0020202020205e00, 0x2981000802040100, 54, 26624),
    Magic(0x0040404040403e00, 0x0004000804018210, 54, 27648), Magic(0x0080808080807e00, 0x0201000192026300, 53, 28672),
    Magic(0x00010101017e0100, 0x1038218000804000, 53, 30720), Magic(0x00020202027c0200, 0x1010004000200040, 54, 32768),
    Magic(0x00040404047a0400, 0x0108420012002881, 54, 33792), Magic(0x0008080808760800, 0x2098008008100080, 54, 34816),
    Magic(0x00101010106e1000, 0x4088010004090010, 54, 35840), Magic(0x00202020205e2000, 0x2000808002000400, 54, 36864),
    Magic(0x00404040403e4000, 0x028a0c0008021025, 54, 37888), Magic(0x00808080807e8000, 0x0000220000810064, 53, 38912),
    Magic(0x000101017e010100, 0x0040400480208000, 53, 40960), Magic(0x000202027c020200, 0x8040200040100042, 54, 43008),
    Magic(0x000404047a040400, 0x2000100080802000, 54, 44032), Magic(0x0008080876080800, 0x3090004040080401, 54, 45056),
    Magic(0x001010106e101000, 0x0100080080800400, 54, 46080), Magic(0x002020205e202000, 0x8230020080800400, 54, 47104),
    Magic(0x004040403e404000, 0x0002000200080104, 54, 48128), Magic(0x008080807e808000, 0x0c00010200004084, 53, 49152),
    Magic(0x0001017e01010100, 0x0000400082800020, 53, 51200), Magic(0x0002027c02020200, 0x0400200080804007, 54, 53248),
    Magic(0x0004047a04040400, 0x8400801000802001, 54, 54272), Magic(0x0008087608080800, 0x0018100080800800, 54, 55296),
    Magic(0x0010106e10101000, 0x0810310005004800, 54, 56320), Magic(0x0020205e20202000, 0x8000020080800400, 54, 57344),
    Magic(0x0040403e40404000, 0x2000302134000208, 54, 58368), Magic(0x0080807e80808000, 0x0000298402000645, 53, 59392),
    Magic(0x00017e0101010100, 0x0000800140018022, 53, 61440), Magic(0x00027c0202020200, 0x00184020100c4000, 54, 63488),
    Magic(0x00047a0404040400, 0x6210002804002000, 54, 64512), Magic(0x0008760808080800, 0x0001000c10010020, 54, 65536),
    Magic(0x00106e1010101000, 0x4201000800110004, 54, 66560), Magic(0x00205e2020202000, 0x2002000810020004, 54, 67584),
    Magic(0x00403e4040404000, 0xa40c21480a040090, 54, 68608), Magic(0x00807e8080808000, 0x0000004081020004, 53, 69632),
    Magic(0x007e010101010100, 0x3302410080002300, 53, 71680), Magic(0x007c020202020200, 0x9210002000401040, 54, 73728),
    Magic(0x007a040404040400, 0x0092110041200300, 54, 74752), Magic(0x0076080808080800, 0x0100100109002300, 54, 75776),
    Magic(0x006e101010101000, 0x000a800800240280, 54, 76800), Magic(0x005e202020202000, 0x0100040080020080, 54, 77824),
    Magic(0x003e404040404000, 0x0000506201080400, 54, 78848), Magic(0x007e808080808000, 0x1002209110440600, 53, 79872),
    Magic(0x7e01010101010100, 0x0140800300182241, 52, 81920), Magic(0x7c02020202020200, 0x8280184280220102, 53, 86016),
    Magic(0x7a04040404040400, 0x0005118042000a22, 53, 88064), Magic(0x7608080808080800, 0x008a000890204006, 53, 90112),
    Magic(0x6e10101010101000, 0x000200304844204a, 53, 92160), Magic(0x5e20202020202000, 0x108200080930041a, 53, 94208),
    Magic(0x3e40404040404000, 0x0482000100840842, 53, 96256), Magic(0x7e80808080808000, 0x0012192404430182, 52, 98304)]

const BISHOP_MAGICS = @SVector [Magic(0x0040201008040200, 0x4404700420508600, 58, 0), Magic(0x0000402010080400, 0x9120018401104008, 59, 64),
    Magic(0x0000004020100a00, 0x4004010401100084, 59, 96), Magic(0x0000000040221400, 0x0051040080062000, 59, 128),
    Magic(0x0000000002442800, 0x1124042010010092, 59, 160), Magic(0x0000000204085000, 0x1100882108001000, 59, 192),
    Magic(0x0000020408102000, 0x180a280248040000, 59, 224), Magic(0x0002040810204000, 0x0038440210900400, 58, 256),
    Magic(0x0020100804020000, 0x80074210e1010301, 59, 320), Magic(0x0040201008040000, 0x0020101012004e42, 59, 352),
    Magic(0x00004020100a0000, 0x8210100110411001, 59, 384), Magic(0x0000004022140000, 0x00008404108010a0, 59, 416),
    Magic(0x0000000244280000, 0x2060071040002804, 59, 448), Magic(0x0000020408500000, 0x6400020202211011, 59, 480),
    Magic(0x0002040810200000, 0x000c0d0111202000, 59, 512), Magic(0x0004081020400000, 0x8004050118020221, 59, 544),
    Magic(0x0010080402000200, 0x8005804088080910, 59, 576), Magic(0x0020100804000400, 0x6004a82001020200, 59, 608),
    Magic(0x004020100a000a00, 0x0101001004002041, 57, 640), Magic(0x0000402214001400, 0x8a1800a082004282, 57, 768),
    Magic(0x0000024428002800, 0x8806001012100098, 57, 896), Magic(0x0002040850005000, 0x0022810040504000, 57, 1024),
    Magic(0x0004081020002000, 0x4040800048480801, 59, 1152), Magic(0x0008102040004000, 0x0a42041304824120, 59, 1184),
    Magic(0x0008040200020400, 0x0204106085200814, 59, 1216), Magic(0x0010080400040800, 0x0890110682040111, 59, 1248),
    Magic(0x0020100a000a1000, 0x4000500001040080, 57, 1280), Magic(0x0040221400142200, 0x0000808018020102, 55, 1408),
    Magic(0x0002442800284400, 0x2401010010104008, 55, 1920), Magic(0x0004085000500800, 0x1052008004100080, 57, 2432),
    Magic(0x0008102000201000, 0x0000950104090801, 59, 2560), Magic(0x0010204000402000, 0x1200808000220864, 59, 2592),
    Magic(0x0004020002040800, 0x8188421201082050, 59, 2624), Magic(0x0008040004081000, 0x0035011001a05c00, 59, 2656),
    Magic(0x00100a000a102000, 0x0051480201104400, 57, 2688), Magic(0x0022140014224000, 0x4110040400180210, 55, 2816),
    Magic(0x0044280028440200, 0x0414140400001100, 55, 3328), Magic(0x0008500050080400, 0x4e60008100688040, 57, 3840),
    Magic(0x0010200020100800, 0x000121022ac40200, 59, 3968), Magic(0x0020400040201000, 0x00040042904a0094, 59, 4000),
    Magic(0x0002000204081000, 0x2948010421001040, 59, 4032), Magic(0x0004000408102000, 0x4060821011020200, 59, 4064),
    Magic(0x000a000a10204000, 0x0001004030008202, 57, 4096), Magic(0x0014001422400000, 0x0018020122080401, 57, 4224),
    Magic(0x0028002844020000, 0x0042ca41a2000400, 57, 4352), Magic(0x0050005008040200, 0x01c0212040810100, 57, 4480),
    Magic(0x0020002010080400, 0x1060480f41020041, 59, 4608), Magic(0x0040004020100800, 0x020102020a040040, 59, 4640),
    Magic(0x0000020408102000, 0x1082008220100402, 59, 4672), Magic(0x0000040810204000, 0x0102020202828102, 59, 4704),
    Magic(0x00000a1020400000, 0x4022108848080004, 59, 4736), Magic(0x0000142240000000, 0x0020400508480000, 59, 4768),
    Magic(0x0000284402000000, 0x1010108590440010, 59, 4800), Magic(0x0000500804020000, 0xd812040810a10140, 59, 4832),
    Magic(0x0000201008040200, 0x0191043000a20038, 59, 4864), Magic(0x0000402010080400, 0x8010020200620000, 59, 4896),
    Magic(0x0002040810204000, 0x0001008801084280, 58, 4928), Magic(0x0004081020400000, 0x0000060882015050, 59, 4992),
    Magic(0x000a102040000000, 0x8120000100809001, 59, 5024), Magic(0x0014224000000000, 0x0200100002104420, 59, 5056),
    Magic(0x0028440200000000, 0x10d100200821010a, 59, 5088), Magic(0x0050080402000000, 0x03002c0448100110, 59, 5120),
    Magic(0x0020100804020000, 0x14004005045c0042, 59, 5152), Magic(0x0040201008040200, 0x4002901008831040, 58, 5184)]

function subindex(occupied::Bitboard, magics::Magic)
    (((occupied & magics.mask).val * magics.num.val) >> magics.shift) + 1
end

# obtain the table index using the shift, occupancy mask, and magic number. Adding the offset.
function tableindex(occupied::Bitboard, magics::Magic)
    subindex(occupied, magics) + magics.offset
end

# generate all possible sliding moves in a given move direction, for a given occupancy.
function slidermoves(sqr::Bitboard, occupied::Bitboard, directions::SArray{Tuple{4},Function,1,4})
    mask = EMPTY
    for direction in directions
    	newsqr = sqr
		while !isempty(newsqr)
			newsqr = direction(newsqr)
    		mask ⊻= newsqr
    		!isempty(newsqr & occupied) && break
    	end
    end
    return mask
end

# initiate the attacks table for rook and bishop pieces
function initSlidingTable(table::Vector{Vector{Bitboard}}, magics::SArray{Tuple{64}, Magic, 1, 64}, directions::SArray{Tuple{4},Function,1,4})
	@inbounds for sqr in 1:64
		occupied = EMPTY
		table[sqr] = Vector{Bitboard}(undef, 2^count(magics[sqr].mask))
		for i in 1:(1 << count(magics[sqr].mask))
			idx = subindex(occupied, magics[sqr])
			table[sqr][idx] = slidermoves(Bitboard(sqr), occupied, directions)
			occupied = Bitboard(occupied.val - magics[sqr].mask.val) & magics[sqr].mask
		end
	end
	return table
end