# Functions to calculate an ethnicity estimate.

using CSV
using DataFrames
using DelimitedFiles
using Printf

const DEBUG = true

const CHROMOSOMES = 22

# Length of the chromosome segments that are used for mapping.
const SEGMENT_LENGTH = 1000000

# Matching segments that are larger than MAX_MATCH_LENGTH are
# excluded from the evaluation because they indicate close family
# members. We do not want them to obfuscate the results..
const MAX_MATCH_LENGTH = 20000000

# Sizes of the different chromosomes. We use larger numbers here
# because chromosome sizes can change with different genome reference versions.
const CHROMOSOME_SIZES = [250000000, 250000000, 200000000, 200000000, 190000000,
                   180000000, 160000000, 150000000, 150000000, 140000000,
                   140000000, 140000000, 120000000, 110000000, 110000000,
                   100000000,  90000000,  90000000,  60000000,  70000000,
                    50000000,  60000000]

"""
    ethnicity(matches, segments, [parent_matches,] birth_country="", excludes=["USA", "Canada", "Australia"])

Calculate an ethnicity estimate from a MyHeritage "DNA matches
list" and the corresponding "shared DNA segments" list.
If a birth country is provided the calculation will be more accurate.
You can specify a number of countries that are excluded from the calculation
because of massive recent migrations.

Important: The CSV files must be in MyHeritage's English file format!

# Examples
```
julia> ethnicity("matches.csv", "shared_segments.csv")
Total: 1311 segments
        Germany  465  35.5%
        Denmark  336  25.6%
    Netherlands  206  15.7%

julia> ethnicity("matches.csv", "shared_segments.csv", birth_country="Germany")
Total: 1124 segments
        Germany  710  63.2%
        Denmark  164  14.6%
    Netherlands  106   9.4%
```
"""
function ethnicity end

function ethnicity(matches_file::AbstractString,
                   segments_file::AbstractString;
                   birth_country = "",
                   excludes=["USA", "Canada", "Australia"])
    local matches
    local segments

    # Read files.
    try
        matches = open(matches_file) do f
            readdlm(f, ',', String)
        end
        segments = open(segments_file) do f
            readdlm(f, ',', String)
        end
    catch ex
        error("Could not read file. ", ex)
    end

    # Convert matches and segments to DataFrames.
    try
        matches = matches_to_dataframe(matches)
        segments = segments_to_dataframe(segments)
    catch ex
        error("Could not interpret files. Wrong format? ", ex)
    end

    # Map matches' segments onto DNA.
    #matches_segments = innerjoin(matches, segments, on = :Name)
    matches_segments = innerjoin(matches, segments, on = :Match_ID)

    dna = DNA()
    map_countries!(dna, matches_segments, birth_country, excludes)

    # Evaluate ethnicities.
    eths, segments_total = ethnicities(dna)
    print_ethnicities(eths, segments_total)
end

function ethnicity(matches_file::AbstractString,
                   segments_file::AbstractString,
                   parent_matches_file::AbstractString;
                   birth_country = "",
                   excludes=["USA", "Canada", "Australia"])
    local matches
    local segments
    local parent

    # Read files.
    try
        matches = open(matches_file) do f
            readdlm(f, ',', String)
        end
        segments = open(segments_file) do f
            readdlm(f, ',', String)
        end
        parent = open(parent_matches_file) do f
            readdlm(f, ',', String)
        end
    catch ex
        error("Could not read file. ", ex)
    end

    # Convert matches and segments to DataFrames.
    try
        matches = matches_to_dataframe(matches)
        segments = segments_to_dataframe(segments)
        parent = matches_to_dataframe(parent)
    catch ex
        error("Could not interpret files. Wrong format? ", ex)
    end

    # Phase matches.
    # Matches that match child and parent.
    child_parent = innerjoin(matches, parent, on = :Match_ID, makeunique=true)
    # Matches that only match the second parent.
    child_parent2 = antijoin(matches, child_parent, on = :Match_ID, makeunique=true)
    # Remove matches that match both parents from the child-parent matches.
    both, child_parent = remove_false_matches(child_parent, parent)

    # Add segments to the matches lists.
    child_parent_segments = innerjoin(child_parent, segments, on = :Match_ID)
    child_parent2_segments = innerjoin(child_parent2, segments, on = :Match_ID)
    both_segments = innerjoin(both, segments, on = :Match_ID)

    # Map matches' segments onto DNA.
    parent_dna = DNA()
    parent2_dna = DNA()
    both_dna = DNA()
    map_countries!(parent_dna, child_parent_segments, birth_country, excludes)
    map_countries!(parent2_dna, child_parent2_segments, birth_country, excludes)
    map_countries!(both_dna, both_segments, birth_country, excludes)

    # Evaluate ethnicities.
    parent_eths, segments_total_1 = ethnicities(parent_dna)
    parent2_eths, segments_total_2 = ethnicities(parent2_dna)
    both_eths, segments_total_3 = ethnicities(both_dna)

    println("\nEthnicities you inherited from your parent.")
    print_ethnicities(parent_eths, segments_total_1)

    println("\nEthnicities you inherited from your other parent.")
    print_ethnicities(parent2_eths, segments_total_2)

    println("\nEthnicities that contributed to you and both of your parents.")
    print_ethnicities(both_eths, segments_total_3)
end

"""
DNA is modeled as an array of chromosomes.
Each chromosome consists of a number of segments represented
by a dictionary that maps country names to the number of matches
from each country.
"""
mutable struct DNA
    chromosomes::Array{Array{Dict{String, Integer}}}
end

function DNA()
    chromosomes = Array{Array{Dict{String, Integer}}}(undef, CHROMOSOMES)
    for i in 1:CHROMOSOMES
        nSegs = div(CHROMOSOME_SIZES[i] + SEGMENT_LENGTH - 1, SEGMENT_LENGTH)
        segments = Array{Dict{String, Integer}}(undef, nSegs)
        for j in 1:nSegs
            segments[j] = Dict{String, Integer}()
        end
        chromosomes[i] = segments
    end
    DNA(chromosomes)
end

"""
    map_countries!(dna::DNA, df::DataFrame)

Map country information from matches onto the DNA chromosomes.
The data frame "segments" must contain the following columns:
:Country, :Chromosome, :StartLocation, :EndLocation.
excludes contains a list of countries that should be excluded from mapping.
"""
function map_countries!(dna::DNA, segments::DataFrame, birth_country::String, excludes::Array{String})
    # Map matches' segments onto the DNA.
    nrows, _ = size(segments)
    for i in 1: nrows
        # Ignore close relatives.
        if segments[i, :EndLocation] - segments[i, :StartLocation] > MAX_MATCH_LENGTH
            continue
        end

        # Map matching segment onto the DNA.
        a = segments[i, :StartLocation] + SEGMENT_LENGTH
        b = segments[i, :EndLocation]
        c = segments[i, :Chromosome]
        country = segments[i, :Country]
        # Ignore countries that should be excluded.
        if in(country, excludes)
            continue
        end
        for j in a:SEGMENT_LENGTH:b
            seg = div(j, SEGMENT_LENGTH)
            if haskey(dna.chromosomes[c][seg], country)
                dna.chromosomes[c][seg][country] += 1
            else
                dna.chromosomes[c][seg][country] = 1
            end
        end
    end
    # Map birth country onto DNA segments that are already populated.
    if birth_country == "" || in(birth_country, excludes)
        return
    end
    for i in eachindex(dna.chromosomes)
        for j in eachindex(dna.chromosomes[i])
            # Check if DNA segment is populated.
            if length(keys(dna.chromosomes[i][j])) == 0
                continue
            end
            # Add birth country.
            if haskey(dna.chromosomes[i][j], birth_country)
                dna.chromosomes[i][j][birth_country] += 1
            else
                dna.chromosomes[i][j][birth_country] = 1
            end
        end
    end
end

"""
    ethnicities(dna::DNA)

Calculate ethnicities for a given DNA sample.

Returns an Array{Array{String}} and the total number of
segments that were evaluated. For each chromosome an array
exists that consists of segment entries. A segment entry is
a String that holds the most likely country origin of the
segment.
"""
function ethnicities(dna::DNA)
    # Create an empty country entry for each DNA segment.
    ethSegments = Array{Array{String}}(undef, CHROMOSOMES)
    for i in 1:CHROMOSOMES
        nSegs = length(dna.chromosomes[i])
        chromosome = Array{String}(undef, nSegs)
        for j in 1:nSegs
            chromosome[j] = ""
        end
        ethSegments[i] = chromosome
    end

    # Determine ethnicity for each segment.
    for i in 1:CHROMOSOMES
        for j in eachindex(dna.chromosomes[i])
            ethSegments[i][j] = most_common_ethnicity(dna.chromosomes[i][j])
        end
    end

    # Count segment ethnicities.
    ethnicities = Dict{String, Integer}()
    # Total number of evaluated segments.
    total = 0
    for i in 1:CHROMOSOMES
        for j in eachindex(ethSegments[i])
            if ethSegments[i][j] != ""
                total += 1
                if haskey(ethnicities, ethSegments[i][j])
                    ethnicities[ethSegments[i][j]] += 1
                else
                    ethnicities[ethSegments[i][j]] = 1
                end
            end
        end
    end
    return ethnicities, total
end

"""
    print_ethnicities(eth::Dict{String, Integer}, segments_total::Integer)

Print the result of the ethnicities function.
"""
function print_ethnicities(eth::Dict{String, Integer}, segments_total::Integer)
    # Sort results.
    results = []
    for (country, value) in eth
        percentage = value / segments_total * 100
        push!(results, (country, value, percentage))
    end
    sort!(results, by = x -> x[2], rev = true)

    # Output results.
    println("Total: ", segments_total, " segments")
    for i in results
        @printf("%15s %4d %5.1f%%\n", i[1], i[2], i[3])
    end
end

"""
    matches_to_dataframe(matches::Array{String, 2})

Converts matches to a valid DataFrame. MyHeritage CSV files
sometimes contain an invalid CSV format and can not be read by
the standard Julia CSV package. So we must do this manually.
"""
function matches_to_dataframe(matches::Array{String, 2})
    df = DataFrame(Match_ID = String[], Match_Name = String[], Country = String[], Total_cM_shared = Float64[])

    # Copy matches rows into DataFrame.
    match_names = Dict{String, Int64}()
    rows, _ = size(matches)
    for i = 2:rows
        match_id = matches[i, 1]
        country = matches[i, 4]
        match_name = matches[i, 2]
        matches_cM = matches[i, 10]
        if country == "" || match_name == "" || matches_cM == ""
            continue
        end

        # Convert cM to Float.
        cM = parse(Float64, replace(matches_cM, "," => ""))
        push!(df, [match_id, match_name, country, cM])
    end
    return df
end

"""
    segments_to_dataframe(segments::Array{String, 2})

Converts matches to a valid DataFrame. MyHeritage CSV files
sometimes contain an invalid CSV format and no valid key column.
They can not always be read by the standard Julia CSV package.
So we must do this manually.
"""
function segments_to_dataframe(segments::Array{String, 2})
    df = DataFrame(Match_ID = String[], Chromosome = Integer[], StartLocation = Integer[], EndLocation = Integer[])

    # Copy segments rows into DataFrame.
    rows, _ = size(segments)
    for i = 2:rows
        match_id = segments[i, 1]
        chromosome = segments[i, 4]
        startloc = segments[i, 5]
        endloc = segments[i, 6]
        if  chromosome == "" || startloc == "" || endloc == ""
            continue
        end

        chromosome = parse(Int64, chromosome)
        startloc = parse(Int64, startloc)
        endloc = parse(Int64, endloc)
        push!(df, [match_id, chromosome, startloc, endloc])
    end
    return df
end

"""
    remove_false_matches(child_matches, parent_matches)

Remove matches from the child_matches if corresponding parent_matches
share less cM with the match than the child.

Return the matches where both parents have contributed and
the matches that are from one parent only.
"""
function remove_false_matches(child_matches::DataFrame, parent_matches::DataFrame)
    both = similar(child_matches, 0)
    parent_cleaned = similar(child_matches, 0)
    # Create a dictionary of parent matches cM.
    parentCM = Dict{String, Float64}()
    for row in eachrow(parent_matches)
        name = row[:Name]
        cM = row[:Total_cM_shared]
        parentCM[name] = cM
    end
    # From the matches that were inherited from the parent
    # remove thosw where child shared cM > parent shared cM.
    for row in eachrow(child_matches)
        name = row[:Name]
        childCM = row[:Total_cM_shared]
        if childCM > parentCM[name]
            push!(both, row)
        else
            push!(parent_cleaned, row)
        end
    end
    return both, parent_cleaned
end

"""
    most_common_ethnicity(d::Dict{String, Integer})::String

Eetermine the ethnicity based on a map that contains a count
of matches from each country.

d is a dictionary that contains the number of matches for each country.

Returns the name of the country that harbors the most matches
or an empty string if there is no clear result.
"""
function most_common_ethnicity(d::Dict{String, Integer})::String
    if length(keys(d)) == 0
        return ""
    end

    # Determine country with the most cousins.
    country = ""
    top = 0
    second = 0
    for (key, value) in d
        DEBUG && print(key, " ", value, "  ")
        if value > second && value <= top
            second = value
        end
        if value > top
            country = key
            second = top
            top = value
        end
    end
    DEBUG && println()

    # Look if the most common country is valid.
    # Experiments with different conditions are allowed here.

    # All matches from one country.
    if second == 0
        return country
    end

    # Condition for the top most matches to be valid.
    if top / second >= 1.5
        return country
    else
        return ""
    end
end


module NaiveNASflux

using Reexport
@reexport using NaiveNASlib
@reexport using Flux
import Flux.Tracker: hook
using Statistics
using Setfield
using LinearAlgebra
import InteractiveUtils: subtypes
import JuMP: @variable, @constraint

export FluxLayer, FluxParLayer, FluxDense, FluxRecurrent, FluxRnn, FluxGru, FluxLstm, FluxConvolutional, FluxConv, FluxConvTranspose, FluxDepthwiseConv, FluxCrossCor, FluxTransparentLayer, FluxParInvLayer, FluxDiagonal, FluxLayerNorm, FluxParNorm, FluxBatchNorm, FluxInstanceNorm, FluxGroupNorm, FluxNoParLayer

export mutable, concat, AbstractMutableComp, MutableLayer, LazyMutable, NoParams

export named, validated, logged

export ActivationContribution, neuron_value, Ewma

export indim, outdim, actdim, layer, layertype

export idmapping, idmapping_nowarn

export KernelSizeAligned, mutate_weights

include("types.jl")
include("util.jl")
include("select.jl")
include("mutable.jl")
include("vertex.jl")
include("pruning.jl")
include("weightinit.jl")

# Modified version of Flux.treelike for mutable structs which are mutable mainly because they are intended to be wrapped in MutationVertices which are not easy to create in the manner which Flux.mapchildren is designed.
function treelike_mutable(m::Module, T, fs = treelike_fields(T))
  @eval m begin
    Flux.children(x::$T) = ($([:(x.$f) for f in fs]...),)
    function Flux.mapchildren(f, x::$T)
        $([:(x.$fn = f(x.$fn)) for fn in fs]...)
        return x
    end
  end
end

treelike_fields(T) = fieldnames(T)

macro treelike_mutable(T, fs = nothing)
  fs == nothing || Meta.isexpr(fs, :tuple) || error("@treelike_mutable T (a, b)")
  fs = fs == nothing ? [] : [:($(map(QuoteNode, fs.args)...),)]
  :(treelike_mutable(@__MODULE__, $(esc(T)), $(fs...)))
end

for subtype in subtypes(AbstractMutableComp)
  @treelike_mutable subtype
end

end # module


using ANYmalCRobot
using MeshCat
using MeshCatMechanisms
using RigidBodyDynamics

# Initialize anymal mechanism and MeshCat visualizer
anymal = ANYmalCRobot.mechanism()
vis = Visualizer()
mvis = MechanismVisualizer(
    anymal,
    URDFVisuals(ANYmalCRobot.urdfpath(), package_path=[ANYmalCRobot.packagepath()]),
    vis)

# set_nominal!() is a helper function to put the robot into a default config
anymal_state = MechanismState(anymal)
ANYmalCRobot.set_nominal!(anymal_state)
set_configuration!(mvis, configuration(anymal_state))

# Helper functions to visualise joint frames and contacts
# Try running visualize_frames() with optional kwarg "show_all_frames = true" to
# show all frames, by default only revolute are shown!
ANYmalCRobot.visualize_frames(anymal, mvis)
ANYmalCRobot.visualize_contacts(anymal, mvis)


using POMDPs, POMDPGym, Test, Crux, Flux, Random, BSON
function test_solver(𝒮fn, mdp, π...)
    # run it once
    Random.seed!(0)
    S1 = 𝒮fn(deepcopy.(π)...)
    π1 = solve(S1, deepcopy(mdp))

    # run it again on the cpu with the same rng
    Random.seed!(0)
    S2 = 𝒮fn(deepcopy.(π)...)
    π2 = solve(S2, deepcopy(mdp))

    # Run it on the gpu
    Random.seed!(0)
    S3 = 𝒮fn(gpu.(deepcopy.(π))...)
    π3 = solve(S3, deepcopy(mdp))

    # compare the results
    s = rand(Crux.dim(state_space(mdp))...)
    try
        value(π[1], s)
        @test all(value(π1, s) .≈ value(π2, s))
        @test all(abs.(value(π2, s) .- value(π3, s)) .< 1e-2)
    catch
        if action(π1, s)[1] isa Symbol
            @test all(action(π1, s) .== action(π2, s))
            @test all(action(π2, s) .== action(π3, s))
        else
            @test all(action(π1, s) .≈ action(π2, s))
            @test all(abs.(action(π2, s) .- action(π3, s)) .< 1e-2)
        end
    end
end

## Training params
N = 100
ΔN = 50

## discrete RL
discrete_mdp = GridWorldMDP()
S = state_space(discrete_mdp)
A() = DiscreteNetwork(Chain(Dense(2, 32, relu), Dense(32, 4)), actions(discrete_mdp))
V() = ContinuousNetwork(Chain(Dense(2, 32, relu), Dense(32, 1)))
AC() = ActorCritic(A(), V())

test_solver((π) -> REINFORCE(π=π, S=S, N=N, ΔN=ΔN), discrete_mdp, A())
test_solver((π) -> A2C(π=π, S=S, N=N, ΔN=ΔN), discrete_mdp, AC())
test_solver((π) -> PPO(π=π, S=S, N=N, ΔN=ΔN), discrete_mdp, AC())
test_solver((π) -> DQN(π=π, S=S, N=N), discrete_mdp, A())

## Continuous RL
continuous_mdp = PendulumPOMDP()
S = state_space(continuous_mdp)
QSA() = ContinuousNetwork(Chain(Dense(3, 32, tanh), Dense(32, 1)))
V() = ContinuousNetwork(Chain( Dense(2, 32, relu), Dense(32, 1)))
A() = ContinuousNetwork(Chain(Dense(2, 32, relu), Dense(32, 1, tanh)), 1)
G() = GaussianPolicy(A(), zeros(Float32, 1))

test_solver((π) -> REINFORCE(π=π, S=S, N=N, ΔN=ΔN), continuous_mdp, G())
test_solver((π) -> A2C(π=π, S=S, N=N, ΔN=ΔN), continuous_mdp, ActorCritic(G(), V()))
test_solver((π) -> PPO(π=π, S=S, N=N, ΔN=ΔN), continuous_mdp, ActorCritic(G(), V()))
test_solver((π) -> DDPG(π=π, S=S, N=N, ΔN=ΔN), continuous_mdp, ActorCritic(A(), QSA()))
test_solver((π) -> TD3(π=π, S=S, N=N, ΔN=ΔN), continuous_mdp, ActorCritic(A(), DoubleNetwork(QSA(), QSA())))
test_solver((π) -> SAC(π=π, S=S, N=N, ΔN=ΔN), continuous_mdp, ActorCritic(G(), DoubleNetwork(QSA(), QSA())))

# Continuous IL
𝒟_demo = expert_trajectories = BSON.load("examples/il/expert_data/pendulum.bson")[:data]
D(output=1) = ContinuousNetwork(Chain(DenseSN(3, 12, relu), DenseSN(12, output)))

test_solver((π, D) -> OnPolicyGAIL(D=D, 𝒟_demo=𝒟_demo, π=π, S=S, N=N, ΔN=ΔN), continuous_mdp, ActorCritic(G(), V()), D())
test_solver((π, D) -> OffPolicyGAIL(D=D, 𝒟_demo=𝒟_demo, π=π, S=S, N=50, ΔN=ΔN), continuous_mdp, ActorCritic(G(), DoubleNetwork(QSA(), QSA())), D(2))
test_solver(π -> BC(π=π, 𝒟_demo=𝒟_demo, S=S, opt=(epochs=1,), log=(period=50,)), continuous_mdp, A())
# NOTE: gradient penalty on the gpu only plays nicely with tanh, not relus in the discriminator?
test_solver((π) -> AdVIL(𝒟_demo=𝒟_demo, π=π, S=S, a_opt=(epochs=1,), log=(period=50,)), continuous_mdp, ActorCritic(A(), QSA()))
test_solver((π) -> SQIL(𝒟_demo=𝒟_demo, π=π, S=S, N=N, ΔN=ΔN), continuous_mdp, ActorCritic(G(), DoubleNetwork(QSA(), QSA())))
test_solver((π) -> AdRIL(𝒟_demo=𝒟_demo, π=π, S=S, N=N, ΔN=ΔN), continuous_mdp, ActorCritic(G(), DoubleNetwork(QSA(), QSA())))
test_solver((π) -> ASAF(𝒟_demo=𝒟_demo, π=π, S=S, N=N, ΔN=ΔN), continuous_mdp, G())

# Batch RL
test_solver(π -> BatchSAC(π=π, 𝒟_train=𝒟_demo, S=S, a_opt=(epochs=1,)), continuous_mdp, ActorCritic(G(), DoubleNetwork(QSA(), QSA())))
test_solver(π -> CQL(π=π, 𝒟_train=𝒟_demo, S=S, a_opt=(epochs=1,)), continuous_mdp, ActorCritic(G(), DoubleNetwork(QSA(), QSA())))


using CSV, DataFrames, Dates, PyCall

export JuliaToPandas
export PandasToJulia
export LSTM
export train
export save_lstm
export load_lstm
export predict
export ragged_preds
export gen_news

"convert a Julia DataFrame to a Pandas DataFrame"
function JuliaToPandas(df::DataFrame)
    pd = pyimport("pandas")

    date_col_name = (eltype.(eachcol(df)) .|> string .== "Date") |> x-> names(df)[x] |> x-> x[1]
    CSV.write("tmp.csv", df)
    pd_df = pd.read_csv("tmp.csv", parse_dates=[date_col_name])
    run(`rm tmp.csv`)
    return pd_df
end

"convert a Pandas DataFrame to a Julia DataFrame"
function PandasToJulia(df::PyObject, missingstrings::Vector=["NA", "na", "nan", "NAN", "NaN", ".", ""])
    df.to_csv("tmp.csv", index=false)
    julia_df = CSV.File("tmp.csv", missingstrings=missingstrings) |> DataFrame
    run(`rm tmp.csv`)
    return julia_df
end

"""
    Primary class of the library, used for transforming data, training the model, and making predictions.
    parameters:
        data : DataFrame
            n x m+1 dataframe
        target_variable : String
            name of the target var
        n_timesteps : Int
            how many historical periods to consider when training the model. For example if the original data is monthly, n_steps=12 would consider data for the last year.
        fill_na_func : PyObject
            Python function to replace within-series NAs. Given a column, the function should return a scalar. E.g. to replace default mean with median, `LSTM(..., fill_na_func = pyimport("numpy").nanmedian)`
        fill_ragged_edges_func : PyObject
            Python function to replace NAs in ragged edges (data missing at end of series). Pass "ARMA" for ARMA filling. Not ARMA filling will be significantly slower as models have to be estimated for each variable to fill ragged edges.
        n_models : Int
            number of models to train and take the average of for more robust estimates
        train_episodes : Int
            number of epochs/episodes to train the model
        batch_size : Int
            number of observations per training batch
        decay : Float32
            learning rate decay
        n_hidden : Int
            number of hidden states in the network
        n_layers : Int
            number of LSTM layers in the network
        dropout : Float32
            dropout rate between the LSTM layers
        criterion : PyObject
            torch loss criterion, defaults to MAE. E.g. `LSTM(..., pyimport("torch").nn.L1Loss())`. Need parentheses at end of torch loss function.
        optimizer : PyObject
            torch optimizer, defaults to Adam. E.g. `LSTM(..., pyimport("torch").optim.Adam)`. Don't need parentheses at end of torch loss function.
        optimizer_parameters : Dict
            list of parameters for optimizer, including learning rate. E.g. Dict("lr" => 1e-2)
    returns: PyObject
        instantiated LSTM model
"""
function LSTM(;
    data::DataFrame = nothing,
    target_variable::String = nothing,
    n_timesteps::Int = nothing,
    fill_na_func::PyObject = pyimport("numpy").nanmean,
    fill_ragged_edges_func = pyimport("numpy").nanmean,
    n_models::Int = 1,
    train_episodes::Int = 200,
    batch_size::Int = 30,
    decay::Float64 = 0.98,
    n_hidden::Int = 20,
    n_layers::Int = 2,
    dropout::Float64 = 0.0,
    criterion::PyObject = pyimport("torch").nn.L1Loss(),
    optimizer::PyObject = pyimport("torch").optim.Adam,
    optimizer_parameters::Dict = Dict("lr" => 1e-2)
)::PyObject

    model = pyimport("nowcast_lstm.LSTM").LSTM(
        data = JuliaToPandas(data),
        target_variable = target_variable,
        n_timesteps = n_timesteps,
        fill_na_func = fill_na_func,
        fill_ragged_edges_func = fill_ragged_edges_func,
        n_models = n_models,
        train_episodes = train_episodes,
        batch_size = batch_size,
        decay = decay,
        n_hidden = n_hidden,
        n_layers = n_layers,
        dropout = dropout,
        criterion = criterion,
        optimizer = optimizer,
        optimizer_parameters = optimizer_parameters,
    )

    return model
end

"train an instantiated LSTM model. pass train(model; quiet=true). quiet=true will suppress printing of training loss at each epoch"
function train(model::PyObject; quiet::Bool = false)::PyObject
    model.train(quiet=quiet)
    return model
end

"get predictions from a trained model. pass predict(model, data; only\\_actuals\\_obs = true) to only get predictions where there is an actual value"
function predict(model::PyObject, data::DataFrame; only_actuals_obs::Bool = false)::DataFrame
    model.predict(JuliaToPandas(data), only_actuals_obs) |> PandasToJulia
end

"save a trained LSTM model to disk. pass save_lstm(model, path_to_save), filename must end in .pkl, e.g. '/Users/trained_model.pkl'"
function save_lstm(model::PyObject, filename::String)
    pyimport("dill").dump(model, py"open"(filename, mode="wb"))
end

"load a trained LSTM model. pass load_lstm(path_to_pkl)"
function load_lstm(filename::String)::PyObject
    model = pyimport("dill").load(py"open"(filename, "rb", -1))
    return model
end

"""
    Get predictions on artificial vintages
    parameters:
        model : PyObject
            trained LSTM model
        pub_lags : Vector
            list of periods back each input variable is set to missing. I.e. publication lag of the variable.
        lag : Int
            simulated periods back. E.g. -2 = simulating data as it would have been 2 months before target period, 1 = 1 month after, etc.
        data : DataFrame
            dataframe to generate the ragged datasets on, if none will calculate on training data
        start_date : String
            String in "YYYY-MM-DD" format: start date of generating ragged preds. To save calculation time, i.e. just calculating after testing date instead of all dates
        end_date : String
            String in "YYYY-MM-DD" format: end date of generating ragged preds
    returns: DataFrame
        predictions on vintages
"""
function ragged_preds(
    model::PyObject,
    pub_lags::Vector{Int64},
    lag::Int64,
    data::DataFrame;
    start_date::String = "",
    end_date::String = "",
)::DataFrame
    if start_date == ""
        start_date = nothing
    end
    if end_date == ""
        end_date = nothing
    end
    model.ragged_preds(pub_lags, lag, JuliaToPandas(data), start_date, end_date) |> PandasToJulia
end

"""
    Generate the news between two data releases using the method of holding out new data feature by feature and recording the differences in model output
    parameters:
        model : PyObject
            trained LSTM model
        target_period : String
            String in "YYYY-MM-DD", target prediction date
        old_data : DataFrame
             dataframe of previous dataset
        new_data : DataFrame
             dataframe of new dataset
    returns: Dictionary
        "news": dataframe of news contribution of each column with updated data. scaled_news is news scaled to sum to actual prediction delta.
        "old_pred": prediction on the previous dataset
        "new_pred": prediction on the new dataset
        "holdout_discrepency": difference between the sum of news via the holdout method and the actual prediction delta
"""
function gen_news(model::PyObject, target_period::String, old_data::DataFrame, new_data::DataFrame)::Dict
    news = model.gen_news(target_period, JuliaToPandas(old_data), JuliaToPandas(new_data))
    julia_news = Dict()
    julia_news["news"] = news["news"] |> PandasToJulia
    julia_news["old_pred"] = news["old_pred"]
    julia_news["new_pred"] = news["new_pred"]
    julia_news["holdout_discrepency"] = news["holdout_discrepency"]
    return julia_news
end


# using Base.Test
# using MIToS.MSA

let alphabet = "ARNDCQEGHILKMFPSTWYV-",
  msa =  ["DAWAEF",
          "DAWAED",
          "DAYCMD" ],
  badmsa=["DAWAEF",
          "DAWAED",
          "DAM"    ]

  print("""

Test Residue
============
""")

  print("""
Convert
""")
  @test Int[ Residue(char) for char in alphabet] == Int[ i for i in 1:21 ]
  @test Int[ Residue(char) for char in lowercase("ARNDCQEGHILKMFPSTWYV.")] == Int[ 21 for i in 1:21 ]

  print("""
GAP
""")
  @test Int(GAP) == 21
  @test Int8(GAP) == Int8(21)
  @test Residue('-') == GAP
  @test Char(GAP) == '-'
  @test UInt8(GAP) == UInt8('-')

  print("""
Convert to and from ASCIIString
""")
  @test res"ARNDCQEGHILKMFPSTWYV-" == Residue[ Residue(char) for char in alphabet]
  @test Residue(alphabet) == Residue[ Residue(char) for char in alphabet]
  @test ascii(Residue(alphabet)) == alphabet
  @test string(Residue(alphabet)) == alphabet # "AR..." instead of the standar "[A,R,..."

  @test convert(Matrix{Residue}, msa) == Residue['D' 'A' 'W' 'A' 'E' 'F'
                                                 'D' 'A' 'W' 'A' 'E' 'D'
                                                 'D' 'A' 'Y' 'C' 'M' 'D']

  @test_throws ErrorException convert(Matrix{Residue}, badmsa)

  print("""
Ambiguous or not standard residues (are gaps on MIToS)
""")

  for res in Char['X', # Any amino acid
                  'B', # Aspartic acid or Asparagine
                  'Z', # Glutamine or Glutamic acid
                  'O', # Pyrrolysine
                  'U', # Selenocysteine
                  'J'] # Leucine or Isoleucine

    @test Residue(res) == GAP

  end

  print("""
Comparisons
""")

  for res in Residue(alphabet)
    @test res == res
  end

  for i in 1:21, j in 1:21
    if j != i
      @test Residue(i) != Residue(j)
    end
  end

  print("""
Random
""")
  for i in 1:8000
    @test rand(Residue) != GAP
    @test typeof( rand(Residue) ) == Residue
  end
  @test size(rand(Residue, 20)) == (20,)
  @test typeof(rand(Residue, 20)) == Array{Residue, 1}
  @test size(rand(Residue, 20,30)) == (20,30)
  @test typeof(rand(Residue, 20, 30)) == Array{Residue, 2}

end

print("""

Three letters <=> One letter (Residue)
--------------------------------------
""")

@test three2residue("ALA") == Residue('A')
@test three2residue("Ala") == Residue('A')
@test three2residue("ala") == Residue('A')

@test_throws ErrorException three2residue("AL")
@test_throws ErrorException three2residue("Alanine")

@test residue2three(Residue('A')) == "ALA"


# This file is a part of Julia. License is MIT: https://julialang.org/licenOAse

import LinearAlgebra: BlasInt
using Logging

# A convenient shortcut to show unexpected behavior from libarpack
const ERR_UNEXPECTED_BEHAVIOR = -999

struct XYAUPD_Exception <: Exception
    info::BlasInt
end

const AUPD_ERRORS = [
    (3, "No shifts could be applied during a cycle of the Implicitly restarted Arnoldi iteration. One possibility is to increase the size of NCV relative to NEV. "),
    (2, "No longer an informational error. Deprecated starting with release 2 of ARPACK."),
    (1, """Maximum number of iterations taken. All possible eigenvalues of OP has been found.
          IPARAM(5) returns the number of wanted converged Ritz values."""),
    (0, "Normal exit."),
    (-1, "N must be positive."),
    (-2, "NEV must be positive."),
    (-3, "NCV-NEV >= 2 and less than or equal to N."),
    (-4, "The maximum number of Arnoldi update iterations allowed must be greater than zero."),
    (-5, " WHICH must be one of 'LM', 'SM', 'LR', 'SR', 'LI', 'SI'"),
    (-6, "BMAT must be one of 'I' or 'G'."),
    (-7, "Length of private work array WORKL is not sufficient."),
    (-8, "Error return from LAPACK eigenvalue calculation."),
    (-9, "Starting vector is zero."),
    (-10, "IPARAM(7) must be 1,2,3,4."),
    (-11, "IPARAM(7) = 1 and BMAT = 'G' are incompatible."),
    (-12, "IPARAM(1) must be equal to 0 or 1."),
    (-13, "NEV and WHICH = 'BE' are incompatible."),
    (-9999, """Could not build an Arnoldi factorization.
           IPARAM(5) returns the size of the current Arnoldi factorization.
           The user is advised to check that enough workspace and array storage has been allocated.""")
]

function Base.showerror(io::IO, ex::XYAUPD_Exception)
    info = ex.info

    if info == ERR_UNEXPECTED_BEHAVIOR
        @error "XYAUPD_Exception: Undefined error"
    else
      idx = searchsorted(AUPD_ERRORS, info, by=first, rev=true)
      if isempty(idx)
          @error "XYAUPD_Exception: Please check XYAUPD error codes in the ARPACK manual." info
      else
          @error "XYAUPD_Exception: $(last(AUPD_ERRORS[first(idx)]))" info
      end
    end
end

struct XYEUPD_Exception <: Exception
    info::BlasInt
end

const EUPD_ERRORS = [
    (1, """The Schur form computed by LAPACK routine lahqr could not be reordered by LAPACK routine trsen.
       Re-enter subroutine neupd with IPARAM(5)NCV and increase the size of the arrays DR and DI to have dimension at least dimension NCV and allocate at least NCV columns for Z.
       NOTE: Not necessary if Z and V share the same space. Please notify the authors if this error occurs."""),
    (0, "Normal exit."),
    (-1, "N must be positive."),
    (-2, "NEV must be positive."),
    (-3, "NCV-NEV >= 2 and less than or equal to N."),
    (-5, "WHICH must be one of 'LM', 'SM', 'LR', 'SR', 'LI', 'SI'"),
    (-6, "BMAT must be one of 'I' or 'G'."),
    (-7, "Length of private work WORKL array is not sufficient."),
    (-8, """Error return from calculation of a real Schur form.
        "Informational error from LAPACK routine lahqr ."""),
    (-9, """Error return from calculation of eigenvectors.
        "Informational error from LAPACK routine dtrevc."""),
    (-10, "IPARAM(7) must be 1,2,3,4."),
    (-11, "IPARAM(7) = 1 and BMAT = 'G' are incompatible."),
    (-12, "HOWMNY = 'S' not yet implemented"),
    (-13, "HOWMNY must be one of 'A' or 'P' if RVEC = .true."),
    (-14, "DNAUPD  did not find any eigenvalues to sufficient accuracy."),
    (-15, """DNEUPD got a different count of the number of converged Ritz values than NAUPD got.
          This indicates the user probably made an error in passing data from NAUPD to NEUPD or that the data was modified before entering NEUPD""")
]

function Base.showerror(io::IO, ex::XYEUPD_Exception)
    info = ex.info
    if info == ERR_UNEXPECTED_BEHAVIOR
        @error "XYEUPD_Exception: Undefined error"
    else
      idx = searchsorted(EUPD_ERRORS, info, by=first, rev=true)
      if isempty(idx)
          @error "XYEUPD_Exception: Please check XYEUPD error codes in the ARPACK manual." info
      else
          @error "XYEUPD_Exception: $(last(EUPD_ERRORS[first(idx)]))" info
      end
    end
end

## aupd and eupd wrappers

function aupd_wrapper(T, matvecA!::Function, matvecB::Function, solveSI::Function, n::Integer,
                      sym::Bool, cmplx::Bool, bmat,
                      nev::Integer, ncv::Integer, which,
                      tol::Real, maxiter::Integer, mode::Integer, v0::Vector, check::Integer)
    lworkl = cmplx ? ncv * (3*ncv + 5) : (sym ? ncv * (ncv + 8) :  ncv * (3*ncv + 6) )
    TR = cmplx ? T.types[1] : T
    TOL = Ref{TR}(tol)

    v     = Matrix{T}(undef, n, ncv)
    workd = Vector{T}(undef, 3*n)
    workl = Vector{T}(undef, lworkl)
    rwork = cmplx ? Vector{TR}(undef, ncv) : Vector{TR}()

    if isempty(v0)
        resid = Vector{T}(undef, n)
        info  = Ref{BlasInt}(0)
    else
        resid = deepcopy(v0)
        info  = Ref{BlasInt}(1)
    end
    iparam = zeros(BlasInt, 11)
    ipntr  = zeros(BlasInt, (sym && !cmplx) ? 11 : 14)
    ido    = Ref{BlasInt}(0)

    iparam[1] = BlasInt(1)       # ishifts
    iparam[3] = BlasInt(maxiter) # maxiter
    iparam[7] = BlasInt(mode)    # mode

    zernm1 = 0:(n-1)

    while true
        if cmplx
            naupd(ido, bmat, n, which, nev, TOL, resid, ncv, v, n,
                  iparam, ipntr, workd, workl, lworkl, rwork, info)
        elseif sym
            saupd(ido, bmat, n, which, nev, TOL, resid, ncv, v, n,
                  iparam, ipntr, workd, workl, lworkl, info)
        else
            naupd(ido, bmat, n, which, nev, TOL, resid, ncv, v, n,
                  iparam, ipntr, workd, workl, lworkl, info)
        end
        if info[] != 0
            if info[] == 1 && check != 0
                return (resid, v, n, iparam, ipntr, workd, workl, lworkl, rwork, TOL)
            end
            throw(XYAUPD_Exception(info[]))
        end

        x = view(workd, ipntr[1] .+ zernm1)
        y = view(workd, ipntr[2] .+ zernm1)
        if mode == 1  # corresponds to dsdrv1, dndrv1 or zndrv1
            if ido[] == -1 || ido[] == 1
                matvecA!(y, x)
            elseif ido[] == 99
                break
            else
                throw(XYAUPD_Exception(ERR_UNEXPECTED_BEHAVIOR))
            end
        elseif mode == 3 && bmat == "I" # corresponds to dsdrv2, dndrv2 or zndrv2
            if ido[] == -1 || ido[] == 1
                y[:] = solveSI(x)
            elseif ido[] == 99
                break
            else
                throw(XYAUPD_Exception(ERR_UNEXPECTED_BEHAVIOR))
            end
        elseif mode == 2 # corresponds to dsdrv3, dndrv3 or zndrv3
            if ido[] == -1 || ido[] == 1
                matvecA!(y, x)
                if sym
                    x[:] = y    # overwrite as per Remark 5 in dsaupd.f
                end
                y[:] = solveSI(y)
            elseif ido[] == 2
                y[:] = matvecB(x)
            elseif ido[] == 99
                break
            else
                throw(XYAUPD_Exception(ERR_UNEXPECTED_BEHAVIOR))
            end
        elseif mode == 3 && bmat == "G" # corresponds to dsdrv4, dndrv4 or zndrv4
            if ido[] == -1
                y[:] = solveSI(matvecB(x))
            elseif  ido[] == 1
                y[:] = solveSI(view(workd,ipntr[3] .+ zernm1))
            elseif ido[] == 2
                y[:] = matvecB(x)
            elseif ido[] == 99
                break
            else
                throw(XYAUPD_Exception(ERR_UNEXPECTED_BEHAVIOR))
            end
        else
            throw(ArgumentError("ARPACK mode ($mode) not yet supported"))
        end
    end

    return (resid, v, n, iparam, ipntr, workd, workl, lworkl, rwork, TOL)
end

function eupd_wrapper(T, n::Integer, sym::Bool, cmplx::Bool, bmat,
                      nev::Integer, which, ritzvec::Bool,
                      TOL::Ref, resid, ncv::Integer, v, ldv, sigma, iparam, ipntr,
                      workd, workl, lworkl, rwork)
    howmny = "A"
    select = Vector{BlasInt}(undef, ncv)
    info   = Ref{BlasInt}(0)

    dmap = x -> abs.(x)
    if iparam[7] == 3 # shift-and-invert
        dmap = x -> abs.(1 ./ (x .- sigma))
    elseif which == "LR" || which == "LA" || which == "BE"
        dmap = real
    elseif which == "SR" || which == "SA"
        dmap = x -> -real(x)
    elseif which == "LI"
        dmap = imag
    elseif which == "SI"
        dmap = x -> -imag(x)
    end

    if cmplx
        d = Vector{T}(undef, nev+1)
        sigmar = Ref{T}(sigma)
        workev = Vector{T}(undef, 2ncv)
        neupd(ritzvec, howmny, select, d, v, ldv, sigmar, workev,
              bmat, n, which, nev, TOL, resid, ncv, v, ldv,
              iparam, ipntr, workd, workl, lworkl, rwork, info)
        if info[] != 0
            throw(XYEUPD_Exception(info[]))
        end

        p = sortperm(dmap(d[1:nev]), rev=true)
        return ritzvec ? (d[p], v[1:n, p],iparam[5],iparam[3],iparam[9],resid) : (d[p],iparam[5],iparam[3],iparam[9],resid)
    elseif sym
        d = Vector{T}(undef, nev)
        sigmar = Ref{T}(sigma)
        seupd(ritzvec, howmny, select, d, v, ldv, sigmar,
              bmat, n, which, nev, TOL, resid, ncv, v, ldv,
              iparam, ipntr, workd, workl, lworkl, info)
        if info[] != 0
            throw(XYEUPD_Exception(info[]))
        end

        p = sortperm(dmap(d), rev=true)
        return ritzvec ? (d[p], v[1:n, p],iparam[5],iparam[3],iparam[9],resid) : (d[p],iparam[5],iparam[3],iparam[9],resid)
    else
        dr = Vector{T}(undef, nev+1)
        di = Vector{T}(undef, nev+1)
        fill!(dr,NaN)
        fill!(di,NaN)
        sigmar = Ref{T}(real(sigma))
        sigmai = Ref{T}(imag(sigma))
        workev = Vector{T}(undef, 3*ncv)
        neupd(ritzvec, howmny, select, dr, di, v, ldv, sigmar, sigmai,
              workev, bmat, n, which, nev, TOL, resid, ncv, v, ldv,
              iparam, ipntr, workd, workl, lworkl, info)
        if info[] != 0
            throw(XYEUPD_Exception(info[]))
        end
        evec = complex.(Matrix{T}(undef, n, nev+1), Matrix{T}(undef, n, nev+1))

        j = 1
        while j <= nev
            if di[j] == 0
                evec[:,j] = v[:,j]
            else # For complex conjugate pairs
                evec[:,j]   = v[:,j] + im*v[:,j+1]
                evec[:,j+1] = v[:,j] - im*v[:,j+1]
                j += 1
            end
            j += 1
        end
        if j == nev+1 && !isnan(di[j])
            if di[j] == 0
                evec[:,j] = v[:,j]
                j += 1
            else
                throw(XYEUPD_Exception(ERR_UNEXPECTED_BEHAVIOR))
            end
        end

        d = complex.(dr, di)

        if j == nev+1
            p = sortperm(dmap(d[1:nev]), rev=true)
        else
            p = sortperm(dmap(d), rev=true)
            p = p[1:nev]
        end

        return ritzvec ? (d[p], evec[1:n, p],iparam[5],iparam[3],iparam[9],resid) : (d[p],iparam[5],iparam[3],iparam[9],resid)
    end
end

for (T, saupd_name, seupd_name, naupd_name, neupd_name) in
    ((:Float64, :dsaupd_, :dseupd_, :dnaupd_, :dneupd_),
     (:Float32, :ssaupd_, :sseupd_, :snaupd_, :sneupd_))
    @eval begin
        function naupd(ido, bmat, n, evtype, nev, TOL::Ref{$T}, resid::Vector{$T}, ncv, v::Matrix{$T}, ldv,
                       iparam, ipntr, workd::Vector{$T}, workl::Vector{$T}, lworkl, info)
            ccall(($(string(naupd_name)), libarpack), Cvoid,
                  (Ref{BlasInt}, Ptr{UInt8}, Ref{BlasInt}, Ptr{UInt8}, Ref{BlasInt},
                   Ptr{$T}, Ptr{$T}, Ref{BlasInt}, Ptr{$T}, Ref{BlasInt},
                   Ptr{BlasInt}, Ptr{BlasInt}, Ptr{$T}, Ptr{$T}, Ref{BlasInt}, Ref{BlasInt}, Clong, Clong),
                  ido, bmat, n, evtype, nev,
                  TOL, resid, ncv, v, ldv,
                  iparam, ipntr, workd, workl, lworkl, info, 1, 2)
        end

        function neupd(rvec, howmny, select, dr, di, z, ldz, sigmar, sigmai,
                  workev::Vector{$T}, bmat, n, evtype, nev, TOL::Ref{$T}, resid::Vector{$T}, ncv, v, ldv,
                  iparam, ipntr, workd::Vector{$T}, workl::Vector{$T}, lworkl, info)
            ccall(($(string(neupd_name)), libarpack), Cvoid,
                  (Ref{BlasInt}, Ptr{UInt8}, Ptr{BlasInt}, Ptr{$T}, Ptr{$T}, Ptr{$T}, Ref{BlasInt},
                   Ref{$T}, Ref{$T}, Ptr{$T}, Ptr{UInt8}, Ref{BlasInt}, Ptr{UInt8}, Ref{BlasInt},
                   Ptr{$T}, Ptr{$T}, Ref{BlasInt}, Ptr{$T}, Ref{BlasInt},
                   Ptr{BlasInt}, Ptr{BlasInt}, Ptr{$T}, Ptr{$T}, Ref{BlasInt}, Ref{BlasInt}, Clong, Clong, Clong),
                  rvec, howmny, select, dr, di, z, ldz,
                  sigmar, sigmai, workev, bmat, n, evtype, nev,
                  TOL, resid, ncv, v, ldv,
                  iparam, ipntr, workd, workl, lworkl, info, 1, 1, 2)
        end

        function saupd(ido, bmat, n, which, nev, TOL::Ref{$T}, resid::Vector{$T}, ncv, v::Matrix{$T}, ldv,
                       iparam, ipntr, workd::Vector{$T}, workl::Vector{$T}, lworkl, info)
            ccall(($(string(saupd_name)), libarpack), Cvoid,
                  (Ref{BlasInt}, Ptr{UInt8}, Ref{BlasInt}, Ptr{UInt8}, Ref{BlasInt},
                   Ptr{$T}, Ptr{$T}, Ref{BlasInt}, Ptr{$T}, Ref{BlasInt},
                   Ptr{BlasInt}, Ptr{BlasInt}, Ptr{$T}, Ptr{$T}, Ref{BlasInt}, Ref{BlasInt}, Clong, Clong),
                  ido, bmat, n, which, nev,
                  TOL, resid, ncv, v, ldv,
                  iparam, ipntr, workd, workl, lworkl, info, 1, 2)
        end

        function seupd(rvec, howmny, select, d, z, ldz, sigma,
                       bmat, n, evtype, nev, TOL::Ref{$T}, resid::Vector{$T}, ncv, v::Matrix{$T}, ldv,
                       iparam, ipntr, workd::Vector{$T}, workl::Vector{$T}, lworkl, info)
            ccall(($(string(seupd_name)), libarpack), Cvoid,
                  (Ref{BlasInt}, Ptr{UInt8}, Ptr{BlasInt}, Ptr{$T}, Ptr{$T}, Ref{BlasInt},
                   Ptr{$T}, Ptr{UInt8}, Ref{BlasInt}, Ptr{UInt8}, Ref{BlasInt},
                   Ptr{$T}, Ptr{$T}, Ref{BlasInt}, Ptr{$T}, Ref{BlasInt},
                   Ptr{BlasInt}, Ptr{BlasInt}, Ptr{$T}, Ptr{$T}, Ref{BlasInt}, Ref{BlasInt}, Clong, Clong, Clong),
                  rvec, howmny, select, d, z, ldz,
                  sigma, bmat, n, evtype, nev,
                  TOL, resid, ncv, v, ldv,
                  iparam, ipntr, workd, workl, lworkl, info, 1, 1, 2)
        end
    end
end

for (T, TR, naupd_name, neupd_name) in
    ((:ComplexF64, :Float64, :znaupd_, :zneupd_),
     (:ComplexF32, :Float32, :cnaupd_, :cneupd_))
    @eval begin
        function naupd(ido, bmat, n, evtype, nev, TOL::Ref{$TR}, resid::Vector{$T}, ncv, v::Matrix{$T}, ldv,
                       iparam, ipntr, workd::Vector{$T}, workl::Vector{$T}, lworkl,
                       rwork::Vector{$TR}, info)
            ccall(($(string(naupd_name)), libarpack), Cvoid,
                  (Ref{BlasInt}, Ptr{UInt8}, Ref{BlasInt}, Ptr{UInt8}, Ref{BlasInt},
                   Ptr{$TR}, Ptr{$T}, Ref{BlasInt}, Ptr{$T}, Ref{BlasInt},
                   Ptr{BlasInt}, Ptr{BlasInt}, Ptr{$T}, Ptr{$T}, Ref{BlasInt}, Ptr{$TR}, Ref{BlasInt}, Clong, Clong),
                  ido, bmat, n, evtype, nev,
                  TOL, resid, ncv, v, ldv,
                  iparam, ipntr, workd, workl, lworkl, rwork, info, 1, 2)
        end

        function neupd(rvec, howmny, select, d, z, ldz, sigma, workev::Vector{$T},
                       bmat, n, evtype, nev, TOL::Ref{$TR}, resid::Vector{$T}, ncv, v::Matrix{$T}, ldv,
                       iparam, ipntr, workd::Vector{$T}, workl::Vector{$T}, lworkl,
                       rwork::Vector{$TR}, info)
            ccall(($(string(neupd_name)), libarpack), Cvoid,
                  (Ref{BlasInt}, Ptr{UInt8}, Ptr{BlasInt}, Ptr{$T}, Ptr{$T}, Ref{BlasInt},
                   Ptr{$T}, Ptr{$T}, Ptr{UInt8}, Ref{BlasInt}, Ptr{UInt8}, Ref{BlasInt},
                   Ptr{$TR}, Ptr{$T}, Ref{BlasInt}, Ptr{$T}, Ref{BlasInt},
                   Ptr{BlasInt}, Ptr{BlasInt}, Ptr{$T}, Ptr{$T}, Ref{BlasInt}, Ptr{$TR}, Ref{BlasInt}, Clong, Clong, Clong),
                  rvec, howmny, select, d, z, ldz,
                  sigma, workev, bmat, n, evtype, nev,
                  TOL, resid, ncv, v, ldv,
                  iparam, ipntr, workd, workl, lworkl, rwork, info, 1, 1, 2)
        end
    end
end


#### solver attributes
MathOptInterface.getattribute(m::Union{MosekSolver,MosekModel},::MathOptInterface.SupportsDuals) = true
MathOptInterface.getattribute(m::Union{MosekSolver,MosekModel},::MathOptInterface.SupportsAddConstraintAfterSolve) = true
MathOptInterface.getattribute(m::Union{MosekSolver,MosekModel},::MathOptInterface.SupportsAddVariableAfterSolve) = true
MathOptInterface.getattribute(m::Union{MosekSolver,MosekModel},::MathOptInterface.SupportsDeleteConstraint) = true
MathOptInterface.getattribute(m::Union{MosekSolver,MosekModel},::MathOptInterface.SupportsDeleteVariable) = true
MathOptInterface.getattribute(m::Union{MosekSolver,MosekModel},::MathOptInterface.SupportsConicThroughQuadratic) = false # though actually the solver does

#### objective
MathOptInterface.getattribute(m::MosekModel,attr::MathOptInterface.ObjectiveValue) = getprimalobj(m.task,m.solutions[attr.resultindex].whichsol)
MathOptInterface.cangetattribute(m::MosekSolver,attr::MathOptInterface.ObjectiveValue) = true
MathOptInterface.cangetattribute(m::MosekModel,attr::MathOptInterface.ObjectiveValue) = attr.resultindex > 0 && attr.resultindex <= length(m.solutions)

MathOptInterface.cangetattribute(m::MosekModel,attr::MathOptInterface.ObjectiveBound) = getintinf(m.task,MSK_IINF_MIO_NUM_RELAX) > 0
MathOptInterface.getattribute(m::MosekModel,attr::MathOptInterface.ObjectiveBound) = getdouinf(m.task,MSK_DINF_MIO_OBJ_BOUND)

MathOptInterface.cangetattribute(m::MosekModel,attr::MathOptInterface.RelativeGap) = getdouinf(m.task,MSK_DINF_MIO_OBJ_REL_GAP) > -1.0
MathOptInterface.getattribute(m::MosekModel,attr::MathOptInterface.RelativeGap) = getdouinf(m.task,MSK_DINF_MIO_OBJ_REL_GAP)

MathOptInterface.cangetattribute(m::MosekSolver,attr::MathOptInterface.SolveTime) = true
MathOptInterface.cangetattribute(m::MosekModel,attr::MathOptInterface.SolveTime) = true
MathOptInterface.getattribute(m::MosekModel,attr::MathOptInterface.SolveTime) = getdouinf(m.task,MSK_DINF_OPTIMIZER_TIME)

MathOptInterface.cangetattribute(m::MosekSolver,attr::MathOptInterface.ObjectiveSense) = true
MathOptInterface.cangetattribute(m::MosekModel,attr::MathOptInterface.ObjectiveSense)  = true
MathOptInterface.cansetattribute(m::MosekSolver,attr::MathOptInterface.ObjectiveSense) = true
MathOptInterface.cansetattribute(m::MosekModel,attr::MathOptInterface.ObjectiveSense)  = true

# NOTE: The MOSEK interface currently only supports Min and Max
function MathOptInterface.getattribute(m::MosekModel,attr::MathOptInterface.ObjectiveSense)
    sense = getobjsense(m.task)
    if sense == MSK_OBJECTIVE_SENSE_MINIMIZE
        MathOptInterface.MinSense
    else
        MathOptInterface.MaxSense
    end
end

function MathOptInterface.setattribute!(m::MosekModel,attr::MathOptInterface.ObjectiveSense, sense::MathOptInterface.OptimizationSense)
    if sense == MathOptInterface.MinSense
        setobjsense(m.task,MSK_OBJECTIVE_SENSE_MINIMIZE)
    elseif sense == MathOptInterface.MaxSense
        setobjsense(m.task,MSK_OBJECTIVE_SENSE_MAXIMIZE)
    else
        error("Sense '$sense' is not supported")
    end
end

#### Solver/Solution information

MathOptInterface.cangetattribute(m::MosekSolver,attr::MathOptInterface.SimplexIterations) = true
MathOptInterface.cangetattribute(m::MosekModel,attr::MathOptInterface.SimplexIterations) = true
function MathOptInterface.getattribute(m::MosekModel,attr::MathOptInterface.SimplexIterations)
    miosimiter = getlintinf(m.task,MSK_LIINF_MIO_SIMPLEX_ITER)
    if miosimiter > 0
        Int(miosimiter)
    else
        Int(getintinf(m.task,MSK_IINF_SIM_PRIMAL_ITER) + getintinf(m.task,MSK_IINF_SIM_DUAL_ITER))
    end
end

MathOptInterface.cangetattribute(m::MosekSolver,attr::MathOptInterface.BarrierIterations) = true
MathOptInterface.cangetattribute(m::MosekModel,attr::MathOptInterface.BarrierIterations) = true
function MathOptInterface.getattribute(m::MosekModel,attr::MathOptInterface.BarrierIterations)
    miosimiter = getlintinf(m.task,MSK_LIINF_MIO_INTPNT_ITER)
    if miosimiter > 0
        Int(miosimiter)
    else
        Int(getintinf(m.task,MSK_IINF_INTPNT_ITER))
    end
end

MathOptInterface.cangetattribute(m::MosekSolver,attr::MathOptInterface.NodeCount) = true
MathOptInterface.cangetattribute(m::MosekModel,attr::MathOptInterface.NodeCount) = true
function MathOptInterface.getattribute(m::MosekModel,attr::MathOptInterface.NodeCount)
        Int(getintinf(m.task,MSK_IINF_MIO_NUM_BRANCH))
end

MathOptInterface.cangetattribute(m::MosekSolver,attr::MathOptInterface.RawSolver) = true
MathOptInterface.cangetattribute(m::MosekModel,attr::MathOptInterface.RawSolver) = true
MathOptInterface.getattribute(m::MosekModel,attr::MathOptInterface.RawSolver) = m.task

MathOptInterface.cangetattribute(m::MosekSolver,attr::MathOptInterface.ResultCount) = true
MathOptInterface.cangetattribute(m::MosekModel,attr::MathOptInterface.ResultCount) = true
MathOptInterface.getattribute(m::MosekModel,attr::MathOptInterface.ResultCount) = length(m.solutions)

#### Problem information

MathOptInterface.cangetattribute(m::MosekSolver,attr::MathOptInterface.NumberOfVariables) = true
MathOptInterface.cangetattribute(m::MosekModel,attr::MathOptInterface.NumberOfVariables) = true
MathOptInterface.getattribute(m::MosekModel,attr::MathOptInterface.NumberOfVariables) = m.publicnumvar

MathOptInterface.cangetattribute(m::MosekSolver,attr::MathOptInterface.NumberOfConstraints) = true
MathOptInterface.cangetattribute{F,D}(m::MosekModel,attr::MathOptInterface.NumberOfConstraints{F,D}) = true
MathOptInterface.getattribute{F,D}(m::MosekModel,attr::MathOptInterface.NumberOfConstraints{F,D}) = length(select(m.constrmap,F,D))

#MathOptInterface.cangetattribute(m::MosekSolver,attr::MathOptInterface.ListOfVariableReferences) = false
#MathOptInterface.cangetattribute(m::MosekSolver,attr::MathOptInterface.ListOfConstraints) = false

MathOptInterface.cangetattribute{F,D}(m::MosekSolver,attr::MathOptInterface.ListOfConstraintReferences{F,D}) = true
MathOptInterface.getattribute{F,D}(m::MosekSolver,attr::MathOptInterface.ListOfConstraintReferences{F,D}) = keys(select(m.constrmap,F,D))

#### Warm start values

MathOptInterface.cansetattribute(m::MosekSolver,attr::MathOptInterface.VariablePrimalStart) = true
MathOptInterface.cansetattribute(m::MosekModel,attr::MathOptInterface.VariablePrimalStart) = true
function MathOptInterface.setattribute!(m::MosekModel,attr::MathOptInterface.VariablePrimalStart, v :: MathOptInterface.VariableReference, val::Float64)
    subj = getindexes(m.x_block,ref2id(v))

    xx = Float64[val]
    for sol in [ MSK_SOL_BAS, MSK_SOL_ITG ]
        putxxslice(m.task,sol,Int(subj),Int(subj+1),xx)
    end
end

function MathOptInterface.setattribute!(m::MosekModel,attr::MathOptInterface.VariablePrimalStart, vs::Vector{MathOptInterface.VariableReference}, vals::Vector{Float64})
    subj = Array{Int}(length(vs))
    for i in 1:length(subj)
        getindexes(m.x_block,ref2id(vs[i]),subj,i)
    end

    for sol in [ MSK_SOL_BAS, MSK_SOL_ITG ]
        if solutiondef(m.task,sol)
            xx = getxx(m.task,sol)
            xx[subj] = vals
            putxx(m.task,sol,xx)
        else
            xx = zeros(Float64,getnumvar(m.task))
            xx[subj] = vals
            putxx(m.task,sol,xx)
        end
    end
end

MathOptInterface.cansetattribute(m::MosekSolver,attr::MathOptInterface.ConstraintPrimalStart) = false # not sure what exactly this would be...
MathOptInterface.cansetattribute(m::MosekModel,attr::MathOptInterface.ConstraintPrimalStart) = false

MathOptInterface.cansetattribute(m::MosekSolver,attr::MathOptInterface.ConstraintDualStart) = false # for now
MathOptInterface.cansetattribute(m::MosekModel,attr::MathOptInterface.ConstraintDualStart) = false

# function MathOptInterface.setattribute!(m::MosekModel,attr::MathOptInterface.ConstraintDualStart, vs::Vector{MathOptInterface.ConstraintReference}, vals::Vector{Float64})
#     subj = Array{Int}(length(vs))
#     for i in 1:length(subj)
#         getindexes(m.x_block,ref2id(vs[i]),subj,i)
#     end

#     for sol in [ MSK_SOL_BAS, MSK_SOL_ITG ]
#         if solutiondef(m.task,sol)
#             xx = getxx(m.task,sol)
#             xx[subj] = vals
#             putxx(m.task,sol,xx)
#         else
#             xx = zeros(Float64,getnumvar(m.task))
#             xx[subj] = vals
#             putxx(m.task,sol,xx)
#         end
#     end
# end

#### Variable solution values

MathOptInterface.cangetattribute(m::MosekModel,attr::MathOptInterface.VariablePrimal) = attr.N > 0 && attr.N <= length(m.solutions)
MathOptInterface.cangetattribute(m::MosekSolver,attr::MathOptInterface.VariablePrimal) = true

MathOptInterface.cangetattribute(m::MosekModel,attr::MathOptInterface.VariablePrimal,vs::Vector{MathOptInterface.VariableReference}) = MathOptInterface.cangetattribute(m,attr)
MathOptInterface.cangetattribute(m::MosekModel,attr::MathOptInterface.VariablePrimal,v::MathOptInterface.VariableReference) = MathOptInterface.cangetattribute(m,attr)

function MathOptInterface.getattribute!(output::Vector{Float64},m::MosekModel,attr::MathOptInterface.VariablePrimal, vs::Vector{MathOptInterface.VariableReference})
    subj = Array{Int}(length(vs))
    for i in 1:length(subj)
        getindexes(m.x_block,ref2id(vs[i]),subj,i)
    end

    output[1:length(output)] = m.solutions[attr.N].xx[subj]
end

function MathOptInterface.getattribute(m::MosekModel,attr::MathOptInterface.VariablePrimal, vs::Vector{MathOptInterface.VariableReference})
    output = Vector{Float64}(length(vs))
    MathOptInterface.getattribute!(output,m,attr,vs)
    output
end

function MathOptInterface.getattribute(m::MosekModel,attr::MathOptInterface.VariablePrimal, vref::MathOptInterface.VariableReference)
    subj = getindexes(m.x_block,ref2id(vref))[1]
    m.solutions[attr.N].xx[subj]
end

MathOptInterface.cangetattribute(m::MosekSolver,attr::MathOptInterface.VariableBasisStatus) = false # is a solution selector missing?
MathOptInterface.cangetattribute(m::MosekModel,attr::MathOptInterface.VariableBasisStatus)  = false

#### Constraint solution values
function cangetattribute(m::MosekModel,attr::MathOptInterface.ConstraintPrimal)
    num = 0
    if solutiondef(m.task,MSK_SOL_ITG) num += 1 end
    if solutiondef(m.task,MSK_SOL_BAS) num += 1 end
    if solutiondef(m.task,MSK_SOL_ITR) num += 1 end
    attr.N > 0 && attr.N <= num
end

MathOptInterface.cangetattribute(m::MosekModel,attr::MathOptInterface.ConstraintPrimal, c) = cangetattribute(m,attr)
MathOptInterface.cangetattribute{F,D}(m::MosekModel,attr::MathOptInterface.ConstraintPrimal, cref::MathOptInterface.ConstraintReference{F,D}) = cangetattribute(m,attr)
MathOptInterface.cangetattribute(m::MosekModel,attr::MathOptInterface.ConstraintPrimal, cref::MathOptInterface.ConstraintReference{MathOptInterface.ScalarAffineFunction{Float64},MathOptInterface.LessThan{Float64}}) = cangetattribute(m,attr)

function MathOptInterface.getattribute{D}(
    m     ::MosekModel,
    attr  ::MathOptInterface.ConstraintPrimal,
    cref  ::MathOptInterface.ConstraintReference{MathOptInterface.SingleVariable,D})

    conid = ref2id(cref)
    idxs  = getindexes(m.xc_block,conid)
    subj  = m.xc_idxs[idxs[1]]

    m.solutions[attr.N].xx[subj]
end

# Semidefinite domain for a variable
function MathOptInterface.getattribute!(
    output::Vector{Float64},
    m     ::MosekModel,
    attr  ::MathOptInterface.ConstraintPrimal,
    cref  ::MathOptInterface.ConstraintReference{MathOptInterface.VectorOfVariables,MathOptInterface.PositiveSemidefiniteConeTriangle})

    cid = ref2id(cref)
    assert(cid < 0)

    # It is in fact a real constraint and cid is the id of an ordinary constraint
    barvaridx = - m.c_slack[-cid]
    output[1:length(output)] = getbarxj(m.task,m.solutions[attr.N].whichsol,barvaridx)
end

# Any other domain for variable vector
function MathOptInterface.getattribute!{D}(
    output::Vector{Float64},
    m     ::MosekModel,
    attr  ::MathOptInterface.ConstraintPrimal,
    cref  ::MathOptInterface.ConstraintReference{MathOptInterface.VectorOfVariables,D})

    xcid = ref2id(cref)
    assert(xcid > 0)

    mask = m.xc_bounds[xcid]
    idxs = getindexes(m.xc_block,xcid)
    subj = m.xc_idxs[idxs]

    output[1:length(output)] = m.solutions[attr.N].xx[subj]
end

function MathOptInterface.getattribute{D}(m     ::MosekModel,
                                          attr  ::MathOptInterface.ConstraintPrimal,
                                          cref  ::MathOptInterface.ConstraintReference{MathOptInterface.ScalarAffineFunction{Float64},D})
    cid = ref2id(cref)
    subi = getindexes(m.c_block,cid)[1]
    m.solutions[attr.N].xc[subi]
end

function MathOptInterface.getattribute!{D}(
    output::Vector{Float64},
    m     ::MosekModel,
    attr  ::MathOptInterface.ConstraintPrimal,
    cref  ::MathOptInterface.ConstraintReference{MathOptInterface.VectorAffineFunction{Float64},D})

    cid = ref2id(cref)
    subi = getindexes(m.c_block,cid)

    if     m.c_block_slack[cid] == 0 # no slack
        output[1:length(output)] = m.solutions[attr.N].xc[subi] + m.c_constant[subi]
    elseif m.c_block_slack[cid] >  0 # qcone slack
        xid = m.c_block_slack[cid]
        xsubj = getindexes(m.x_block, xid)
        output[1:length(output)] = m.solutions[attr.N].xx[xsubj]  + m.c_constant[subi]
    elseif m.c_block_slack[cid]  # psd slack
        xid = - m.c_block_slack[cid]
        output[1:length(output)] = getbarxj(m.task,m.solutions[attr.N].whichsol,Int32(xid)) + m.c_constant[subi]
    end
end

MathOptInterface.cangetattribute(m::MosekModel,attr::MathOptInterface.ConstraintDual) = attr.N > 0 && attr.N <= length(m.solutions)
MathOptInterface.cangetattribute(m::MosekModel,attr::MathOptInterface.ConstraintDual, crefs::Vector{MathOptInterface.ConstraintReference}) = MathOptInterface.cangetattribute(m,attr)
MathOptInterface.cangetattribute(m::MosekModel,
                                 attr::MathOptInterface.ConstraintDual,
                                 cref::MathOptInterface.ConstraintReference{F,D}) where {F,D} = MathOptInterface.cangetattribute(m,attr)

function MathOptInterface.getattribute(
    m     ::MosekModel,
    attr  ::MathOptInterface.ConstraintDual,
    cref  ::MathOptInterface.ConstraintReference{MathOptInterface.SingleVariable,D}) where { D <: MathOptInterface.AbstractSet }

    xcid = ref2id(cref)
    idxs = getindexes(m.xc_block,xcid) # variable ids

    assert(blocksize(m.xc_block,xcid) > 0)

    subj  = getindexes(m.x_block, m.xc_idxs[idxs][1])[1]
    if (getobjsense(m.task) == MSK_OBJECTIVE_SENSE_MINIMIZE)
        if     m.xc_bounds[xcid] & boundflag_lower != 0 && m.xc_bounds[xcid] & boundflag_upper != 0
            m.solutions[attr.N].slx[subj] - m.solutions[attr.N].sux[subj]
        elseif (m.xc_bounds[xcid] & boundflag_lower) != 0
            m.solutions[attr.N].slx[subj]
        elseif (m.xc_bounds[xcid] & boundflag_upper) != 0
            - m.solutions[attr.N].sux[subj]
        elseif (m.xc_bounds[xcid] & boundflag_cone) != 0
            m.solutions[attr.N].snx[subj]
        else
            error("Dual value available for this constraint")
        end
    else
        if     m.xc_bounds[xcid] & boundflag_lower != 0 && m.xc_bounds[xcid] & boundflag_upper != 0
            m.solutions[attr.N].sux[subj] - m.solutions[attr.N].slx[subj]
        elseif (m.xc_bounds[xcid] & boundflag_lower) != 0
            - m.solutions[attr.N].slx[subj]
        elseif (m.xc_bounds[xcid] & boundflag_upper) != 0
            m.solutions[attr.N].sux[subj]
        elseif (m.xc_bounds[xcid] & boundflag_cone) != 0
            - m.solutions[attr.N].snx[subj]
        else
            error("Dual value available for this constraint")
        end
    end
end

# Semidefinite domain for a variable
function MathOptInterface.getattribute!(
    output::Vector{Float64},
    m     ::MosekModel,
    attr  ::MathOptInterface.ConstraintDual,
    cref  ::MathOptInterface.ConstraintReference{MathOptInterface.VectorOfVariables,MathOptInterface.PositiveSemidefiniteConeTriangle})

    whichsol = getsolcode(m,attr.N)
    cid = ref2id(cref)
    assert(cid < 0)

    # It is in fact a real constraint and cid is the id of an ordinary constraint
    barvaridx = - m.c_slack[-cid]
    if (getobjsense(m.task) == MSK_OBJECTIVE_SENSE_MINIMIZE)
        output[1:length(output)] = getbarsj(m.task,whichsol,barvaridx)
    else
        output[1:length(output)] = - getbarsj(m.task,whichsol,barvaridx)
    end
end

# Any other domain for variable vector
function MathOptInterface.getattribute!{D}(
    output::Vector{Float64},
    m     ::MosekModel,
    attr  ::MathOptInterface.ConstraintDual,
    cref  ::MathOptInterface.ConstraintReference{MathOptInterface.VectorOfVariables,D})

    xcid = ref2id(cref)
    assert(xcid > 0)

    mask = m.xc_bounds[xcid]
    idxs = getindexes(m.xc_block,xcid)
    subj = m.xc_idxs[idxs]

    if (getobjsense(m.task) == MSK_OBJECTIVE_SENSE_MINIMIZE)
        if     m.xc_bounds[conid] & boundflag_lower != 0 && m.xc_bounds[conid] & boundflag_upper != 0
            output[1:length(output)] = m.solutions[attr.N].slx[subj] - m.solutions[attr.N].sux[subj]
        elseif (m.xc_bounds[conid] & boundflag_lower) != 0
            output[1:length(output)] = m.solutions[attr.N].slx[subj]
        elseif (m.xc_bounds[conid] & boundflag_upper) != 0
            output[1:length(output)] = - m.solutions[attr.N].sux[subj]
        elseif (m.xc_bounds[conid] & boundflag_cone) != 0
            output[1:length(output)] = m.solutions[attr.N].snx[subj]
        else
            error("Dual value available for this constraint")
        end
    else
        if     m.xc_bounds[conid] & boundflag_lower != 0 && m.xc_bounds[conid] & boundflag_upper != 0
            output[1:length(output)] = m.solutions[attr.N].sux[subj] - m.solutions[attr.N].slx[subj]
        elseif (m.xc_bounds[conid] & boundflag_lower) != 0
            output[1:length(output)] = - m.solutions[attr.N].slx[subj]
        elseif (m.xc_bounds[conid] & boundflag_upper) != 0
            output[1:length(output)] = m.solutions[attr.N].sux[subj]
        elseif (m.xc_bounds[conid] & boundflag_cone) != 0
            output[1:length(output)] = - m.solutions[attr.N].snx[subj]
        else
            error("Dual value available for this constraint")
        end
    end
end

function MathOptInterface.getattribute{D}(m     ::MosekModel,
                                          attr  ::MathOptInterface.ConstraintDual,
                                          cref  ::MathOptInterface.ConstraintReference{MathOptInterface.ScalarAffineFunction{Float64},D})
    cid = ref2id(cref)
    subi = getindexes(m.c_block,cid)[1]

    if getobjsense(m.task) == MSK_OBJECTIVE_SENSE_MINIMIZE
        m.solutions[attr.N].y[subi]
    else
        - m.solutions[attr.N].y[subi]
    end
end

function MathOptInterface.getattribute(
    m     ::MosekModel,
    attr  ::MathOptInterface.ConstraintDual,
    cref  ::MathOptInterface.ConstraintReference{MathOptInterface.VectorAffineFunction{Float64},D}) where { D <: MathOptInterface.AbstractSet }

    n = blocksize(m.c_block,ref2id(cref))
    res = Vector{Float64}(n)
    MathOptInterface.getattribute!(res,m,attr,cref)
    res
end
function MathOptInterface.getattribute!(
    output::Vector{Float64},
    m     ::MosekModel,
    attr  ::MathOptInterface.ConstraintDual,
    cref  ::MathOptInterface.ConstraintReference{MathOptInterface.VectorAffineFunction{Float64},D}) where { D <: MathOptInterface.AbstractSet }

    cid = ref2id(cref)
    subi = getindexes(m.c_block,cid)

    if getobjsense(m.task) == MSK_OBJECTIVE_SENSE_MINIMIZE
        if     m.c_block_slack[cid] == 0 # no slack
            output[1:length(output)] = m.solutions[attr.N].y[subi]
        elseif m.c_block_slack[cid] >  0 # qcone slack
            xid = m.c_block_slack[cid]
            subj = getindexes(m.x_block, xid)
            output[1:length(output)] = m.solutions[attr.N].snx[xsubj]
        elseif m.c_block_slack[cid]  # psd slack
            xid = - m.c_block_slack[cid]
            output[1:length(output)] = getbarsj(m.task,m.solutions[attr.N].whichsol,Int32(xid))
        end
    else
        if     m.c_block_slack[cid] == 0 # no slack
            output[1:length(output)] = - m.solutions[attr.N].y[subi]
        elseif m.c_block_slack[cid] >  0 # qcone slack
            xid = m.c_block_slack[cid]
            subj = getindexes(m.x_block, xid)
            output[1:length(output)] = - m.solutions[attr.N].snx[subj]
        elseif m.c_block_slack[cid]  # psd slack
            xid = - m.c_block_slack[cid]
            output[1:length(output)] = - getbarsj(m.task,m.solutions[attr.N].whichsol,Int32(xid))
        end
    end
end

solsize{F <: MathOptInterface.AbstractScalarFunction,D}(m::MosekModel, cref :: MathOptInterface.ConstraintReference{F,D}) = 1
function solsize{D}(m::MosekModel, cref :: MathOptInterface.ConstraintReference{MathOptInterface.VectorOfVariables,D})
    cid = ref2id(cref)
    if cid < 0
        blocksize(m.c_block,-cid)
    else
        blocksize(m.xc_block,cid)
    end
end

function solsize{D}(m::MosekModel, cref :: MathOptInterface.ConstraintReference{MathOptInterface.VectorAffineFunction,D})
    cid = ref2id(cref)
    blocksize(m.c_block,cid)
end

function MathOptInterface.getattribute{F <: MathOptInterface.AbstractScalarFunction,D}(m::MosekModel,attr::MathOptInterface.ConstraintPrimal, cref :: MathOptInterface.ConstraintReference{F,D})
    cid = ref2id(cref)
    output = Vector{Float64}(solsize(m,cref))
    MathOptInterface.getattribute!(output,m,attr,cref)
    output
end

#### Status codes
MathOptInterface.cangetattribute(m::MosekModel,attr::MathOptInterface.TerminationStatus) = true
function MathOptInterface.getattribute(m::MosekModel,attr::MathOptInterface.TerminationStatus)
    if     m.trm == MSK_RES_OK
        MathOptInterface.Success
    elseif m.trm == MSK_RES_TRM_MAX_ITERATIONS
        MathOptInterface.IterationLimit
    elseif m.trm == MSK_RES_TRM_MAX_TIME
        MathOptInterface.TimeLimit
    elseif m.trm == MSK_RES_TRM_OBJECTIVE_RANGE
        MathOptInterface.ObjectiveLimit
    elseif m.trm == MSK_RES_TRM_MIO_NEAR_REL_GAP
        MathOptInterface.AlmostSuccess
    elseif m.trm == MSK_RES_TRM_MIO_NEAR_ABS_GAP
        MathOptInterface.AlmostSuccess
    elseif m.trm == MSK_RES_TRM_MIO_NUM_RELAXS
        MathOptInterface.OtherLimit
    elseif m.trm == MSK_RES_TRM_MIO_NUM_BRANCHES
        MathOptInterface.NodeLimit
    elseif m.trm == MSK_RES_TRM_NUM_MAX_NUM_INT_SOLUTIONS
        MathOptInterface.SolutionLimit
    elseif m.trm == MSK_RES_TRM_STALL
        MathOptInterface.SlowProgress
    elseif m.trm == MSK_RES_TRM_USER_CALLBACK
        MathOptInterface.Interrupted
    elseif m.trm == MSK_RES_TRM_MAX_NUM_SETBACKS
        MathOptInterface.OtherLimit
    elseif m.trm == MSK_RES_TRM_NUMERICAL_PROBLEM
        MathOptInterface.SlowProgress
    elseif m.trm == MSK_RES_TRM_INTERNAL
        MathOptInterface.OtherError
    elseif m.trm == MSK_RES_TRM_INTERNAL_STOP
        MathOptInterface.OtherError
    else
        MathOptInterface.OtherError
    end
end

function MathOptInterface.cangetattribute(m::MosekModel,attr::MathOptInterface.PrimalStatus)
    if attr.N < 0 || attr.N > length(m.solutions)
        false
    else
        solsta = m.solutions[attr.N].solsta
        if     solsta == MSK_SOL_STA_UNKNOWN true
        elseif solsta == MSK_SOL_STA_OPTIMAL true
        elseif solsta == MSK_SOL_STA_PRIM_FEAS true
        elseif solsta == MSK_SOL_STA_DUAL_FEAS true
        elseif solsta == MSK_SOL_STA_PRIM_AND_DUAL_FEAS true
        elseif solsta == MSK_SOL_STA_NEAR_OPTIMAL true
        elseif solsta == MSK_SOL_STA_NEAR_PRIM_FEAS true
        elseif solsta == MSK_SOL_STA_NEAR_DUAL_FEAS true
        elseif solsta == MSK_SOL_STA_NEAR_PRIM_AND_DUAL_FEAS true
        elseif solsta == MSK_SOL_STA_PRIM_INFEAS_CER false
        elseif solsta == MSK_SOL_STA_DUAL_INFEAS_CER true
        elseif solsta == MSK_SOL_STA_NEAR_PRIM_INFEAS_CER false
        elseif solsta == MSK_SOL_STA_NEAR_DUAL_INFEAS_CER true
        elseif solsta == MSK_SOL_STA_PRIM_ILLPOSED_CER false
        elseif solsta == MSK_SOL_STA_DUAL_ILLPOSED_CER true
        elseif solsta == MSK_SOL_STA_INTEGER_OPTIMAL true
        elseif solsta == MSK_SOL_STA_NEAR_INTEGER_OPTIMAL true
        else false
        end
    end
end
function MathOptInterface.getattribute(m::MosekModel,attr::MathOptInterface.PrimalStatus)
    solsta = m.solutions[attr.N].solsta
    if     solsta == MSK_SOL_STA_UNKNOWN
        MathOptInterface.UnknownResultStatus
    elseif solsta == MSK_SOL_STA_OPTIMAL
        MathOptInterface.FeasiblePoint
    elseif solsta == MSK_SOL_STA_PRIM_FEAS
        MathOptInterface.FeasiblePoint
    elseif solsta == MSK_SOL_STA_DUAL_FEAS
        MathOptInterface.UnknownResultStatus
    elseif solsta == MSK_SOL_STA_PRIM_AND_DUAL_FEAS
        MathOptInterface.FeasiblePoint
    elseif solsta == MSK_SOL_STA_NEAR_OPTIMAL
        MathOptInterface.NearlyFeasiblePoint
    elseif solsta == MSK_SOL_STA_NEAR_PRIM_FEAS
        MathOptInterface.NearlyFeasiblePoint
    elseif solsta == MSK_SOL_STA_NEAR_DUAL_FEAS
        MathOptInterface.UnknownResultStatus
    elseif solsta == MSK_SOL_STA_NEAR_PRIM_AND_DUAL_FEAS
        MathOptInterface.NearFeasiblePoint
    elseif solsta == MSK_SOL_STA_PRIM_INFEAS_CER
        MathOptInterface.UnknownResultStatus
    elseif solsta == MSK_SOL_STA_DUAL_INFEAS_CER
        MathOptInterface.InfeasibilityCertificate
    elseif solsta == MSK_SOL_STA_NEAR_PRIM_INFEAS_CER
        MathOptInterface.UnknownResultStatus
    elseif solsta == MSK_SOL_STA_NEAR_DUAL_INFEAS_CER
        MathOptInterface.NearlyInfeasibilityCertificate
    elseif solsta == MSK_SOL_STA_PRIM_ILLPOSED_CER
        MathOptInterface.UnknownResultStatus
    elseif solsta == MSK_SOL_STA_DUAL_ILLPOSED_CER
        MathOptInterface.ReductionCertificate
    elseif solsta == MSK_SOL_STA_INTEGER_OPTIMAL
        MathOptInterface.FeasiblePoint
    elseif solsta == MSK_SOL_STA_NEAR_INTEGER_OPTIMAL
        MathOptInterface.NearlyFeasiblePoint
    else
        MathOptInterface.UnknownResultStatus
    end
end

function MathOptInterface.cangetattribute(m::MosekModel,attr::MathOptInterface.DualStatus)
    if attr.N < 0 || attr.N > length(m.solutions)
        false
    else
        solsta = m.solutions[attr.N].solsta
        if     solsta == MSK_SOL_STA_UNKNOWN true
        elseif solsta == MSK_SOL_STA_OPTIMAL true
        elseif solsta == MSK_SOL_STA_PRIM_FEAS true
        elseif solsta == MSK_SOL_STA_DUAL_FEAS true
        elseif solsta == MSK_SOL_STA_PRIM_AND_DUAL_FEAS true
        elseif solsta == MSK_SOL_STA_NEAR_OPTIMAL true
        elseif solsta == MSK_SOL_STA_NEAR_PRIM_FEAS true
        elseif solsta == MSK_SOL_STA_NEAR_DUAL_FEAS true
        elseif solsta == MSK_SOL_STA_NEAR_PRIM_AND_DUAL_FEAS true
        elseif solsta == MSK_SOL_STA_PRIM_INFEAS_CER true
        elseif solsta == MSK_SOL_STA_DUAL_INFEAS_CER false
        elseif solsta == MSK_SOL_STA_NEAR_PRIM_INFEAS_CER true
        elseif solsta == MSK_SOL_STA_NEAR_DUAL_INFEAS_CER false
        elseif solsta == MSK_SOL_STA_PRIM_ILLPOSED_CER true
        elseif solsta == MSK_SOL_STA_DUAL_ILLPOSED_CER false
        elseif solsta == MSK_SOL_STA_INTEGER_OPTIMAL false
        elseif solsta == MSK_SOL_STA_NEAR_INTEGER_OPTIMAL false
        else false
        end
    end
end
function MathOptInterface.getattribute(m::MosekModel,attr::MathOptInterface.DualStatus)
    solsta = m.solutions[attr.N].solsta
    if     solsta == MSK_SOL_STA_UNKNOWN
        MathOptInterface.UnknownResultStatus
    elseif solsta == MSK_SOL_STA_OPTIMAL
        MathOptInterface.FeasiblePoint
    elseif solsta == MSK_SOL_STA_PRIM_FEAS
        MathOptInterface.UnknownResultStatus
    elseif solsta == MSK_SOL_STA_DUAL_FEAS
        MathOptInterface.FeasiblePoint
    elseif solsta == MSK_SOL_STA_PRIM_AND_DUAL_FEAS
        MathOptInterface.FeasiblePoint
    elseif solsta == MSK_SOL_STA_NEAR_OPTIMAL
        MathOptInterface.NearlyFeasiblePoint
    elseif solsta == MSK_SOL_STA_NEAR_PRIM_FEAS
        MathOptInterface.UnknownResultStatus
    elseif solsta == MSK_SOL_STA_NEAR_DUAL_FEAS
        MathOptInterface.NearlyFeasiblePoint
    elseif solsta == MSK_SOL_STA_NEAR_PRIM_AND_DUAL_FEAS
        MathOptInterface.NearFeasiblePoint
    elseif solsta == MSK_SOL_STA_PRIM_INFEAS_CER
        MathOptInterface.InfeasibilityCertificate
    elseif solsta == MSK_SOL_STA_DUAL_INFEAS_CER
        MathOptInterface.UnknownResultStatus
    elseif solsta == MSK_SOL_STA_NEAR_PRIM_INFEAS_CER
        MathOptInterface.NearlyInfeasibilityCertificate
    elseif solsta == MSK_SOL_STA_NEAR_DUAL_INFEAS_CER
        MathOptInterface.UnknownResultStatus
    elseif solsta == MSK_SOL_STA_PRIM_ILLPOSED_CER
        MathOptInterface.ReductionCertificate
    elseif solsta == MSK_SOL_STA_DUAL_ILLPOSED_CER
        MathOptInterface.UnknownResultStatus
    elseif solsta == MSK_SOL_STA_INTEGER_OPTIMAL
        MathOptInterface.FeasiblePoint
    elseif solsta == MSK_SOL_STA_NEAR_INTEGER_OPTIMAL
        MathOptInterface.NearlyFeasiblePoint
    else
        MathOptInterface.UnknownResultStatus
    end
end


const MOI = MathOptInterface

MOI.Utilities.@product_of_sets(Zeros, MOI.Zeros)

MOI.Utilities.@product_of_sets(Nonpositives, MOI.Nonpositives)

MOI.Utilities.@struct_of_constraints_by_set_types(
    StructCache,
    MOI.Zeros,
    MOI.Nonpositives,
    MOI.SecondOrderCone,
    MOI.PositiveSemidefiniteConeTriangle,
)

const OptimizerCache{T} = MOI.Utilities.GenericModel{
    T,
    MOI.Utilities.ObjectiveContainer{T},
    MOI.Utilities.VariablesContainer{T},
    StructCache{T}{
        MOI.Utilities.MatrixOfConstraints{
            T,
            MOI.Utilities.MutableSparseMatrixCSC{
                T,
                Int64,
                MOI.Utilities.OneBasedIndexing,
            },
            Vector{T},
            Zeros{T},
        },
        MOI.Utilities.MatrixOfConstraints{
            T,
            MOI.Utilities.MutableSparseMatrixCSC{
                T,
                Int64,
                MOI.Utilities.OneBasedIndexing,
            },
            Vector{T},
            Nonpositives{T},
        },
        MOI.Utilities.VectorOfConstraints{
            MOI.VectorOfVariables,
            MOI.SecondOrderCone,
        },
        MOI.Utilities.VectorOfConstraints{
            MOI.VectorOfVariables,
            MOI.PositiveSemidefiniteConeTriangle,
        },
    },
}

Base.@kwdef mutable struct ConeData
    psc::Vector{Vector{Int}} = Vector{Int}[] # semidefinite
    soc::Vector{Vector{Int}} = Vector{Int}[] # second order
end

mutable struct Optimizer <: MOI.AbstractOptimizer
    cones::Union{Nothing, ConeData}
    zeros::Union{Nothing, Zeros{Float64}}
    nonps::Union{Nothing, Nonpositives{Float64}}
    sol::Result
    options::Options
end

function Optimizer(;kwargs...)
    optimizer = Optimizer(
        nothing,
        nothing,
        nothing,
        Result(),
        Options()
        )
    for (key, value) in kwargs
        MOI.set(optimizer, MOI.RawOptimizerAttribute(string(key)), value)
    end
    return optimizer
end

#=
    Basic Attributes
=#

MOI.get(::Optimizer, ::MOI.SolverName) = "ProxSDP"

MOI.get(::Optimizer, ::MOI.SolverVersion) = "1.8.0"

function MOI.set(optimizer::Optimizer, param::MOI.RawOptimizerAttribute, value)
    fields = fieldnames(Options)
    name = Symbol(param.name)
    if name in fields
        setfield!(optimizer.options, name, value)
    else
        error("No parameter matching $(name)")
    end
    return value
end

function MOI.get(optimizer::Optimizer, param::MOI.RawOptimizerAttribute)
    fields = fieldnames(Options)
    name = Symbol(param.name)
    if name in fields
        return getfield(optimizer.options, name)
    else
        error("No parameter matching $(name)")
    end
end

MOI.supports(::Optimizer, ::MOI.Silent) = true

function MOI.set(optimizer::Optimizer, ::MOI.Silent, value::Bool)
    if value == true
        optimizer.options.timer_verbose = false
    end
    optimizer.options.log_verbose = !value
end

function MOI.get(optimizer::Optimizer, ::MOI.Silent)
    if optimizer.options.log_verbose
        return false
    elseif optimizer.options.timer_verbose
        return false
    else
        return true
    end
end

MOI.supports(::Optimizer, ::MOI.TimeLimitSec) = true

function MOI.set(optimizer::Optimizer, ::MOI.TimeLimitSec, value)
    optimizer.options.time_limit = value
end

function MOI.get(optimizer::Optimizer, ::MOI.TimeLimitSec)
    return optimizer.options.time_limit
end

MOI.supports(::Optimizer, ::MOI.NumberOfThreads) = false

function MOI.is_empty(optimizer::Optimizer)
    return optimizer.cones === nothing &&
    optimizer.zeros === nothing &&
    optimizer.nonps === nothing &&
    optimizer.sol.status == 0
end

function MOI.empty!(optimizer::Optimizer)
    optimizer.cones = nothing
    optimizer.zeros = nothing
    optimizer.nonps = nothing
    optimizer.sol = Result()
    return
end

function _rows(
    optimizer::Optimizer,
    ci::MOI.ConstraintIndex{MOI.VectorAffineFunction{T},MOI.Zeros},
) where T
    return MOI.Utilities.rows(optimizer.zeros, ci)
end

function _rows(
    optimizer::Optimizer,
    ci::MOI.ConstraintIndex{MOI.VectorAffineFunction{T},MOI.Nonpositives},
) where T
    return MOI.Utilities.rows(optimizer.nonps, ci)
end

# MOI.supports

function MOI.supports(
    ::Optimizer,
    ::Union{
        MOI.ObjectiveSense,
        MOI.ObjectiveFunction{MOI.ScalarAffineFunction{T}},
    },
) where T
    return true
end

function MOI.supports_constraint(
    ::Optimizer,
    ::Type{MOI.VectorAffineFunction{T}},
    ::Type{<:Union{MOI.Zeros, MOI.Nonpositives}}
) where T
    return true
end

function MOI.supports_add_constrained_variables(
    ::Optimizer,
    ::Type{<:Union{
        MOI.SecondOrderCone,
        MOI.PositiveSemidefiniteConeTriangle,
        }
    }
)
    return true
end

function cone_data(src::OptimizerCache, ::Type{T}) where T
    cache = MOI.Utilities.constraints(
        src.constraints,
        MOI.VectorOfVariables,
        T,
    )
    indices = MOI.get(
        cache,
        MOI.ListOfConstraintIndices{MOI.VectorOfVariables, T}(),
    )
    funcs = MOI.get.(cache, MOI.ConstraintFunction(), indices)
    return Vector{Int64}[Int64[v.value for v in f.variables] for f in funcs]
end

# Vectorized length for matrix dimension n
sympackedlen(n) = div(n * (n + 1), 2)
# Matrix dimension for vectorized length n
sympackeddim(n) = div(isqrt(1 + 8n) - 1, 2)

matindices(n::Integer) = LinearIndices(trues(n,n))[findall(LinearAlgebra.tril(trues(n,n)))]

function _optimize!(dest::Optimizer, src::OptimizerCache)
    MOI.empty!(dest)
    TimerOutputs.reset_timer!()

    @timeit "pre-processing" begin

    #=
        Affine
    =#
    Ab = MOI.Utilities.constraints(
        src.constraints,
        MOI.VectorAffineFunction{Float64},
        MOI.Zeros,
    )
    A = Ab.coefficients
    b = -Ab.constants
    Gh = MOI.Utilities.constraints(
        src.constraints,
        MOI.VectorAffineFunction{Float64},
        MOI.Nonpositives,
    )
    G = Gh.coefficients
    h = -Gh.constants
    @assert A.n == G.n
    #=
        Objective
    =#
    max_sense = MOI.get(src, MOI.ObjectiveSense()) == MOI.MAX_SENSE
    obj =
        MOI.get(src, MOI.ObjectiveFunction{MOI.ScalarAffineFunction{Float64}}())
    objective_constant = MOI.constant(obj)
    c = zeros(A.n)
    obj_sign = ifelse(max_sense, -1.0, 1.0)
    for term in obj.terms
        c[term.variable.value] += obj_sign * term.coefficient
    end
    dest.zeros = deepcopy(Ab.sets) # TODO copy(Ab.sets)
    dest.nonps = deepcopy(Gh.sets) # TODO copy(Gh.sets)
    # TODO: simply this after
    # https://github.com/jump-dev/MathOptInterface.jl/issues/1711
    _A = if A.m == 0
        SparseArrays.sparse(Int64[], Int64[], Float64[], 0, A.n)
    else
        convert(SparseArrays.SparseMatrixCSC{Float64,Int64}, A)
    end
    _G = if G.m == 0
        SparseArrays.sparse(Int64[], Int64[], Float64[], 0, G.n)
    else
        convert(SparseArrays.SparseMatrixCSC{Float64,Int64}, G)
    end
    aff = AffineSets(A.n, A.m, G.m, 0,
        _A, _G, b, h, c)
    #=
        Cones
    =#
    con = ConicSets(
        SDPSet[],
        SOCSet[]
        )
    soc_s = cone_data(src, MOI.SecondOrderCone)
    for soc in soc_s
        push!(con.socone, SOCSet(soc, length(soc)))
    end
    psc_s = cone_data(src, MOI.PositiveSemidefiniteConeTriangle)
    for psc in psc_s
        tri_len = length(psc)
        sq_side = sympackeddim(tri_len)
        mat_inds = matindices(sq_side)
        push!(con.sdpcone, SDPSet(psc, mat_inds, tri_len, sq_side))
    end
    dest.cones = ConeData(
        psc_s,
        soc_s,
    )
    #
    end # timeit

    #=
        Solve modified problem
    =#

    options = dest.options

    # warm = WarmStart()

    if options.disable_julia_logger
        # disable logger
        global_log = Logging.current_logger()
        Logging.global_logger(Logging.NullLogger())
    end

    sol = @timeit "Main" chambolle_pock(aff, con, options)

    if options.disable_julia_logger
        # re-enable logger
        Logging.global_logger(global_log)
    end

    if options.timer_verbose
        TimerOutputs.print_timer(TimerOutputs.DEFAULT_TIMER)
        print("\n")
        TimerOutputs.print_timer(TimerOutputs.flatten(TimerOutputs.DEFAULT_TIMER))
        print("\n")
    end
    if options.timer_file
        f = open("time.log","w")
        TimerOutputs.print_timer(f,TimerOutputs.DEFAULT_TIMER)
        print(f,"\n")
        TimerOutputs.print_timer(f,TimerOutputs.flatten(TimerOutputs.DEFAULT_TIMER))
        print(f,"\n")
        close(f)
    end

    #=
        Fix solution
    =#

    sol.objval = obj_sign * sol.objval + objective_constant
    sol.dual_objval = obj_sign * sol.dual_objval + objective_constant

    dest.sol = sol

    return
end

function MOI.optimize!(dest::Optimizer, src::OptimizerCache)
    _optimize!(dest, src)
    return MOI.Utilities.identity_index_map(src), false
end

function MOI.optimize!(dest::Optimizer, src::MOI.ModelLike)
    cache = OptimizerCache{Float64}()
    index_map = MOI.copy_to(cache, src)
    _optimize!(dest, cache)
    return index_map, false
end

MOI.supports_incremental_interface(::Optimizer) = false

#=
    Attributes
=#

MOI.get(optimizer::Optimizer, ::MOI.SolveTimeSec) = optimizer.sol.time

"""
    PDHGIterations()

The number of PDHG iterations completed during the solve.
"""
struct PDHGIterations <: MOI.AbstractModelAttribute end

MOI.is_set_by_optimize(::PDHGIterations) = true

function MOI.get(optimizer::Optimizer, ::PDHGIterations)
    return Int64(optimizer.sol.iter)
end

function MOI.get(optimizer::Optimizer, ::MOI.RawStatusString)
    return optimizer.sol.status_string
end

function MOI.get(optimizer::Optimizer, ::MOI.TerminationStatus)
    s = optimizer.sol.status
    @assert 0 <= s <= 6
    if s == 0
        return MOI.OPTIMIZE_NOT_CALLED
    elseif s == 1
        return MOI.OPTIMAL
    elseif s == 2
        return MOI.TIME_LIMIT
    elseif s == 3
        return MOI.ITERATION_LIMIT
    elseif s == 4
        return MOI.INFEASIBLE_OR_UNBOUNDED
    elseif s == 5
        return MOI.DUAL_INFEASIBLE
    elseif s == 6
        return MOI.INFEASIBLE
    end
end

function MOI.get(optimizer::Optimizer, attr::MOI.ObjectiveValue)
    MOI.check_result_index_bounds(optimizer, attr)
    value = optimizer.sol.objval
    return value
end

function MOI.get(optimizer::Optimizer, attr::MOI.DualObjectiveValue)
    MOI.check_result_index_bounds(optimizer, attr)
    value = optimizer.sol.dual_objval
    return value
end

function MOI.get(optimizer::Optimizer, attr::MOI.PrimalStatus)
    s = optimizer.sol.status
    if attr.result_index > 1 || s == 0
        return MOI.NO_SOLUTION
    end
    if s == 5 && optimizer.sol.certificate_found
        return MOI.INFEASIBILITY_CERTIFICATE
    end
    if optimizer.sol.primal_feasible_user_tol#s == 1
        return MOI.FEASIBLE_POINT
    else
        return MOI.INFEASIBLE_POINT
    end
end

function MOI.get(optimizer::Optimizer, attr::MOI.DualStatus)
    s = optimizer.sol.status
    if attr.result_index > 1 || s ==0
        return MOI.NO_SOLUTION
    end
    if s == 6 && optimizer.sol.certificate_found
        return MOI.INFEASIBILITY_CERTIFICATE
    end
    if optimizer.sol.dual_feasible_user_tol
        return MOI.FEASIBLE_POINT
    else
        return MOI.INFEASIBLE_POINT
    end
end

function MOI.get(optimizer::Optimizer, ::MOI.ResultCount)
    return optimizer.sol.result_count
end

#=
    Results
=#

function MOI.get(
    optimizer::Optimizer,
    attr::MOI.VariablePrimal,
    vi::MOI.VariableIndex,
)
    MOI.check_result_index_bounds(optimizer, attr)
    return optimizer.sol.primal[vi.value]
end

function MOI.get(
    optimizer::Optimizer,
    attr::MOI.ConstraintPrimal,
    ci::MOI.ConstraintIndex{MOI.VectorAffineFunction{Float64}, MOI.Zeros},
)
    MOI.check_result_index_bounds(optimizer, attr)
    return optimizer.sol.slack_eq[_rows(optimizer, ci)]
end

function MOI.get(
    optimizer::Optimizer,
    attr::MOI.ConstraintPrimal,
    ci::MOI.ConstraintIndex{MOI.VectorAffineFunction{Float64}, MOI.Nonpositives},
)
    MOI.check_result_index_bounds(optimizer, attr)
    return optimizer.sol.slack_in[_rows(optimizer, ci)]
end

function MOI.get(
    optimizer::Optimizer,
    attr::MOI.ConstraintPrimal,
    ci::MOI.ConstraintIndex{MOI.VectorOfVariables, MOI.PositiveSemidefiniteConeTriangle}
)
    MOI.check_result_index_bounds(optimizer, attr)
    return optimizer.sol.primal[optimizer.cones.psc[ci.value]]
end

function MOI.get(
    optimizer::Optimizer,
    attr::MOI.ConstraintPrimal,
    ci::MOI.ConstraintIndex{MOI.VectorOfVariables, MOI.SecondOrderCone}
)
    MOI.check_result_index_bounds(optimizer, attr)
    return optimizer.sol.primal[optimizer.cones.soc[ci.value]]
end

function MOI.get(
    optimizer::Optimizer,
    attr::MOI.ConstraintDual,
    ci::MOI.ConstraintIndex{MOI.VectorAffineFunction{Float64},MOI.Zeros},
)
    MOI.check_result_index_bounds(optimizer, attr)
    return -optimizer.sol.dual_eq[_rows(optimizer, ci)]
end

function MOI.get(
    optimizer::Optimizer,
    attr::MOI.ConstraintDual,
    ci::MOI.ConstraintIndex{MOI.VectorAffineFunction{Float64},MOI.Nonpositives},
)
    MOI.check_result_index_bounds(optimizer, attr)
    return -optimizer.sol.dual_in[_rows(optimizer, ci)]
end

function MOI.get(
    optimizer::Optimizer,
    attr::MOI.ConstraintDual,
    ci::MOI.ConstraintIndex{MOI.VectorOfVariables, MOI.PositiveSemidefiniteConeTriangle}
)
    MOI.check_result_index_bounds(optimizer, attr)
    return optimizer.sol.dual_cone[optimizer.cones.psc[ci.value]]
end

function MOI.get(
    optimizer::Optimizer,
    attr::MOI.ConstraintDual,
    ci::MOI.ConstraintIndex{MOI.VectorOfVariables, MOI.SecondOrderCone}
)
    MOI.check_result_index_bounds(optimizer, attr)
    return optimizer.sol.dual_cone[optimizer.cones.soc[ci.value]]
end


module EntityIds

using ..Ahorn, Maple

EntityTriggerType = Union{Maple.Entity, Maple.Trigger}

entityIdState = nothing
validIds = Ahorn.MultiRange()

# These types should be unsafe for getting new ids because of ridge gate/condition block
unsafeForReassign = Type[
    Strawberry,
    GoldenStrawberry,
    MemorialTextController,
    Key
]

function nextId()
    res, state = entityIdState === nothing ? iterate(validIds) : iterate(validIds, entityIdState)
    global entityIdState = state

    return res
end

function getUsedIds(map::Maple.Map)
    usedIds = Int[]

    for room in map.rooms
        append!(usedIds, Int.(getfield.(room.entities, :id)))
        append!(usedIds, Int.(getfield.(room.triggers, :id)))
    end

    return usedIds
end

function groupByIds(map::Maple.Map)
    res = Dict{Int, Array{EntityTriggerType, 1}}()

    for room in map.rooms
        for targets in [room.triggers, room.entities]
            for target in targets
                id = Int(target.id)

                get!(Array{EntityTriggerType, 1}, res, id)
                push!(res[id], target)
            end
        end
    end

    return res
end

function attemptIdFixing(map::Maple.Map, idGroupDict=groupByIds(map), force::Bool=false)
    success = true
    reassigned = 0

    for (id, targets) in idGroupDict
        if length(targets) > 1
            unsafeCount = 0

            for (i, target) in enumerate(targets)
                unsafeTargets = Bool[typeof(target) in unsafeForReassign for target in targets]
                unsafeCount = sum(unsafeTargets)

                if force
                    if i != 1
                        target.id = nextId()
                        reassigned += 1
                    end

                else
                    if unsafeCount > 0 && !unsafeTargets[i]
                        target.id = nextId()
                        reassigned += 1

                    elseif unsafeCount == 0 && i != 1
                        target.id = nextId()
                        reassigned += 1
                    end
                end
            end

            if unsafeCount > 1 && !force
                success = false
            end
        end
    end

    return success, reassigned
end

function getValidIdRanges(map::Maple.Map)
    usedIds = getUsedIds(map)
    sortedIds = sort(usedIds)

    gaps = UnitRange{Int}[]
    prev = -1

    for v in sortedIds
        if v !== prev && v - 1 !== prev
            push!(gaps, prev + 1:v - 1)
        end

        prev = v
    end

    push!(gaps, prev + 1:typemax(Int))

    return Ahorn.MultiRange(gaps)
end

function updateValidIds(map::Maple.Map)
    global validIds = getValidIdRanges(map)
    global entityIdState = nothing
end

end


using MetalheadUtils
using Test
using Images, Metalhead

image_urls = ["https://upload.wikimedia.org/wikipedia/commons/d/d7/Green_Sea_Turtle_grazing_seagrass.jpg", "https://upload.wikimedia.org/wikipedia/commons/0/0a/The_Great_Wave_off_Kanagawa.jpg",
"https://upload.wikimedia.org/wikipedia/commons/b/b4/Vassily_Kandinsky%2C_1913_-_Composition_7.jpg",
"https://upload.wikimedia.org/wikipedia/commons/0/00/Tuebingen_Neckarfront.jpg",
"https://upload.wikimedia.org/wikipedia/commons/6/68/Pillars_of_creation_2014_HST_WFC3-UVIS_full-res_denoised.jpg",
"https://upload.wikimedia.org/wikipedia/commons/thumb/e/ea/Van_Gogh_-_Starry_Night_-_Google_Art_Project.jpg/1024px-Van_Gogh_-_Starry_Night_-_Google_Art_Project.jpg"];

@testset "MetalheadUtils.jl" begin
    images = imgFromURL.(image_urls)
    @test length(images) == length(image_urls)
    @test images[1] isa Array{RGB{FixedPointNumbers.Normed{UInt8,8}},2}

    img1 = postprocess(Metalhead.preprocess(images[1]))
    @test img1 isa Array{RGB{FixedPointNumbers.Normed{UInt8,8}},2}
    r = img1 - impreprocess(images[1])
    @test maximum(abs.(channelview(r))) <= 1/500
end

using Flux
@testset "Features" begin
    r = Chain(x->x+1, x->x+2)
    @test Features(r, 20, [1]) == [21]
    @test Features(r, 20, [1,2]) == [21,23]

    vgg = VGG19()
    img1 = Metalhead.preprocess(imgFromURL(image_urls[1]))
    @test Features(vgg, img1, [1,2]) == [Features(vgg, img1, [1])[1], Features(vgg, img1, [2])[1]]
end


const _show_pad = 12
const _show_signif = 5

function Base.show(io::IO, bs::BootstrapSample)
    r = _show_pad
    s = _show_signif
    df = estimate_summary(bs)
    df = split(string(df), '\n')[2:end]
    df[1] = replace(df[1], "Row", "Var")
    println(io, "Bootstrap Sampling")
    println(io, rpad("  Estimates:", r))
    for i in 1:length(df)
        println(io, string("    ", df[i]))
    end
    println(io, rpad("  Sampling:", r),
            replace(string(typeof(sampling(bs))), "Bootstrap.", ""))
    println(io, rpad("  Samples:", r), nrun(bs))
    println(io, rpad("  Data:", r), data_summary(bs.data))
end

"""
Summarizes an object in the form of 'TYPE: { SIZE }'
"""
function data_summary(x)
    t = string(typeof(x))
    n = length(size(x)) > 0 ? size(x) : length(x)
    ns = join(n, " \u00D7 ") # Unicode 'times'
    s = "$t: { $ns }"
    return s
end

function estimate_summary(bs::BootstrapSample)
    n = nvar(bs)
    df = DataFrame(Estimate = [original(bs)...],
                   Bias = bias(bs),
                   StdError = se(bs))
    return df
end


# Interface methods

rasterlayers(::Type{EarthEnv{HabitatHeterogeneity}}) =
    (:cv, :evenness, :range, :shannon, :simpson, :std, :Contrast, :Correlation,
     :Dissimilarity, :Entropy, :Homogeneity, :Maximum, :Uniformity, :Variance)

rastername(::Type{EarthEnv{HabitatHeterogeneity}}, layer::Symbol, resolution::Integer) =
    "$(layer)_$(resolution)km.tif"

rasterpath(T::Type{<:EarthEnv{HabitatHeterogeneity}}, layer::Symbol, resolution::Integer) =
    joinpath(rasterpath(T), string(resolution) * "km", rastername(T, layer, resolution))

rasterurl(T::Type{EarthEnv{HabitatHeterogeneity}}, layer, resolution::Integer) =
    joinpath(rasterurl(T), "$(layer)_01_05_$(string(resolution))km_$(_getprecision(layer, resolution)).tif")

function download_raster(T::Type{EarthEnv{HabitatHeterogeneity}}; layer::Symbol=:cv, resolution::Integer=25)
    resolution in [1, 5, 25] || throw(ArgumentError("The resolution must be 1km, 5km, or 25km"))
    layer in rasterlayers(T) || throw(ArgumentError("The layer must be one of $(rasterlayers(T))"))

    path = rasterpath(T, layer, resolution)
    url = rasterurl(T, layer, resolution)
    return _maybe_download(url, path)
end

# Utility methods

_pathsegment(::Type{HabitatHeterogeneity}) = "habitat_heterogeneity"

function _getprecision(layer, resolution)
    precision = "uint16"
    # WELLCOME TO HELL
    precision = ((resolution == 10) && (layer == :cv)) ? "uint32" : precision
    precision = (layer == :Contrast) ? "uint32" : precision
    precision = (layer == :Dissimilarity) ? "uint32" : precision
    precision = (layer == :Variance) ? "uint32" : precision
    precision = ((resolution == 25) && (layer == :Entropy)) ? "uint32" : precision
    # /HELL
end