module V4

using MSTM_jll: mpiexec, mstm
using DataFrames
using DocStringExtensions
using Printf
using Scanf
using UUIDs
using ...STMMRunner
export run_mstm

abstract type MSTMOutput end

struct NearField
    spheres::DataFrame
    field::DataFrame
end

struct MSTMFixedOutput <: MSTMOutput
    q_ext_unpolarized::Float64
    q_abs_unpolarized::Float64
    q_scat_unpolarized::Float64
    q_ext_parallel::Float64
    q_abs_parallel::Float64
    q_scat_parallel::Float64
    q_ext_perpendicular::Float64
    q_abs_perpendicular::Float64
    q_scat_perpendicular::Float64
    q_scat_down_unpolarized::Float64
    q_scat_up_unpolarized::Float64
    q_scat_down_parallel::Float64
    q_scat_up_parallel::Float64
    q_scat_down_perpendicular::Float64
    q_scat_up_perpendicular::Float64
    q_ext_down_unpolarized::Float64
    q_ext_up_unpolarized::Float64
    q_ext_down_parallel::Float64
    q_ext_up_parallel::Float64
    q_ext_down_perpendicular::Float64
    q_ext_up_perpendicular::Float64
    q_scat_waveguide_unpolarized::Float64
    q_scat_waveguide_parallel::Float64
    q_scat_waveguide_perpendicular::Float64
    scattering_matrix::DataFrame
    near_field::Union{NearField,Nothing}
end

struct MSTMLatticeOutput <: MSTMOutput
    ref_unpolarized::Float64
    abs_unpolarized::Float64
    scat_unpolarized::Float64
    ref_parallel::Float64
    abs_parallel::Float64
    scat_parallel::Float64
    ref_perpendicular::Float64
    abs_perpendicular::Float64
    scat_perpendicular::Float64
    scattering_matrix::DataFrame
end

struct MSTMRandomOutput <: MSTMOutput
    q_ext::Float64
    q_abs::Float64
    q_scat::Float64
    scattering_matrix::DataFrame
end

struct MSTMMonteCarloOutput <: MSTMOutput
    q_ext::Float64
    q_abs::Float64
    q_scat::Float64
    q_scat_down_unpolarized::Float64
    q_scat_up_unpolarized::Float64
    q_scat_down_parallel::Float64
    q_scat_up_parallel::Float64
    q_scat_down_perpendicular::Float64
    q_scat_up_perpendicular::Float64
    scattering_matrix_azimuthal_average::DataFrame
    scattering_matrix_diffuse::DataFrame
    scattering_matrix_expansion_coefficients::DataFrame
end

"""
Use the given configuration to run MSTM v4.

$(SIGNATURES)

- If `keep = true`, the working directory will not be removed after the run.
"""
function run_mstm(cfg::STMMConfig; keep::Bool = false)
    current_dir = pwd()

    id = string(uuid1())
    if cfg.working_directory == ""
        dir = joinpath(pwd(), "mstm_tmp_$id")
    else
        dir = cfg.working_directory
    end
    results = nothing
    proc = nothing

    try
        if !ispath(dir)
            @debug "[Run MSTM] Creating workding directory"
            mkdir(dir)
        end

        @debug "[Run MSTM] Entering working directory"
        cd(dir)

        @debug "[Run MSTM] Writing MSTM input file"
        write_input(cfg)

        # We cannot simply do `$(mpiexec()) ... $(mstm())` here due to the limit of Cmd interpolation.
        # See https://github.com/JuliaLang/julia/issues/39282 for more details.
        @debug "[Run MSTM] Running MSTM..."
        proc = open(
            setenv(`$(mpiexec().exec[1]) -n $(cfg.number_processors) $(mstm().exec[1])`, mstm().env),
            stdout;
            write = true
        )
        wait(proc)

        @debug "[Run MSTM] Collecting MSTM output"
        results = collect_output(cfg)
    catch e
        @error "[Run MSTM] $e"
        @error "[Run MSTM] Runtime error occurred"
    finally
        @debug "[Run MSTM] Leaving working directory"
        cd(current_dir)

        if !isnothing(proc)
            @debug "[Run MSTM] Killing MSTM process"
            kill(proc)
            sleep(0.2)
            if process_running(proc)
                @warn "Failed to kill, please retry manually."
            end
        end

        if !keep && occursin(String(id), dir)
            @assert ispath(dir)
            @debug "[Run MSTM] Removing temporary directory"
            rm(dir; force = true, recursive = true)
        end

        return results
    end
end

function format_sphere(sphere::Sphere)::String
    radius_and_center = @sprintf "%.4e %.4e %.4e %.4e " sphere.x sphere.y sphere.z sphere.r

    if !isempty(sphere.t_matrix)
        return radius_and_center * sphere.t_matrix
    else
        mₗ = sphere.m / (1.0 - sphere.β * sphere.m)
        mᵣ = sphere.m / (1.0 + sphere.β * sphere.m)

        return radius_and_center *
               (@sprintf "(%.4e,%.4e) (%.4e,%.4e)" mₗ.re mₗ.im mᵣ.re mᵣ.im)
    end
end

function format_spheres(cfg::STMMConfig)
    return """
number_spheres
$(length(cfg.spheres))
sphere_data
$(join(map(format_sphere, cfg.spheres),"\n"))
end_of_sphere_data"""
end

function format_fft_options(cfg::STMMConfig)
    if !cfg.use_fft_translation
        return "fft_translation_option\nfalse"
    else
        return """
        fft_translation_option
        true
        min_fft_nsphere
        $(length(cfg.spheres))
        cell_volume_fraction
        $(cfg.cell_volume_fraction)
        node_order
        $(cfg.node_order)"""
    end
end

# FIXME Optically active layers are not supported in MSTM v4 yet.
# See https://github.com/dmckwski/MSTM/issues/6
function format_layers(cfg::STMMConfig)::String
    N = length(cfg.layers)

    return """
number_plane_boundaries
$(N - 1)
layer_thickness
$(N >= 3 ? join(map(x -> x.d, cfg.layers[2:N-1]), ",") : "")
layer_ref_index
$(join(map(x -> (@sprintf "(%.4e,%.4e)" x.m.re x.m.im), cfg.layers), ","))"""
end

function format_orientation(cfg::STMMConfig)::String
    if cfg.orientation == RandomOrientation
        if cfg.use_monte_carlo_integration
            return "incidence_average\ntrue\nnumber_incident_directions\n$(cfg.number_incident_directions)"
        else
            return "random_orientation\ntrue"
        end
    else
        return "incident_alpha_deg\n$(cfg.α)\nincident_beta_deg\n$(cfg.β)\nazimuthal_average\n$(cfg.azimuthal_average)"
    end
end

function format_near_field(cfg::STMMConfig)::String
    if cfg.calculate_near_field && cfg.orientation == FixedOrientation
        return """
    calculate_near_field
    true
    near_field_output_file
    $(cfg.near_field_output_file)
    near_field_minimum_border
    $(cfg.near_field_x_min),$(cfg.near_field_y_min),$(cfg.near_field_z_min)
    near_field_maximum_border
    $(cfg.near_field_x_max),$(cfg.near_field_y_max),$(cfg.near_field_z_max)
    near_field_step_size
    $(cfg.near_field_step_size)
    near_field_calculation_model
    $(cfg.near_field_calculation_model)
    store_surface_vector
    $(cfg.store_surface_vector)""" * (cfg.store_surface_vector ? """
    \nnear_field_expansion_spacing
    $(cfg.near_field_expansion_spacing)
    near_field_expansion_order
    $(cfg.near_field_expansion_order)""" : "")
    else
        return "calculate_near_field\nfalse"
    end
end

function format_lattice(cfg::STMMConfig)
    if !cfg.periodic
        return "periodic_lattice\nfalse"
    else
        if cfg.orientation == RandomOrientation
            throw(ErrorException("Lattice mode is incompatible with random operation!"))
        end

        x_min = minimum(s.x - s.r for s in cfg.spheres)
        x_max = maximum(s.x + s.r for s in cfg.spheres)
        y_min = minimum(s.y - s.r for s in cfg.spheres)
        y_max = maximum(s.y + s.r for s in cfg.spheres)
        dx = x_max - x_min
        dy = y_max - y_min
        cx = max(dx, cfg.cell_size[1])
        cy = max(dy, cfg.cell_size[2])
        return "periodic_lattice\ntrue\ncell_width\n$cx,$cy"
    end
end

# We set `print_sphere_data` to `false` since we do not need them.
function write_input(cfg::STMMConfig)
    inp = """
    $(format_spheres(cfg))
    $(format_layers(cfg))
    output_file
    $(cfg.output_file)
    run_file
    $(cfg.run_print_file)
    print_sphere_data
    false
    mie_epsilon
    $(cfg.mie_epsilon)
    translation_epsilon
    $(cfg.translation_epsilon)
    solution_epsilon
    $(cfg.solution_epsilon)
    t_matrix_convergence_epsilon
    $(cfg.t_matrix_epsilon)
    max_iterations
    $(cfg.max_iterations)
    max_t_matrix_order
    $(cfg.max_t_matrix_order)
    store_translation_matrix
    $(cfg.store_translation_matrix)
    $(format_fft_options(cfg))
    $(format_orientation(cfg))
    $(format_near_field(cfg))
    incident_frame
    $(cfg.frame == IncidentFrame)
    normalize_s11
    $(cfg.normalize_scattering_matrix)
    gaussian_beam_constant
    $(cfg.gaussian_beam_constant)
    gaussian_beam_focal_point
    $(@sprintf("(%.4e,%.4e,%.4e)", cfg.gaussian_beam_focal_point...))
    calculate_scattering_matrix
    $(cfg.calculate_scattering_matrix)
    scattering_map_model
    $(cfg.scattering_map_model)
    scattering_map_increment
    $(cfg.Δθ)
    scattering_map_dimension
    $(cfg.scattering_map_dimension)
    $(format_lattice(cfg))
    end_of_options
    """

    open("mstm.inp", "w") do io
        write(io, inp)
    end
end

function read_ints(s::AbstractString)::Vector{Int}
    return map(x -> parse(Int, x), filter(!isempty, split(s, r"\s+")))
end

function read_floats(s::AbstractString)::Vector{Float64}
    return map(x -> parse(Float64, x), filter(!isempty, split(s, r"\s+")))
end

function collect_output(cfg::STMMConfig)::MSTMOutput
    @assert isfile(cfg.output_file)

    out = split(read(open(cfg.output_file), String), "\n")
    i = 1

    if cfg.orientation == FixedOrientation
        q = cfg.periodic ? zeros(9) : zeros(24)
    else
        q = cfg.use_monte_carlo_integration ? zeros(9) : zeros(3)
    end

    θ = Float64[]
    kx = Float64[]
    ky = Float64[]
    s11 = Float64[]
    s12 = Float64[]
    s13 = Float64[]
    s14 = Float64[]
    s21 = Float64[]
    s22 = Float64[]
    s23 = Float64[]
    s24 = Float64[]
    s31 = Float64[]
    s32 = Float64[]
    s33 = Float64[]
    s34 = Float64[]
    s41 = Float64[]
    s42 = Float64[]
    s43 = Float64[]
    s44 = Float64[]
    type = String[]

    while !occursin("calculation results for run", out[i])
        i += 1
    end

    # Skip header
    i += cfg.orientation == FixedOrientation ? 4 : 6

    if cfg.orientation == FixedOrientation
        if cfg.periodic
            # Lattice mode
            q[1:9] = collect(@scanf out[i+1] "%e%e%e%e%e%e%e%e%e" zeros(9)...)[2:end]

            i += 6
        elseif length(cfg.layers) == 1
            # One layer

            # Read total efficiencies
            q[1:9] = read_floats(out[i+1])
            i += 2

            # Read hemispherical scattering efficiencies
            q[10:15] = read_floats(out[i+1])
            i += 3
        else
            # Multiple layers

            # Read total efficiencies
            q[1:9] = read_floats(out[i+1])
            i += 2

            # Read down/up extinction efficiencies
            q[16:21] = read_floats(out[i+1])
            i += 2

            # Read hemispherical scattering efficiencies
            q[10:15] = read_floats(out[i+1])
            i += 2

            # Read waveguide scattering efficiencies
            q[22:24] = read_floats(out[i+1])
            i += 3
        end

        current_type = "total"
        if occursin("reflection", out[i])
            current_type = "reflection"
            i += 4
        elseif occursin("kx", out[i])
            current_type = "backward"
            i += 1
        else
            i += 3
        end

        # Read scattering matrix
        while !isempty(out[i])
            if occursin("transmission", out[i])
                current_type = "transmission"
                i += 2
                continue
            elseif occursin("forward", out[i])
                current_type = "forward"
                i += 2
                if cfg.periodic
                    i += 2
                end
                continue
            end

            v = read_floats(out[i])
            i += 1

            push!(type, current_type)

            # TODO: need to check whether lattice output can support θ instead kx-ky convention
            if cfg.scattering_map_model == 0 && !cfg.periodic
                push!(θ, v[1])
            else
                push!(kx, v[1])
                push!(ky, v[2])
            end

            push!(s44, v[end])
            push!(s43, v[end-1])
            push!(s42, v[end-2])
            push!(s41, v[end-3])
            push!(s34, v[end-4])
            push!(s33, v[end-5])
            push!(s32, v[end-6])
            push!(s31, v[end-7])
            push!(s24, v[end-8])
            push!(s23, v[end-9])
            push!(s22, v[end-10])
            push!(s21, v[end-11])
            push!(s14, v[end-12])
            push!(s13, v[end-13])
            push!(s12, v[end-14])
            push!(s11, v[end-15])
        end

        if cfg.scattering_map_model == 0 && !cfg.periodic
            scattering_matrix = DataFrame(;
                θ,
                s11,
                s12,
                s13,
                s14,
                s21,
                s22,
                s23,
                s24,
                s31,
                s32,
                s33,
                s34,
                s41,
                s42,
                s43,
                s44,
                type
            )
        else
            scattering_matrix = DataFrame(;
                kx,
                ky,
                s11,
                s12,
                s13,
                s14,
                s21,
                s22,
                s23,
                s24,
                s31,
                s32,
                s33,
                s34,
                s41,
                s42,
                s43,
                s44,
                type
            )
        end

        if cfg.calculate_near_field && isfile(cfg.near_field_output_file)
            nf = split(read(open(cfg.near_field_output_file), String), "\n")

            Ns = read_ints(nf[3])[1]
            sx = Float64[]
            sy = Float64[]
            sz = Float64[]
            r = Float64[]
            for j = 1:Ns
                v = read_floats(nf[j+3])
                push!(sx, v[1])
                push!(sy, v[2])
                push!(sz, v[3])
                push!(r, v[4])
            end
            spheres = DataFrame(; x = sx, y = sy, z = sz, r)

            Nb = read_ints(nf[Ns+4])[1]
            Nx, Ny, Nz = read_ints(nf[Ns+Nb+7])

            # Read near field data
            x = Float64[]
            y = Float64[]
            z = Float64[]
            Ex₌ = ComplexF64[]
            Ex⊥ = ComplexF64[]
            Ey₌ = ComplexF64[]
            Ey⊥ = ComplexF64[]
            Ez₌ = ComplexF64[]
            Ez⊥ = ComplexF64[]
            Hx₌ = ComplexF64[]
            Hx⊥ = ComplexF64[]
            Hy₌ = ComplexF64[]
            Hy⊥ = ComplexF64[]
            Hz₌ = ComplexF64[]
            Hz⊥ = ComplexF64[]
            for j = 1:Nx*Ny*Nz
                v = read_floats(nf[j+7+Ns+Nb])
                push!(x, v[1])
                push!(y, v[2])
                push!(z, v[3])
                push!(Ex₌, complex(v[4], v[5]))
                push!(Ey₌, complex(v[6], v[7]))
                push!(Ez₌, complex(v[8], v[9]))
                push!(Hx₌, complex(v[10], v[11]))
                push!(Hy₌, complex(v[12], v[13]))
                push!(Hz₌, complex(v[14], v[15]))
                push!(Ex⊥, complex(v[16], v[17]))
                push!(Ey⊥, complex(v[18], v[19]))
                push!(Ez⊥, complex(v[20], v[21]))
                push!(Hx⊥, complex(v[22], v[23]))
                push!(Hy⊥, complex(v[24], v[25]))
                push!(Hz⊥, complex(v[26], v[27]))
            end

            field = DataFrame(;
                x,
                y,
                z,
                Ex₌,
                Ex⊥,
                Ey₌,
                Ey⊥,
                Ez₌,
                Ez⊥,
                Hx₌,
                Hx⊥,
                Hy₌,
                Hy⊥,
                Hz₌,
                Hz⊥
            )

            near_field = NearField(spheres, field)
        else
            near_field = nothing
        end

        return cfg.periodic ? MSTMLatticeOutput(q..., scattering_matrix) : MSTMFixedOutput(q..., scattering_matrix, near_field)
    elseif !cfg.use_monte_carlo_integration
        # Random orientation, analytical average

        # Read efficiencies
        q[1:3] = read_floats(out[i+1])
        i += 5

        # Read scattering matrix
        Nθ = Int(round((cfg.θ_max - cfg.θ_min) / cfg.Δθ)) + 1
        for j = 1:Nθ
            v = read_floats(out[i+j])

            push!(θ, v[1])
            push!(s11, v[2])
            push!(s12, v[3])
            push!(s22, v[4])
            push!(s33, v[5])
            push!(s34, v[6])
            push!(s44, v[7])
        end
        scattering_matrix = DataFrame(; θ, s11, s12, s22, s33, s34, s44)

        return MSTMRandomOutput(q..., scattering_matrix)
    else
        # Random orientation, Monte Carlo

        # Read efficiencies
        q[1:3] = read_floats(out[i+1])
        i += 2

        q[4:9] = read_floats(out[i+1])
        i += 4

        # Read scattering matrix
        Nθ, _ = read_ints(out[i])
        i += 1

        for j = 1:Nθ
            v = read_floats(out[i+j])
            push!(θ, v[1])
            push!(s11, v[2])
            push!(s12, v[3])
            push!(s22, v[4])
            push!(s33, v[5])
            push!(s34, v[6])
            push!(s44, v[7])
        end

        scattering_matrix_azimuthal_average = DataFrame(; θ, s11, s12, s22, s33, s34, s44)

        for v in [θ, s11, s12, s22, s33, s34, s44]
            resize!(v, 0)
        end

        i += Nθ + 1
        @assert occursin("diffuse", out[i])
        i += 1

        for j = 1:Nθ
            v = read_floats(out[i+j])
            push!(θ, v[1])
            push!(s11, v[2])
            push!(s12, v[3])
            push!(s22, v[4])
            push!(s33, v[5])
            push!(s34, v[6])
            push!(s44, v[7])
        end

        scattering_matrix_diffuse = DataFrame(; θ, s11, s12, s22, s33, s34, s44)

        i += Nθ + 1
        @assert occursin(
            "azimuthal averaged scattering matrix expansion coefficients",
            out[i],
        )
        i += 2

        n = Int[]
        a11 = Float64[]
        a44 = Float64[]
        a12 = Float64[]
        a34 = Float64[]
        a22p = Float64[]
        a22m = Float64[]

        while !isempty(out[i])
            v = read_floats(out[i])

            push!(n, v[1])
            push!(a11, v[2])
            push!(a44, v[3])
            push!(a12, v[4])
            push!(a34, v[5])
            push!(a22p, v[6])
            push!(a22m, v[7])

            i += 1
        end

        scattering_matrix_expansion_coefficients =
            DataFrame(; n, a11, a44, a12, a34, a22p, a22m)

        return MSTMMonteCarloOutput(
            q...,
            scattering_matrix_azimuthal_average,
            scattering_matrix_diffuse,
            scattering_matrix_expansion_coefficients,
        )
    end
end

end


#!/usr/bin/env julia

"""
gff_feature.jl - Houses functions that process GFF features.

By: Shaun Adkins (sadkins@som.umaryland.edu)
"""

isstranded(strand_type::String) = return (strand_type == "no" ? false : true)

function create_uniq_coords_dict(features::Array{GFF3.Record,1}, stranded_type::String)
    uniq_coords = Dict{String, Dict}()
    seqids = Set(map(x -> GFF3.seqid(x), features))
    for seqid in seqids
        uniq_coords[seqid] = get_nonoverlapping_coords_by_seqid(features, seqid)
        # If alignment data is unstranded, then symdiff both strands to + strand
        if !isstranded(stranded_type)
            # Ensure a default empty set is used if that strand has no coordinates
            symdiff!(get!(uniq_coords[seqid],'+', BitSet()), pop!(uniq_coords[seqid], '-', BitSet()))
        end
    end
    return uniq_coords
end

function get_feature_coords_set(feature::GFF3.Record)
    """Get the range of coordinates for this feature, returned as a Set."""
    return BitSet(GFF3.leftposition(feature) : GFF3.rightposition(feature))
end

function get_featurename_from_attrs(feature::GFF3.Record, attr_type::String)
    """Get attribute ID to use as the feature name."""
    gene_vector = GFF3.attributes(feature, attr_type)    # col 9
    validate_feature_attribute(gene_vector)
    return gene_vector[1]
end

function get_feature_nonoverlapping_coords(feature::GFF3.Record, uniq_coords::Dict{String, Dict}, stranded::Bool)
    """Get set of nonoverlapping coordinates for given feature."""
    seqid = GFF3.seqid(feature)
    strand = getstrand(feature, stranded)
    feat_coords = get_feature_coords_set(feature)
    return intersect(feat_coords, uniq_coords[seqid][strand])
end

function get_nonoverlapping_coords_by_seqid(features::Array{GFF3.Record,1}, seqid::String)
    """ Get dictionary nonoverlapping coordinates based on the features on this sequence ID."""
    uniq_seqid_coords = Dict{Char,BitSet}()
    # Get only features with this reference id
    seqid_feats = filter(x -> GFF3.seqid(x) == seqid, features)
    strand = Set(map(x -> GFF3.strand(x), seqid_feats))
    for s in strand
        strandchar = convert(Char, s)
        uniq_seqid_coords[strandchar] = get_nonoverlapping_coords_by_seqid_and_strand(seqid_feats, s)
    end
    return uniq_seqid_coords
end

function get_nonoverlapping_coords_by_seqid_and_strand(features::Array{GFF3.Record,1}, strand::Strand)
    """ Get set of nonoverlapping coordinates based on the features for this strand of this sequence ID."""
    # Get features one strand at a time
    seqid_feats_by_strand = filter(x-> GFF3.strand(x) == strand, features)
    return BitSet(mapreduce(x -> get_feature_coords_set(x), symdiff, seqid_feats_by_strand))
end

function getstrand(feature::GFF3.Record, stranded::Bool)
    """Get strand of GFF3 feature with respect to strandedness arguments."""
    return getstrand(convert(Interval, feature), stranded)
end

function validate_feature_attribute(gene_vector::Vector{String})
    """Ensure only one attribute of this type exists for this feature record."""
    length(gene_vector) > 0 || error("ERROR - Attribute field 'ID' found to have no entries.\n")
    length(gene_vector) == 1 || error("ERROR - Attribute field 'ID' found to have multiple entries.\n")
end


## Majority system for individual dances

"""
    skating_single_dance(results)
    skating_single_dance(results, majority_from; [initial_place, initial_column, depth])

Calculates a skating tableau and placement for judgment results of a single dance contained
in the DataFrame `results`. It must contain the starter numbers of the dancers as a first
column named `:Numbers`, and the places assigned by the different judges in the following
columns, for example as in
```julia
DataFrame(Number = [11, 12, 13, 14, 15],
        JudgeA = [1, 2, 3, 4, 5],
        JudgeB = [1, 3, 2, 4, 5],
        JudgeC = [1, 3, 2, 5, 4],
        JudgeD = [5, 1, 2, 3, 4],
        JudgeE = [2, 1, 5, 3, 4])
```

Returns a DataFrame with the skating tableau represented as strings, as well as a DataFrame
with the starter numbers and places.

For calculations on cropped tables, the majority of votes, the initial placement to fill,
the initial column in the tableau where skating should start as well as the desired depth of
the calculation can be defined by keyword arguments.
"""
function skating_single_dance(results::DataFrame, majority_from::Int; initial_place::Int = 1,
    initial_column::Int = 1, depth = size(results, 1))

    calculation, sum_of_eval = prepare_sums(results, depth)

    calculation[!, :Place] .= 0.0
    calculation_text = string.(copy(calculation))

    current_place = initial_place
    max_cols = depth
    current_col = initial_column
    tmp_col = initial_column
    append_sum = false
    while current_place <= (initial_column - 1 + size(results, 1))
        if count(>=(majority_from), calculation[!, current_col+1], dims = 1)[1] == 1
            # @info "Clear Majority"
            idx = findfirst(>=(majority_from), calculation[!, current_col+1])
            write_result!(calculation, calculation_text, sum_of_eval, idx, current_place, current_col, current_col, max_cols)
            current_place += 1
            current_col += 1
        else
            # @info "Multiple Majorities"
            idx = findall(>=(majority_from), calculation[!, current_col+1])
            tmp_col = copy(current_col)
            while !isempty(idx)
                if count(==(maximum(calculation[idx, tmp_col+1])), calculation[idx, tmp_col+1]) == 1
                    # @info "Single maximal majority"
                    id = idx[findfirst(==(maximum(calculation[idx, tmp_col+1])), calculation[idx, tmp_col+1])]
                    write_result!(calculation, calculation_text, sum_of_eval, id, current_place, current_col, tmp_col, max_cols; append_sum = append_sum)
                    current_place += 1
                    append_sum = false
                    remove!(idx, id)
                else
                    # equal majority
                    # look at sum of evaluations up to current column
                    if length(findall(==(maximum(sum_of_eval[idx, tmp_col+1])), sum_of_eval[idx, tmp_col+1])) == 1
                        # @info "Single sum minority"
                        id = findfirst(==(minimum(sum_of_eval[idx, tmp_col+1])), sum_of_eval[idx, tmp_col+1])
                        write_result!(calculation, calculation_text, sum_of_eval, idx[id], current_place, current_col, tmp_col, max_cols; append_sum = true)
                        current_place += 1
                        append_sum = true
                        remove!(idx, idx[id])
                    else
                        # @info "Multiple equal sums"
                        calculation_text[idx, (tmp_col+1)] = string.(calculation[idx, (tmp_col+1)]) .* "*" .* "(" .* string.(sum_of_eval[idx, (tmp_col+1)]) .* ")"
                        tmp_col += 1
                        if tmp_col > max_cols
                            # @info "tmp_cols > max_cols"
                            places = range(current_place, length = length(idx))
                            place = mean(places)
                            calculation[idx, :Place] .= place
                            calculation_text[idx, :Place] .= string(place)
                            current_place = maximum(places) + 1
                            calculation[idx, (current_col+1):(max_cols+1)] .= 0
                            idx = []
                            break
                        end
                    end

                end
            end
            current_col += 1
        end
    end

    return calculation_text, calculation[!, [:Number, :Place]]
end

function skating_single_dance(results::DataFrame; initial_place = 1, initial_column = 1, depth = size(results, 1))
    return skating_single_dance(results::DataFrame, calc_majority(results);
        initial_place = initial_place, initial_column = initial_column, depth = depth)
end

"""
    calc_majority(results)

Returns the judge majority for a result DataFrame `results`.
"""
function calc_majority(results)
    return ceil(Int, (size(results, 2) - 2) / 2 + 1)
end

"""
    write_result!(calculation, calculation_text, sum_of_eval, id, current_place, current_col, tmp_col, max_cols; append_sum = false)

Write the result of the skating procedure to the DataFrames containing placement and summary
information. Remove the parts that are not needed anymore.
"""
function write_result!(calculation, calculation_text, sum_of_eval, index, current_place, current_col, tmp_col, max_cols;
    append_sum = false)
    calculation[index, :Place] = current_place
    calculation_text[index, :Place] = string(current_place)
    str = string(calculation[index, (tmp_col+1)]) * "*"
    if append_sum
        str = str * "(" * string(sum_of_eval[index, (tmp_col+1)]) * ")"
    end
    calculation_text[index, (tmp_col+1)] = str
    calculation_text[index, (tmp_col+2):(max_cols+1)] .= "-"
    calculation[index, (current_col+1):(max_cols+1)] .= 0
end


using Fermi.HartreeFock

import Base: show

export RMP2

"""
    Fermi.HartreeFock.RHFAlgorithm

Abstract type for RMP2 implementations.
"""
abstract type RMP2Algorithm end

"""
    Fermi.MollerPlesset.get_rmp2_alg()

Returns a singleton type corresponding to a RMP2 implementation based on the options.
"""
function get_rmp2_alg()
    implemented = [RMP2a()]
    N = Options.get("mp2_alg")
    try
        return implemented[N]
    catch BoundsError
        throw(FermiException("implementation number $N not available for RMP2."))
    end
end

"""
    RMP2{T} <: AbstractMPWavefunction

    TODO
"""
struct RMP2{T} <: AbstractMPWavefunction
    correlation::T
    energy::AbstractFloat
end

# For each implementation a singleton type must be create
struct RMP2a <: RMP2Algorithm end
include("RMP2a.jl")

function RMP2(x...)
    if !any(i-> i isa RMP2Algorithm, x)
        RMP2(x..., get_rmp2_alg())
    else
        # Print the type of arguments given for a better feedback
        args = "("
        for a in x[1:end-1]
            args *= "$(typeof(a)), "
        end
        args = args[1:end-2]*")"
        throw(FermiException("invalid arguments for RMP2 method: $args"))
    end
end

## MISCELLANEOUS
# Pretty printing
function string_repr(X::RMP2)
    out = ""
    out = out*" ⇒ Fermi Restricted MP2 Wave function\n"
    out = out*" ⋅ Correlation Energy:     $(X.correlation)\n"
    out = out*" ⋅ Total Energy:           $(X.energy)"
    return out
end

function show(io::IO, ::MIME"text/plain", X::RMP2)
    print(io, string_repr(X))
end


export gmm_generate

import ..GMM

using mlpack._Internal.io

import mlpack_jll
const gmm_generateLibrary = mlpack_jll.libmlpack_julia_gmm_generate

# Call the C binding of the mlpack gmm_generate binding.
function gmm_generate_mlpackMain()
  success = ccall((:gmm_generate, gmm_generateLibrary), Bool, ())
  if !success
    # Throw an exception---false means there was a C++ exception.
    throw(ErrorException("mlpack binding error; see output"))
  end
end

" Internal module to hold utility functions. "
module gmm_generate_internal
  import ..gmm_generateLibrary

import ...GMM

# Get the value of a model pointer parameter of type GMM.
function IOGetParamGMM(paramName::String)::GMM
  GMM(ccall((:IO_GetParamGMMPtr, gmm_generateLibrary), Ptr{Nothing}, (Cstring,), paramName))
end

# Set the value of a model pointer parameter of type GMM.
function IOSetParamGMM(paramName::String, model::GMM)
  ccall((:IO_SetParamGMMPtr, gmm_generateLibrary), Nothing, (Cstring, Ptr{Nothing}), paramName, model.ptr)
end

# Serialize a model to the given stream.
function serializeGMM(stream::IO, model::GMM)
  buf_len = UInt[0]
  buf_ptr = ccall((:SerializeGMMPtr, gmm_generateLibrary), Ptr{UInt8}, (Ptr{Nothing}, Ptr{UInt}), model.ptr, Base.pointer(buf_len))
  buf = Base.unsafe_wrap(Vector{UInt8}, buf_ptr, buf_len[1]; own=true)
  write(stream, buf)
end
# Deserialize a model from the given stream.
function deserializeGMM(stream::IO)::GMM
  buffer = read(stream)
  GMM(ccall((:DeserializeGMMPtr, gmm_generateLibrary), Ptr{Nothing}, (Ptr{UInt8}, UInt), Base.pointer(buffer), length(buffer)))
end
end # module

"""
    gmm_generate(input_model, samples; [seed, verbose])

This program is able to generate samples from a pre-trained GMM (use gmm_train
to train a GMM).  The pre-trained GMM must be specified with the `input_model`
parameter.  The number of samples to generate is specified by the `samples`
parameter.  Output samples may be saved with the `output` output parameter.

The following command can be used to generate 100 samples from the pre-trained
GMM `gmm` and store those generated samples in `samples`:

```julia
julia> samples = gmm_generate(gmm, 100)
```

# Arguments

 - `input_model::GMM`: Input GMM model to generate samples from.
 - `samples::Int`: Number of samples to generate.
 - `seed::Int`: Random seed.  If 0, 'std::time(NULL)' is used.  Default
      value `0`.

 - `verbose::Bool`: Display informational messages and the full list of
      parameters and timers at the end of execution.  Default value `false`.

# Return values

 - `output::Array{Float64, 2}`: Matrix to save output samples in.

"""
function gmm_generate(input_model::GMM,
                      samples::Int;
                      seed::Union{Int, Missing} = missing,
                      verbose::Union{Bool, Missing} = missing,
                      points_are_rows::Bool = true)
  # Force the symbols to load.
  ccall((:loadSymbols, gmm_generateLibrary), Nothing, ());

  IORestoreSettings("GMM Sample Generator")

  # Process each input argument before calling mlpackMain().
  gmm_generate_internal.IOSetParamGMM("input_model", convert(GMM, input_model))
  IOSetParam("samples", samples)
  if !ismissing(seed)
    IOSetParam("seed", convert(Int, seed))
  end
  if verbose !== nothing && verbose === true
    IOEnableVerbose()
  else
    IODisableVerbose()
  end

  IOSetPassed("output")
  # Call the program.
  gmm_generate_mlpackMain()

  return IOGetParamMat("output", points_are_rows)
end


function read_ettj(dt::Date)
	url = "https://www.anbima.com.br/informacoes/est-termo/CZ-down.asp"
	header = [
	    "Host" => "www.anbima.com.br"
	    "User-Agent" => "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:80.0) Gecko/20100101 Firefox/80.0"
	    "Accept" => "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8"
	    "Accept-Language" => "pt-BR,pt;q=0.8,en-US;q=0.5,en;q=0.3"
	    "Accept-Encoding" => "gzip, deflate, br"
	    "Content-Type" => "application/x-www-form-urlencoded"
	    "Origin" => "https://www.anbima.com.br"
	    "Connection" => "keep-alive"
	    "Referer" => "https://www.anbima.com.br/informacoes/est-termo/default.asp"
	    "Upgrade-Insecure-Requests" => "1"
	]
	str_dt = Dates.format(dt, "dd%2Fmm%2Fyyyy")
	body = "Idioma=PT&Dt_Ref=$str_dt&saida=csv"
	r = HTTP.post(url, header, body)

    response_body = String(r.body)
    @assert _has_data_ettj(response_body) "no ETTJ data for date $dt"
    filedate, out = _parse_ettj_result(response_body)
    @assert dt == filedate "Wrong date: expected $dt, got $filedate"
    return out
end

function _has_data_ettj(str::String)
	out = split(str, "\r\n")
	return length(out) >= 3
end

function _parse_ettj_result(str::String)
	out = split(str, "\r\n")
	date = _parse_ettj_date(String(out[1]))
	cvpre = _parse_ettj_parameters(String(out[2]))
	cvipca = _parse_ettj_parameters(String(out[3]))
	return date, Dict([cvpre, cvipca])
end

function _parse_ettj_date(line::String)::Date
	s = split(line, ";")
	return Date(String(s[1]), "dd/mm/yyyy")
end

function _parse_ettj_parameters(line::String)::Pair{String, SvenssonCurve}
	s = split(line, ";")
	@assert length(s) == 7 "bad ettj csv: expected 7 columns, got $(length(s))"
	name = String(s[1])
	cv = SvenssonCurve(
		parse_value(Float64, s[2]),
		parse_value(Float64, s[3]),
		parse_value(Float64, s[4]),
		parse_value(Float64, s[5]),
		parse_value(Float64, s[6]),
		parse_value(Float64, s[7])
	)
	return (name => cv)
end


@testset "Site counting" begin

    dna_alphabets = (DNAAlphabet{4}, DNAAlphabet{2})
    rna_alphabets = (RNAAlphabet{4}, RNAAlphabet{2})

    @testset "Specific count methods" begin

        function generate_possibilities_tester(::Type{A}) where A <: NucAlphs
            symbols = alphabet(A)
            arra = Vector{eltype(A)}()
            arrb = Vector{eltype(A)}()
            for i in 1:length(symbols), j in i:length(symbols)
                push!(arra, symbols[i])
                push!(arrb, symbols[j])
            end
            return BioSequence{A}(arra), BioSequence{A}(arrb)
        end

        @testset "2 bit" begin
            dnaA, dnaB = generate_possibilities_tester(DNAAlphabet{2})
            rnaA, rnaB = generate_possibilities_tester(RNAAlphabet{2})

            @test count(Conserved, dnaA, dnaB) == count(Conserved, dnaB, dnaA) == 4
            @test count(Mutated, dnaA, dnaB) == count(Mutated, dnaB, dnaA) == 6

            @test count(Conserved, rnaA, rnaB) == count(Conserved, rnaB, rnaA) == 4
            @test count(Mutated, rnaA, rnaB) == count(Mutated, rnaB, rnaA) == 6
        end

        @testset "4 bit" begin
            dnaA, dnaB = generate_possibilities_tester(DNAAlphabet{4})
            rnaA, rnaB = generate_possibilities_tester(RNAAlphabet{4})

            @test count(Conserved, dnaA, dnaB) == count(Conserved, dnaB, dnaA) == (4, 10)
            @test count(Mutated, dnaA, dnaB) == count(Mutated, dnaB, dnaA) == (6, 10)

            @test count(Conserved, rnaA, rnaB) == count(Conserved, rnaB, rnaA) == (4, 10)
            @test count(Mutated, rnaA, rnaB) == count(Mutated, rnaB, rnaA) == (6, 10)
        end
    end

    @testset "Randomized tests" begin
        # A test counting function which is naive.
        @inline function issite(::Type{Conserved}, a::BioSequence, b::BioSequence, idx)
            return a[idx] == b[idx]
        end
        @inline function issite(::Type{Mutated}, a::BioSequence, b::BioSequence, idx)
            return a[idx] != b[idx]
        end
        @inline function testcount2(::Type{P}, a::BioSequence, b::BioSequence) where P <: BioSequences.Position
            k = 0
            @inbounds for idx in 1:min(lastindex(a), lastindex(b))
                k += issite(P, a, b, idx)
            end
            return k
        end
        @inline function testcount(::Type{P}, a::BioSequence, b::BioSequence) where P <: BioSequences.Position
            k, c = 0, 0
            @inbounds for idx in 1:min(lastindex(a), lastindex(b))
                isvalid = !(isgap(a[idx]) || isgap(b[idx])) && !(isambiguous(a[idx]) || isambiguous(b[idx]))
                k += issite(P, a, b, idx) && isvalid
                c += isvalid
            end
            return k, c
        end
        function testcounting(::Type{S}, a::BioSequence, b::BioSequence) where S <: Site
            @test count(S, a, b) == count(S, b, a) == testcount(S, a, b)
        end
        function testcounting2(::Type{S}, a::BioSequence, b::BioSequence) where S <: Site
            @test count(S, a, b) == count(S, b, a) == testcount2(S, a, b)
        end
        function testforencs(a::Int, b::Int, subset::Bool)
            for alphabet in (DNAAlphabet, RNAAlphabet)
                for _ in  1:10
                    seqA = random_seq(alphabet{a}, rand(10:100))
                    seqB = random_seq(alphabet{b}, rand(10:100))
                    sa = seqA
                    sb = seqB
                    if subset
                        intA = random_interval(1, length(seqA))
                        intB = random_interval(1, length(seqB))
                        subA = seqA[intA]
                        subB = seqB[intB]
                        sa = subA
                        sb = subB
                    end
                    if a == 2 && b == 2
                        testcounting2(Conserved, sa, sb)
                        testcounting2(Mutated, sa, sb)
                    else
                        testcounting(Conserved, sa, sb)
                        testcounting(Mutated, sa, sb)
                    end
                end
            end
        end

        @testset "Testing 4-bit seq pairs" begin
            @testset "Full random sequences" begin
                testforencs(4, 4, false)
            end
            @testset "Subset random sequences" begin
                testforencs(4, 4, true)
            end
        end

        @testset "Testing 2-bit seq pairs" begin
            @testset "Full random sequences" begin
                testforencs(2, 2, false)
            end
            @testset "Subset random sequences" begin
                testforencs(2, 2, true)
            end
        end

        @testset "Testing mixed bit seq pairs" begin
            @testset "Full random sequences" begin
                testforencs(4, 2, false)
                testforencs(2, 4, false)
            end
            @testset "Subset random sequences" begin
                testforencs(4, 2, true)
                testforencs(2, 4, true)
            end
        end
    end

    @testset "Pairwise methods" begin
        @testset "4-bit encoded sequences" begin
            dnas = [dna"ATCGCCA-", dna"ATCGCCTA", dna"ATCGCCT-", dna"GTCGCCTA"]
            rnas = [rna"AUCGCCA-", rna"AUCGCCUA", rna"AUCGCCU-", rna"GUCGCCUA"]

            answer_mutated = [(0,0) (1,7) (1,7) (2,7);
                              (1,7) (0,0) (0,7) (1,8);
                              (1,7) (0,7) (0,0) (1,7);
                              (2,7) (1,8) (1,7) (0,0)]

            answer_conserved = [(0,0) (6,7) (6,7) (5,7);
                                (6,7) (0,0) (7,7) (7,8);
                                (6,7) (7,7) (0,0) (6,7);
                                (5,7) (7,8) (6,7) (0,0)]
            for i in (dnas, rnas)
                @test count_pairwise(Mutated, i...) == answer_mutated
                @test count_pairwise(Conserved, i...) == answer_conserved
            end
        end
        @testset "2-bit encoded sequences" begin
            dnas = [BioSequence{DNAAlphabet{2}}("ATCGCCAC"),
                    BioSequence{DNAAlphabet{2}}("ATCGCCTA"),
                    BioSequence{DNAAlphabet{2}}("ATCGCCTT"),
                    BioSequence{DNAAlphabet{2}}("GTCGCCTA")]
            rnas = [BioSequence{RNAAlphabet{2}}("AUCGCCAC"),
                    BioSequence{RNAAlphabet{2}}("AUCGCCUA"),
                    BioSequence{RNAAlphabet{2}}("AUCGCCUU"),
                    BioSequence{RNAAlphabet{2}}("GUCGCCUA")]
            answer_mutated = [0 2 2 3;
                              2 0 1 1;
                              2 1 0 2;
                              3 1 2 0]
            answer_conserved = [0 6 6 5;
                                6 0 7 7;
                                6 7 0 6;
                                5 7 6 0]
            for i in (dnas, rnas)
                @test count_pairwise(Mutated, i...) == answer_mutated
                @test count_pairwise(Conserved, i...) == answer_conserved
            end
        end
    end

    @testset "Windowed methods" begin
        @testset "4-bit encoded sequences" begin
            dnaA = dna"ATCGCCA-M"
            dnaB = dna"ATCGCCTAA"
            rnaA = rna"AUCGCCA-M"
            rnaB = rna"AUCGCCUAA"

            for seqs in ((dnaA, dnaB), (rnaA, rnaB))
                @test count(Conserved, seqs[1], seqs[2], 3, 1) == [IntervalValue(1, 3, (3,3)),
                                                                   IntervalValue(2, 4, (3,3)),
                                                                   IntervalValue(3, 5, (3,3)),
                                                                   IntervalValue(4, 6, (3,3)),
                                                                   IntervalValue(5, 7, (2,3)),
                                                                   IntervalValue(6, 8, (1,2)),
                                                                   IntervalValue(7, 9, (0,1))]
                @test count(Mutated, seqs[1], seqs[2], 3, 1) == [IntervalValue(1, 3, (0,3)),
                                                                 IntervalValue(2, 4, (0,3)),
                                                                 IntervalValue(3, 5, (0,3)),
                                                                 IntervalValue(4, 6, (0,3)),
                                                                 IntervalValue(5, 7, (1,3)),
                                                                 IntervalValue(6, 8, (1,2)),
                                                                 IntervalValue(7, 9, (1,1))]
            end
        end

        @testset "2-bit encoded sequences" begin
            dnaA = BioSequence{DNAAlphabet{2}}("ATCGCCATT")
            dnaB = BioSequence{DNAAlphabet{2}}("ATCGCCTAA")
            rnaA = BioSequence{RNAAlphabet{2}}("AUCGCCAUU")
            rnaB = BioSequence{RNAAlphabet{2}}("AUCGCCUAA")

            for seqs in ((dnaA, dnaB), (rnaA, rnaB))
                @test count(Conserved, seqs[1], seqs[2], 3, 1) == [IntervalValue(1, 3, 3),
                                                                   IntervalValue(2, 4, 3),
                                                                   IntervalValue(3, 5, 3),
                                                                   IntervalValue(4, 6, 3),
                                                                   IntervalValue(5, 7, 2),
                                                                   IntervalValue(6, 8, 1),
                                                                   IntervalValue(7, 9, 0)]
                @test count(Mutated, seqs[1], seqs[2], 3, 1) == [IntervalValue(1, 3, 0),
                                                                 IntervalValue(2, 4, 0),
                                                                 IntervalValue(3, 5, 0),
                                                                 IntervalValue(4, 6, 0),
                                                                 IntervalValue(5, 7, 1),
                                                                 IntervalValue(6, 8, 2),
                                                                 IntervalValue(7, 9, 3)]
            end
        end

        @testset "Mixed encodings" begin
            dnaA = dna"ATCGCCA-M"
            dnaB = BioSequence{DNAAlphabet{2}}("ATCGCCTAA")
            rnaA = rna"AUCGCCA-M"
            rnaB = BioSequence{RNAAlphabet{2}}("AUCGCCUAA")

            for seqs in ((dnaA, dnaB), (rnaA, rnaB))
                @test count(Conserved, seqs[1], seqs[2], 3, 1) == [IntervalValue(1, 3, (3,3)),
                                                                   IntervalValue(2, 4, (3,3)),
                                                                   IntervalValue(3, 5, (3,3)),
                                                                   IntervalValue(4, 6, (3,3)),
                                                                   IntervalValue(5, 7, (2,3)),
                                                                   IntervalValue(6, 8, (1,2)),
                                                                   IntervalValue(7, 9, (0,1))]
                @test count(Mutated, seqs[1], seqs[2], 3, 1) == [IntervalValue(1, 3, (0,3)),
                                                                 IntervalValue(2, 4, (0,3)),
                                                                 IntervalValue(3, 5, (0,3)),
                                                                 IntervalValue(4, 6, (0,3)),
                                                                 IntervalValue(5, 7, (1,3)),
                                                                 IntervalValue(6, 8, (1,2)),
                                                                 IntervalValue(7, 9, (1,1))]
            end
        end
    end
end


const _etol_f64 = 1.45e-8
const _etol_f32 = 3.45f-4

@inline _etol(::Float64) = _etol_f64
@inline _etol(::Type{Float64}) = _etol_f64
@inline _etol(::Float32) = _etol_f32
@inline _etol(::Type{Float32}) = _etol_f32

@inline _kcfg(n::Int) = (threads = 128, blocks = cld(n, 128))

abstract type ReadOnlyArray{T, N} <: AbstractArray{T, N} end
abstract type NoAccessArray{T, N} <: AbstractArray{T, N} end

@inline Base.getindex(A::NoAccessArray, ::Int) = error("getindex not defined for ", typeof(A))

@inline Base.setindex!(
    A::Union{ReadOnlyArray, NoAccessArray},
     ::Any,
     ::Int
) = error("setindex! not defined for ", typeof(A))

@inline Base.show(io::IO, ::MIME"text/plain", A ::NoAccessArray) = Base.show(io, A)
@inline Base.show(io::IO, A::NoAccessArray) = print(io, join(size(A), "×"), " ", typeof(A))

function _solve_linear_system(A::AbstractArray{T, 2}, b::AbstractArray{T, 1}) where T
    gmres(A, b,
        verbose=true,
        restart=min(200, size(A, 2)),
        Pl=DiagonalPreconditioner(A)
    )
end


# ---------------------------------------------------------------
# Read all the NCA5 files and reformat them to a file
# thats usable throughout the report. All heights in mm
#
# For each scenario, the following files are made:
#   NCA5_scn_gmsl.nc: Global-mean sea level for each scenario
#   NCA5_scn_grid.nc: Local sea level on a 1° grid
#   NCA5_scn_tg.nc: Local sea level at PSMSL tide-gauge locations
# This script only has to be run once.
# ---------------------------------------------------------------
module ConvertNCA5ToGrid

using NetCDF
using DelimitedFiles
using Statistics
using NCDatasets
using Interpolations
using DelimitedFiles

function RunConvertNCA5ToGrid(settings)
    println("\nProcessing NCA5 scenarios...")
    # ConvertGMSL(settings)
    for scn ∈ 1:length(settings["NCA5_scenarios"])
        println(" "*settings["NCA5_scenarios"][scn]*"...")
        ConvertGMSL(scn,settings)
        ConvertRegional(scn,settings)
        println(" "*settings["NCA5_scenarios"][scn]*" done")

    end
    println("Processing NCA5 scenarios done\n")
    return nothing
end

function ConvertGMSL(scn,settings)
    # -------------------------------------------
    # This reads the Sep 14 data
    # GMSL is in different file
    # Read the truncated file with GMSL values
    # Save as netcdf
    # -------------------------------------------
    println("  Processing GMSL...")
    scn_gmsl   = Dict()
    scenario_target = [30,50,100,150,200]

    # Read file info
    fn = settings["dir_NCA5_raw"] * "gmsl/gmsl"*lpad(scenario_target[scn],3,"0")*"/AIS_gmsl"*lpad(scenario_target[scn],3,"0")*".nc"
    quantiles_raw = ncread(fn,"quantiles")
    idx_low = findfirst(quantiles_raw.>0.1699)
    idx_med = findfirst(quantiles_raw.>0.4999)
    idx_high = findfirst(quantiles_raw.>0.8299)
    scn_gmsl["quantiles"] = quantiles_raw[[idx_low,idx_med,idx_high]]
    scn_gmsl["years"] = ncread(fn,"years")

    # Read individual fields
    for process ∈ settings["processes"]
        if process != "verticallandmotion"
            println("   Reading "*process)
            fn = settings["dir_NCA5_raw"] * "gmsl/gmsl"*lpad(scenario_target[scn],3,"0")*"/"*process*"_gmsl"*lpad(scenario_target[scn],3,"0")*".nc"
            if (process == "landwaterstorage")
                scn_gmsl[process] = (ncread(fn,"sea_level_change",start=[1,2,1],count=[1,-1,-1])[:,:,[idx_low,idx_med,idx_high]])[1,:,:];
            else
                scn_gmsl[process]  = (ncread(fn,"sea_level_change",start=[1,1,1],count=[1,-1,-1])[:,:,[idx_low,idx_med,idx_high]])[1,:,:];
            end
        end
    end

    fn_gmsl_out = settings["dir_NCA5"] * "NCA5_"*settings["NCA5_scenarios"][scn]*"_gmsl.nc"
    fh = Dataset(fn_gmsl_out,"c")
    defDim(fh,"years", length(scn_gmsl["years"]))
    defDim(fh,"percentiles",3)
    defVar(fh,"years",trunc.(Int16,scn_gmsl["years"]),("years",),deflatelevel=6)
    defVar(fh,"percentiles",trunc.(Int16,[17,50,83]),("percentiles",),deflatelevel=6)
    for process ∈ settings["processes"]
        if process != "verticallandmotion"
            defVar(fh,process,trunc.(Int16,scn_gmsl[process]),("years","percentiles",),deflatelevel=6)
        end
    end
    close(fh)
    println("  Processing GMSL done")
end

# function ConvertGMSL(settings)
#     # -------------------------------------------
#     # GMSL is in different file
#     # Read the truncated file with GMSL values
#     # Save as netcdf
#     # Currently, I don't have the GMSL components
#     # -------------------------------------------
#     println("  Processing GMSL...")
#     fn_in = settings["dir_NCA5_raw"]*"NCA5_GMSL.csv"
#     data_raw = readdlm(fn_in,',',skipstart=1)
#     traw = [2020, 2030, 2040, 2050, 2060, 2070, 2080, 2090, 2100, 2110, 2120,2130,2140, 2150, 2200, 2250, 2300]
#     gmsl = Dict()
#     gmsl["years"] = traw
#     for (idx,scn) ∈ enumerate(settings["NCA5_scenarios"])
#         gmsl[scn] = 10 .* convert.(Int,data_raw[3*(idx-1)+1:3*idx,7:end]')
#         @. gmsl[scn] = gmsl[scn][:,[2,1,3]]
#     end

#     # Save as netcdf
#     for (idx,scn) ∈ enumerate(settings["NCA5_scenarios"])
#         fn_gmsl_out = settings["dir_NCA5"] * "NCA5_"*scn*"_gmsl.nc"
#         fh = Dataset(fn_gmsl_out,"c")
#         defDim(fh,"years", length(gmsl["years"]))
#         defDim(fh,"percentiles",3)
#         defVar(fh,"years",trunc.(Int16,gmsl["years"]),("years",),deflatelevel=6)
#         defVar(fh,"percentiles",trunc.(Int16,[17,50,83]),("percentiles",),deflatelevel=6)
#         defVar(fh,"total",trunc.(Int16,gmsl[scn]),("years","percentiles",),deflatelevel=6)
#         close(fh)
#     end
#     println("  Processing GMSL done")
# end

function ConvertRegional(scn,settings)
    # -------------------------------------------
    # Read all processes for individual scenarios
    # Save lat/lon grid and tide-gauge values in
    # individual files
    # -------------------------------------------
    println("  Processing regional sea level...")
    scn_grid = Dict()
    scn_tg   = Dict()
    scenario_target = [30,50,100,150,200]

    # Read file info
    fn = settings["dir_NCA5_raw"] * "regional/gmsl"*lpad(scenario_target[1],3,"0")*"/AIS_gmsl"*lpad(scenario_target[1],3,"0")*".nc"
    quantiles_raw = ncread(fn,"quantiles")
    idx_low = findfirst(quantiles_raw.>0.1699)
    idx_med = findfirst(quantiles_raw.>0.4999)
    idx_high = findfirst(quantiles_raw.>0.8299)
    scn_grid["quantiles"] = quantiles_raw[[idx_low,idx_med,idx_high]]
    scn_tg["quantiles"] = quantiles_raw[[idx_low,idx_med,idx_high]]
    scn_grid["years"] = ncread(fn,"years")
    scn_tg["years"] = ncread(fn,"years")
    acc_loc_grid = ncread(fn,"locations") .> 4000
    acc_loc_tg = ncread(fn,"locations") .< 4000
    latmat = reshape(ncread(fn,"lat")[acc_loc_grid],(360,181))
    lonmat = reshape(ncread(fn,"lon")[acc_loc_grid],(360,181))
    @. lonmat = mod(lonmat,360)
    scn_grid["lon"] = lonmat[:,1]
    scn_grid["lat"] = reverse(latmat[1,:])

    scn_tg["loc"] = ncread(fn,"locations")[acc_loc_tg]
    scn_tg["lon"] = mod.(ncread(fn,"lon")[acc_loc_tg],360)
    scn_tg["lat"] = ncread(fn,"lat")[acc_loc_tg]

    # Read individual fields
    for process ∈ settings["processes"]
        println("   Reading "*process)
        fn = settings["dir_NCA5_raw"] * "regional/gmsl"*lpad(scenario_target[scn],3,"0")*"/"*process*"_gmsl"*lpad(scenario_target[scn],3,"0")*".nc"
        if (process == "landwaterstorage") || (process == "verticallandmotion")
            scn_grid[process] = reshape(ncread(fn,"sea_level_change",start=[findfirst(acc_loc_grid),2,1],count=[-1,-1,-1])[:,:,[idx_low,idx_med,idx_high]],(360,181,length(scn_grid["years"]),3));
            scn_tg[process] = ncread(fn,"sea_level_change",start=[1,2,1],count=[sum(acc_loc_tg),-1,-1])[:,:,[idx_low,idx_med,idx_high]];
        else
            scn_grid[process] = reshape(ncread(fn,"sea_level_change",start=[findfirst(acc_loc_grid),1,1],count=[-1,-1,-1])[:,:,[idx_low,idx_med,idx_high]],(360,181,length(scn_grid["years"]),3));
            scn_tg[process]  = ncread(fn,"sea_level_change",start=[1,1,1],count=[sum(acc_loc_tg),-1,-1])[:,:,[idx_low,idx_med,idx_high]];
        end
        scn_grid[process] = convert.(Float32,scn_grid[process]);
        scn_tg[process] = convert.(Float32,scn_tg[process]);
        reverse!(scn_grid[process],dims=2);
    end

    # Estimate climate-driven sea-level changes (ice, sterodynamic, land water storage, but not VLM)
    # Assume errors are independent, so they're added in quadrature
    println("   Summing climate-related fields...")

    scn_grid["total_climate"] = zeros(Float32,size(scn_grid["total"]))
    scn_tg["total_climate"] = zeros(Float32,size(scn_tg["total"]))
    for process ∈ settings["processes_clim"]
        @. scn_grid["total_climate"][:,:,:,2] += scn_grid[process][:,:,:,2]
        @. scn_grid["total_climate"][:,:,:,1] += (abs((scn_grid[process][:,:,:,1] -  scn_grid[process][:,:,:,2]))^2)
        @. scn_grid["total_climate"][:,:,:,3] += (abs((scn_grid[process][:,:,:,3] -  scn_grid[process][:,:,:,2]))^2)
        @. scn_tg["total_climate"][:,:,2] += scn_tg[process][:,:,2]
        @. scn_tg["total_climate"][:,:,1] += (abs((scn_tg[process][:,:,1] -  scn_tg[process][:,:,2]))^2)
        @. scn_tg["total_climate"][:,:,3] += (abs((scn_tg[process][:,:,3] -  scn_tg[process][:,:,2]))^2)
    end
    @. scn_grid["total_climate"][:,:,:,1] = scn_grid["total_climate"][:,:,:,2] - sqrt(@. scn_grid["total_climate"][:,:,:,1])
    @. scn_grid["total_climate"][:,:,:,3] = scn_grid["total_climate"][:,:,:,2] + sqrt(@. scn_grid["total_climate"][:,:,:,3])
    @. scn_tg["total_climate"][:,:,1] = scn_tg["total_climate"][:,:,2] - sqrt(@. scn_tg["total_climate"][:,:,1])
    @. scn_tg["total_climate"][:,:,3] = scn_tg["total_climate"][:,:,2] + sqrt(@. scn_tg["total_climate"][:,:,3])

    # Fix NaNs
    @. scn_grid["total_climate"][scn_grid["total_climate"]<-32000] = -32000
    @. scn_grid["total_climate"][scn_grid["total_climate"]>32000] = -32000
    @. scn_tg["total_climate"][scn_tg["total_climate"]<-32000] = -32000
    @. scn_tg["total_climate"][scn_tg["total_climate"]>32000] = -32000
    for process ∈ settings["processes"]
        @. scn_grid[process][scn_grid[process]<-32000] = -32000
        @. scn_grid[process][scn_grid[process]>32000] = -32000
        @. scn_tg[process][scn_tg[process]<-32000] = -32000
        @. scn_tg[process][scn_tg[process]>32000] = -32000
    end

    # Save scenario file
    println("   Saving")
    fn_grid_out = settings["dir_NCA5"] * "NCA5_"*settings["NCA5_scenarios"][scn]*"_grid.nc"
    fn_tg_out   = settings["dir_NCA5"] * "NCA5_"*settings["NCA5_scenarios"][scn]*"_tg.nc"

    fh = Dataset(fn_grid_out,"c")
    defDim(fh,"years", length(scn_grid["years"]))
    defDim(fh,"percentiles",3)
    defDim(fh,"lon", length(scn_grid["lon"]))
    defDim(fh,"lat", length(scn_grid["lat"]))
    defVar(fh,"lon",scn_grid["lon"],("lon",),deflatelevel=6)
    defVar(fh,"lat",scn_grid["lat"],("lat",),deflatelevel=6)
    defVar(fh,"years",trunc.(Int16,scn_grid["years"]),("years",),deflatelevel=6)
    defVar(fh,"percentiles",trunc.(Int16,[17,50,83]),("percentiles",),deflatelevel=6)
    for process ∈ settings["processes"]
        defVar(fh,process,trunc.(Int16,scn_grid[process]),("lon","lat","years","percentiles",),deflatelevel=6)
    end
    defVar(fh,"total_climate",trunc.(Int16,scn_grid["total_climate"]),("lon","lat","years","percentiles",),deflatelevel=6)
    close(fh)

    fh = Dataset(fn_tg_out,"c")
    defDim(fh,"years", length(scn_tg["years"]))
    defDim(fh,"percentiles",3)
    defDim(fh,"loc", length(scn_tg["loc"]))
    defVar(fh,"loc",scn_tg["loc"],("loc",),deflatelevel=6)
    defVar(fh,"lon",scn_tg["lon"],("loc",),deflatelevel=6)
    defVar(fh,"lat",scn_tg["lat"],("loc",),deflatelevel=6)
    defVar(fh,"years",trunc.(Int16,scn_tg["years"]),("years",),deflatelevel=6)
    defVar(fh,"percentiles",trunc.(Int16,[17,50,83]),("percentiles",),deflatelevel=6)
    for process ∈ settings["processes"]
        defVar(fh,process,trunc.(Int16,scn_tg[process]),("loc","years","percentiles",),deflatelevel=6)
    end
    defVar(fh,"total_climate",trunc.(Int16,scn_tg["total_climate"]),("loc","years","percentiles",),deflatelevel=6)
    close(fh)
    println("  Processing regional sea level done")
    return nothing
end

end


module Xyce
using CxxWrap
using XyceWrapper_jll
@wrapmodule(XyceWrapper_jll.xycelib)

function __init__()
    @initcxx
end

function simulate(netlist; type=TRAN, name="handler", vectors=["TIME", "V(*)", "I(*)"])
    cxxvec = StdVector(StdString.(vectors))
    oh = OutputHandler(name, type, cxxvec)
    x = GenCouplingSimulator()
    # this avoids aborting when Xyce encounters an error
    set_report_handler()
    arg = ["xyce", netlist]
    argch = Base.cconvert.(CxxPtr{CxxChar}, arg)
    argv = Base.cconvert(CxxPtr{CxxPtr{CxxChar}}, argch)
    initializeEarly(x, 2, argv)
    addOutputInterface(x, CxxPtr(oh))
    initializeLate(x)
    runSimulation(x)
    names = String.(getFieldnames(oh))
    res = Dict()
    for (i, n) in enumerate(names)
        res[n] = Array(getRealData(oh, i-1))
    end
    return res
end

end # module


include("load_train_test_split.jl")

using AutoDrivers
using AutoDrivers.GaussianMixtureRegressionDrivers

include("../envs/Auto2D.jl")
include("load_policy.jl")
include("additional_metrics.jl")

const METRIC_SAVE_FILE_DIR = dirname(@__FILE__) * "/results/"

const N_SEGMENTS = 1000
const N_SIMULATIONS_PER_TRACE = 20
const EVAL_PRIME_DURATION = 0.0 # [s] - length of priming period
const EVAL_PRIME_STEPS = 0
const EVAL_DURATION = 10.0 # [s] - length of trace
const EVAL_DURATION_STEPS = 100
const CONTEXT = IntegratedContinuous(NGSIM_TIMESTEP, 3)
const METRICS = TraceMetricExtractor[
    RootWeightedSquareError(SPEED,  0.5),
    RootWeightedSquareError(SPEED,  1.0),
    RootWeightedSquareError(SPEED,  2.0),
    RootWeightedSquareError(SPEED,  3.0),
    RootWeightedSquareError(SPEED,  4.0),
    RootWeightedSquareError(SPEED,  5.0),

    RootWeightedSquareError(POSFT,  0.5),
    RootWeightedSquareError(POSFT,  1.0),
    RootWeightedSquareError(POSFT,  2.0),
    RootWeightedSquareError(POSFT,  3.0),
    RootWeightedSquareError(POSFT,  4.0),
    RootWeightedSquareError(POSFT,  5.0),

    RWSEPosG(0.5),
    RWSEPosG(1.0),
    RWSEPosG(2.0),
    RWSEPosG(3.0),
    RWSEPosG(4.0),
    RWSEPosG(5.0),

    EmergentKLDivergence(INV_TTC, 0., 10., 100),
    EmergentKLDivergence(SPEED, -5., 50., 100),
    EmergentKLDivergence(ACC, -10., 10., 100),
    EmergentKLDivergence(TURNRATEF, -2., 2., 100),
    EmergentKLDivergence(JERK, -100., 100., 100),

    SumSquareJerk(),
    CollisionRate(),
    OffRoadRate(),
    LaneChangeRate(),
    HardBrakeRate(),
]

const EXTRACT_CORE = true
const EXTRACT_TEMPORAL = false
const EXTRACT_WELL_BEHAVED = true
const EXTRACT_NEIGHBOR_FEATURES = false
const EXTRACT_CARLIDAR_RANGERATE = true
const CARLIDAR_NBEAMS = 20
const ROADLIDAR_NBEAMS = 0
const ROADLIDAR_NLANES = 2
const CARLIDAR_MAX_RANGE = 100.0
const ROADLIDAR_MAX_RANGE = 100.0

evaldata = load_evaldata()
assignment = load_assignment()

function create_evaldata(evaldata::EvaluationData, foldset::FoldSet; nsegs::Int=1, nframes::Int=101)
    segments = Array(TrajdataSegment, nsegs)
    indeces = collect(foldset)
    i = 0
    # seg_exclude = [68] # Array of trajectories to exclude
    while i < nsegs
        seg = evaldata.segments[indeces[rand(1:length(indeces))]]

        # Make sure segment is long enough and not in first or last 500 frames
        frame_lo = max(seg.frame_lo, 500)
        frame_hi = min(seg.frame_hi, AutoCore.nframes(evaldata.trajdatas[seg.trajdata_index]) - 1000)
        if frame_hi - frame_lo + 1 > nframes #&& !(seg.trajdata_index in seg_exclude))
            dom_hi = (frame_hi - frame_lo + 1) - nframes
            domain = 1:dom_hi

            frame_lo += rand(domain)
            frame_hi = frame_lo + nframes - 1

            segments[i+=1] = TrajdataSegment(seg.trajdata_index, seg.egoid, frame_lo, frame_hi)
        end
    end

    EvaluationData(evaldata.trajdatas, segments)
end

function create_simparams(evaldata::EvaluationData)
    # Construct extractor
    extractor = Auto2D.MultiFeatureExtractor(
        EXTRACT_CORE,
        EXTRACT_TEMPORAL,
        EXTRACT_WELL_BEHAVED,
        EXTRACT_NEIGHBOR_FEATURES,
        EXTRACT_CARLIDAR_RANGERATE,
        CARLIDAR_NBEAMS,
        ROADLIDAR_NBEAMS,
        ROADLIDAR_NLANES,
        carlidar_max_range=CARLIDAR_MAX_RANGE,
        roadlidar_max_range=ROADLIDAR_MAX_RANGE,
        )

    # Convert array of trajdatas to dict
    trajdatas = Dict(zip(collect(1:length(evaldata.trajdatas)), evaldata.trajdatas))

    # Construct and return simparams
    Auto2D.SimParams(trajdatas, evaldata.segments, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
        false, true, false, -2.0, 1, EVAL_PRIME_STEPS, EVAL_DURATION_STEPS, AccelTurnrate, extractor)
end

srand(0)
eval_seg_nframes = ceil(Int, (EVAL_PRIME_DURATION + EVAL_DURATION)/NGSIM_TIMESTEP) + 1
VALDATA_SUBSET = create_evaldata(evaldata, foldset_match(assignment, FOLD_TEST), nsegs=N_SEGMENTS, nframes=eval_seg_nframes)
FOLDSET_TEST = foldset_match(fill(1, N_SEGMENTS), 1)
SIMPARAMS = create_simparams(VALDATA_SUBSET)

function load_models(; context::IntegratedContinuous = CONTEXT)
    models = Dict{AbstractString, DriverModel}()

    models["SG"] = StaticGaussianDriver{AccelTurnrate}(context, MvNormal([0.07813232200000027,0.0025751835870002756], [[0.533053, 0.000284046] [0.000284046, 0.000348645]]))

    mlon = IntelligentDriverModel(
        σ=0.1,
        k_spd=1.0,
        T=0.5,
        s_min=2.0,
        a_max=3.0,
        d_cmf=2.5,
        )
    mlat = ProportionalLaneTracker(σ=0.1, kp=3.0, kd=2.0)
    mlane = MOBIL(context,
                  politeness=0.1,
                  advantage_threshold=0.01,
                 )

    models["controller"] = Tim2DDriver(context, mlon=mlon, mlat=mlat, mlane=mlane, rec=SceneRecord(3, context.Δt))

    include(joinpath(ROOT_FILEPATH, "julia/pull_traces", "multifeatureset.jl"))
    extractor = MultiFeatureExtractor(EXTRACT_CORE, EXTRACT_TEMPORAL,
                                    EXTRACT_WELL_BEHAVED, EXTRACT_NEIGHBOR_FEATURES,
                                    EXTRACT_CARLIDAR_RANGERATE, CARLIDAR_NBEAMS,
                                    ROADLIDAR_NBEAMS, ROADLIDAR_NLANES)
    models["GMR"] = open(io->read(io, GaussianMixtureRegressionDriver, extractor), "GMR.txt", "r")

    filepath = joinpath(ROOT_FILEPATH, "julia", "validation",  "models", "gail_gru.h5")
    iteration = 413
    models["gail_gru"] = Auto2D.load_gru_driver(filepath, iteration)

    filepath = "./models/gail_mlp.h5"
    iteration = 447
    models["gail_mlp"] = load_gru_driver(filepath, iteration, gru_layer=false)

    filepath = "./models/bc_gru.h5"
    iteration = -1
    models["bc_gru"] = Auto2D.load_gru_driver(filepath, iteration, bc_policy=true)

    filepath = "./models/bc_mlp.h5"
    iteration = -1
    models["bc_mlp"] = load_gru_driver(filepath, iteration, gru_layer=false, bc_policy=true)

    models
end

function validate(model::DriverModel;
    simparams::Auto2D.SimParams = SIMPARAMS,
    metrics::Vector{TraceMetricExtractor} = METRICS,
    foldset::FoldSet = FOLDSET_TEST,
    n_simulations_per_trace::Int = N_SIMULATIONS_PER_TRACE,
    save::Bool=false,
    modelname::AbstractString=AutomotiveDrivingModels.get_name(model)
    )

    metrics_df = allocate_metrics_dataframe(METRICS, 1)
    calc_metrics!(metrics_df, model, metrics, simparams, foldset,
                        n_simulations_per_trace=n_simulations_per_trace,
                        row = 1, prime_history=EVAL_PRIME_STEPS)

    if save
        filename = "valid_"*modelname*".csv"
        writetable(joinpath(METRIC_SAVE_FILE_DIR, filename), metrics_df)
    end

    metrics_df
end

print("DONE!")


module DnDstats
using DataFrames

export compare

weaponlist = DataFrame(
name = ["Club", "Dagger", "Great-Club", "Hand-Axe", "Javelin", "Light Hammer", "Mace", "Quarterstaff", "Sickle", "Spear", "Crossbow, light", "Dart", "Shortbow", "Sling", "Battleaxe", "Flail", "Glaive", "Greataxe", "Great-Sword", "Halberd", "Lance", "Long-sword", "Maul", "Morning-star", "Pike", "Rapier", "Scimitar", "Short-sword", "Trident", "War pick", "War-hammer", "Whip", "Blowgun", "Crossbow, hand", "Crossbow, heavy", "Longbow", "Net"],
cost = [0.1, 2, 0.2, 5, 0.5, 2, 5, 0.2, 1, 1, 25, 0.05, 25, 0.1, 10, 10, 20, 30, 50, 20, 10, 15, 10, 15, 5, 25, 25, 10, 5, 5, 15, 2, 10, 75, 50, 50, 1],
damagedicenumber = [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0],
damagedicetype = [4, 4, 8, 6, 6, 4, 6, 6, 4, 6, 8, 4, 6, 4, 8, 8, 10, 12, 6, 10, 12, 8, 6, 8, 10, 8, 6, 6, 6, 8, 8, 4, 1, 6, 10, 8, 0],
damagetype = ["Bludgeoning", "Piercing", "Bludgeoning", "Slashing", "Piercing", "Bludgeoning", "Bludgeoning", "Bludgeoning", "Slashing", "Piercing", "Piercing", "Piercing", "Piercing", "Piercing", "Slashing", "Bludgeoning", "Slashing", "Slashing", "Slashing", "Slashing", "Piercing", "Slashing", "Bludgeoning", "Piercing", "Piercing", "Piercing", "Slashing", "Piercing", "Piercing", "Piercing", "Bludgeoning", "Slashing", "Piercing", "Piercing", "Piercing", "Piercing", "None"],
weight = [2, 1, 10, 2, 2, 2, 4, 4, 2, 3, 5, .25, 2, 0, 4, 2, 6, 7, 6, 6, 6, 3, 10, 4, 18, 2, 3, 2, 4, 2, 2, 3, 1, 3, 18, 2, 3],
complexity = ["simple", "simple", "simple", "simple", "simple", "simple", "simple", "simple","simple", "simple", "simple", "simple", "simple", "simple", "martial", "martial", "martial", "martial", "martial", "martial", "martial", "martial", "martial", "martial", "martial", "martial", "martial", "martial", "martial", "martial", "martial", "martial", "martial", "martial", "martial", "martial", "martial"],
range = ["melee", "melee", "melee", "melee", "melee", "melee", "melee", "melee", "melee", "melee", "ranged", "ranged", "ranged", "ranged", "melee", "melee", "melee", "melee", "melee", "melee", "melee", "melee", "melee", "melee", "melee", "melee", "melee", "melee", "melee", "melee", "melee", "melee", "ranged", "ranged", "ranged", "ranged", "ranged"],
shortrange = [5, 20, 5, 20, 30, 20, 5, 5, 5, 20, 80, 20, 80, 30, 5, 5, 10, 5, 5, 10, 10, 5, 5, 5, 10, 5, 5, 5, 20, 5, 5, 10, 25, 30, 100, 150, 5],
longrange = [5, 60, 5, 60, 120, 60, 5, 5, 5, 60, 320, 60, 320, 120, 5, 5, 10, 5, 5, 10, 10, 5, 5, 5, 10, 5, 5, 5, 60, 5, 5, 10, 100, 120, 400, 600, 15],
ammunition = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0],
finesse = [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0],
heavy = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0],
light = [1, 1, 0, 1, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0],
loading = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0],
reach = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0],
special = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1],
thrown = [0, 1, 0, 1, 1, 1, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1],
twohanded = [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 1, 1, 1, 1, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0],
versatile = [0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0]
), false

function chancetohit(attackbonus,ac)
    if attackbonus == "auto"
        return 1
    end
    hits = 1
    for roll in 2:19
        if roll + attackbonus >= ac
            hits = hits + 1
        end
    end
    return hits/20
end

function damage(dice,type,modifier)
    dicetypes = length(dice)
    dicedamage = 0
    for x in 1:dicetypes
        dicedamage = dicedamage + (((type[x]+1)/2)*dice[x])
    end
    return dicedamage + modifier
end

function avgdamage(attackbonus,ac,dice,type,modifier, crit = true)
    percenthit = chancetohit(attackbonus,ac)
    damageperhit = damage(dice,type,modifier)
    if crit == true
        crit = damage(2*dice,type,modifier)
        return damageperhit*(percenthit-0.05)+(crit*0.05)
    else
        return damageperhit*percenthit
    end
end

function damageperattack(weapon1, acrange = 8:20)
    totaldamage1 = 0
    for ac in acrange
        damage1 = avgdamage(weapon1[2], ac, weapon1[3], weapon1[4], weapon1[5], weapon1[6])
    totaldamage1 = totaldamage1 + damage1
    end
    avgdamage1 = totaldamage1/length(acrange)
    return avgdamage1
end

function compare(weapon1, weapon2, weapon1attacks, weapon2attacks, acrange = 8:20) #each weapon should be a length 6 vector [name, attackbonus, dice, type, modifier, crit]
    name1 = weapon1[1]
    name2 = weapon2[1]
    avgdamage1 = damageperattack(weapon1, acrange)*weapon1attacks
    avgdamage2 = damageperattack(weapon2, acrange)*weapon2attacks
    if avgdamage1 > avgdamage2
        println("The $name1 is more powerful than the $name2.")
        println("The $name1 deals $avgdamage1 and the $name2 deals $avgdamage2")
        return weapon1[1]
    elseif avgdamage1 < avgdamage2
        println("The $name2 is more powerful than the $name1.")
        println("The $name2 deals $avgdamage2 and the $name1 deals $avgdamage1")
        return weapon2[1]
    else
        println("The $name1 does as much damage as the $name2.")
        println("The $name1 deals $avgdamage1 and the $name2 deals $avgdamage2")
        return "Neither"
    end
end

end


opr_pm = [+|-]
opr_dm = [/|*]

oprlr_eq_dif = [==|!=]
oprlr_lgt = [<|>]
oprlr_lgt_eq = [<=|>=]

comma = ","
o_bracket = ["["]
c_bracket = ["]"]

oprl_algebc = {oprlr_eq_dif} | {oprlr_lgt} | {oprlr_lgt_eq}
opr_algebc = opr_pm | opr_dm

oprl_and_or = [:blank:]+"and"[:blank:]+ | [:blank:]+"or"[:blank:]+
opr_not = "not"[:blank:]*

oprlr = {oprlr_eq_dif} | {oprlr_lgt} | {oprlr_lgt_eq} | {orplr_and_or}

opr_concat = "++"+
ct_str = "\"".*"\""?[{opr_concat}{ct_string}?{ct_str}]

ct_int = ^{opr_pm}?[0-9]+$
ct_float = {ct_int}[.][0-9]*$
ctn = {ct_int}|{ct_float}
id = ^[:alpha:]+[:alnum:]*

loop_while = "while" [:blank:]+"("{expr_bool}")"[:blank:]+ "{"{cmd}"}"
loop_for = "for"[:blank:]+"("{int_'dclr'}{comma}{ct_int}{comma}{ct_int}")"[:blank:]*"{"{cmd}"}"
ctb = "1" | "0" | "true" | "false"
expr_bool =  {expr_arit} {opr_logic} {expr_arit} | {ctb}

term = {ctn}|{id}

expr_arit =  {term} {opr_algebc} {term} | "(" {expr_arit} [{opr_algebc}{expr_arit}]*")"[{opr_algebc}{expr_arit}]*
data = {expr_arit} | {term} | {expr_bool}
int_dclr = "int" {id} "=" [0-9]*

float_type = "float"
bool_type = "bool"
char_type = "char"
string_type = "string"
void_type = "void"

data_type = {float_type} | {bool_type} | {char_type} | {string_type} | {void}

var_dclr = {int_dclr} | [{bool_type} | {float_type} | {char_type} | {string_type}] {atrib}]
atrib = {id} "=" {data}
cmd = {loop} | {atrib} | {expr_arit} | {expr_bool} | {term}

params = {id}|{id}{comma}{id}?[{comma}{params}]
fn_dclr = {data_type} {id}"("{params}")"[:blank:]*"{"{cmd}*?["return" {data}]"}"
fn_call = {id}"("?{params}")"

comment_block = "/*"[:alnum:]*"*/" | "/*"{comment_block}"*/"
coment_line = "//"[:alnum:]*
comment = comment_block | comment_line


################################################################################
#
#	INTERFACE FOR DEFAULT POINTS IN RECIPROCAL SPACE
#	(implementations see below)
#
################################################################################

# general interface
function getReciprocalPoint(
            :: Type{R},
            unitcell   :: U,
            identifier :: Symbol,
            instance   :: Int64 = 1
        ) :: R where {D,N,LS,LB, R<:AbstractReciprocalPoint{D}, S<:AbstractSite{LS,D}, B<:AbstractBond{LB,N}, U<:AbstractUnitcell{S,B}}

    # maybe instance is passed from another source in some notation as e.g. X_2
    if instance == 1 && occursin('_', string(identifier))
        return getReciprocalPoint(R, unitcell, Val(Symbol(split(string(identifier),'_')[1])), Val(Meta.eval(Meta.parse(split(string(identifier),'_')[2]))))
    end
    # return the specific val typed function
    return getReciprocalPoint(R, unitcell, Val(identifier), Val(instance))
end
function getReciprocalPoint(
            :: Type{R},
            unitcell   :: U,
            identifier :: Tuple{Symbol, Int64}
        ) :: R where {D,N,LS,LB, R<:AbstractReciprocalPoint{D}, S<:AbstractSite{LS,D}, B<:AbstractBond{LB,N}, U<:AbstractUnitcell{S,B}}

    # return the specific val typed function
    return getReciprocalPoint(R, unitcell, Val(identifier[1]), Val(identifier[2]))
end

# interface for own concrete reciprocal point type
function getReciprocalPoint(
            unitcell   :: U,
            identifier :: Symbol,
            instance   :: Int64 = 1
        ) :: ReciprocalPoint{D} where {D,N,LS,LB, S<:AbstractSite{LS,D}, B<:AbstractBond{LB,N}, U<:AbstractUnitcell{S,B}}

    # maybe instance is passed from another source in some notation as e.g. X_2
    if instance == 1 && occursin('_', string(identifier))
        return getReciprocalPoint(ReciprocalPoint{D}, unitcell, Val(Symbol(split(string(identifier),'_')[1])), Val(Meta.eval(Meta.parse(split(string(identifier),'_')[2]))))
    end
    # return the specific val typed function
    return getReciprocalPoint(ReciprocalPoint{D}, unitcell, Val(identifier), Val(instance))
end

# Fallback for all functions
function getReciprocalPoint(
            :: Type{R},
            unitcell   :: U,
            identifier :: Val{I1},
            instance   :: Val{I2}
        ) :: R where {I1,I2,D,N,LS,LB, R<:AbstractReciprocalPoint{D}, S<:AbstractSite{LS,D}, B<:AbstractBond{LB,N},U<:AbstractUnitcell{S,B}}

    # fallback / fail due to missing implementation
    error("Implementation of reciprocal point " * string(I1) * " (instance " * string(I2) * ") not implemented yet " *
    "for concrete reciprocal point type " * string(R) * " and unitcells of dimension D=" * string(D) * " and N="*string(N))
end

# export the general interface function
export getReciprocalPoint

################################################################################
#
#	IMPLEMENTATIONS OF DEFAULT POINTS IN RECIPROCAL SPACE
#
################################################################################

#########################
#
#	GAMMA POINT
#
#########################

# Gamma point in arbitrary dimension (Fallback)
function getReciprocalPoint(
            :: Type{R},
            unitcell   :: U,
            identifier :: Val{:Gamma},
            instance   :: Val{I}
        ) :: R where {I,D,N,LS,LB, R<:AbstractReciprocalPoint{D}, S<:AbstractSite{LS,D}, B<:AbstractBond{LB,N},U<:AbstractUnitcell{S,B}}

    # fallback / fail due to missing implementation
    error("Implementation of Gamma point in " * string(D) * " spatial dimensions missing " *
    "for concrete reciprocal point type " * string(R))
end

# Gamma point in 2D
function getReciprocalPoint(
            :: Type{R},
            unitcell   :: U,
            identifier :: Val{:Gamma},
            instance   :: Val{I}
        ) :: R where {I,N,LS,LB, R<:AbstractReciprocalPoint{2}, S<:AbstractSite{LS,2}, B<:AbstractBond{LB,N},U<:AbstractUnitcell{S,B}}

    # return the specific type
    return newReciprocalPoint(
        R,
        [0.0, 0.0],
        "Gamma",
        "\\Gamma"
    )
end

# Gamma point in 3D
function getReciprocalPoint(
            :: Type{R},
            unitcell   :: U,
            identifier :: Val{:Gamma},
            instance   :: Val{I}
        ) :: R where {I,N,LS,LB, R<:AbstractReciprocalPoint{3}, S<:AbstractSite{LS,3}, B<:AbstractBond{LB,N},U<:AbstractUnitcell{S,B}}

    # return the specific type
    return newReciprocalPoint(
        R,
        [0.0, 0.0, 0.0],
        "Gamma",
        "\\Gamma"
    )
end

#########################
#
#	K POINT (and K')
#
#########################

# K point (Based on BZ object) in 2D
function getReciprocalPoint(
            :: Type{R},
            unitcell   :: U,
            identifier :: Val{:K},
            instance   :: Val{I}
        ) :: R where {D,I,N,LS,LB, R<:AbstractReciprocalPoint{2}, S<:AbstractSite{LS,2}, B<:AbstractBond{LB,N},U<:AbstractUnitcell{S,B}}

    # get the bz
    bz = getBrillouinZone(getReciprocalUnitcell(unitcell))

    # return the specific type
    return newReciprocalPoint(
        R,
        corners(bz)[faces(bz)[1][mod(I,length(corners(bz))) + 1]],
        "K",
        "K"
    )
end

# K' point (Based on BZ object) in 2D
function getReciprocalPoint(
            :: Type{R},
            unitcell   :: U,
            identifier :: Val{:Kp},
            instance   :: Val{I}
        ) :: R where {D,I,N,LS,LB, R<:AbstractReciprocalPoint{2}, S<:AbstractSite{LS,2}, B<:AbstractBond{LB,N},U<:AbstractUnitcell{S,B}}

    # get the bz
    bz = getBrillouinZone(getReciprocalUnitcell(unitcell))

    # return the specific type
    return newReciprocalPoint(
        R,
        corners(bz)[faces(bz)[1][mod(I+1,length(corners(bz))) + 1]],
        "K\'",
        "K\'"
    )
end

# K point (Based on BZ object) in 3D
function getReciprocalPoint(
            :: Type{R},
            unitcell   :: U,
            identifier :: Val{:K},
            instance   :: Val{I}
        ) :: R where {D,I,N,LS,LB, R<:AbstractReciprocalPoint{3}, S<:AbstractSite{LS,3}, B<:AbstractBond{LB,N},U<:AbstractUnitcell{S,B}}

    # get the bz
    bz = getBrillouinZone(getReciprocalUnitcell(unitcell))

    # get the faceindex
    faceindex = mod(div(I, 10), length(faces(bz))) + 1
    face = faces(bz)[faceindex]
    # first and second index
    cornerindex = mod(I, 10)
    i = face[mod(cornerindex, length(face))+1]

    # return the specific type
    return newReciprocalPoint(
        R,
        corners(bz)[i],
        "K",
        "K"
    )
end

# K' point (Based on BZ object) in 3D
function getReciprocalPoint(
            :: Type{R},
            unitcell   :: U,
            identifier :: Val{:Kp},
            instance   :: Val{I}
        ) :: R where {D,I,N,LS,LB, R<:AbstractReciprocalPoint{3}, S<:AbstractSite{LS,3}, B<:AbstractBond{LB,N},U<:AbstractUnitcell{S,B}}

    # get the bz
    bz = getBrillouinZone(getReciprocalUnitcell(unitcell))

    # get the faceindex
    faceindex = mod(div(I, 10), length(faces(bz))) + 1
    face = faces(bz)[faceindex]
    # first and second index
    cornerindex = mod(I, 10)
    i = face[mod(cornerindex+1, length(face))+1]

    # return the specific type
    return newReciprocalPoint(
        R,
        corners(bz)[i],
        "K\'",
        "K\'"
    )
end

#########################
#
#	M POINT (and M')
#
#########################

# M point (Based on BZ object) in 2D
function getReciprocalPoint(
            :: Type{R},
            unitcell   :: U,
            identifier :: Val{:M},
            instance   :: Val{I}
        ) :: R where {D,I,N,LS,LB, R<:AbstractReciprocalPoint{2}, S<:AbstractSite{LS,2}, B<:AbstractBond{LB,N},U<:AbstractUnitcell{S,B}}

    # get the bz
    bz = getBrillouinZone(getReciprocalUnitcell(unitcell))

    # first and second index
    i1 = faces(bz)[1][mod(I-1,length(faces(bz)[1])) + 1]
    i2 = faces(bz)[1][mod(I,length(faces(bz)[1])) + 1]
    # find the first and next corner
    c1 = corners(bz)[i1]
    c2 = corners(bz)[i2]

    # return the specific type
    return newReciprocalPoint(
        R,
        (c1.+c2)./2,
        "M",
        "M"
    )
end

# M' point (Based on BZ object) in 2D
function getReciprocalPoint(
            :: Type{R},
            unitcell   :: U,
            identifier :: Val{:Mp},
            instance   :: Val{I}
        ) :: R where {D,I,N,LS,LB, R<:AbstractReciprocalPoint{2}, S<:AbstractSite{LS,2}, B<:AbstractBond{LB,N},U<:AbstractUnitcell{S,B}}

    # get the bz
    bz = getBrillouinZone(getReciprocalUnitcell(unitcell))

    # first and second index
    i1 = faces(bz)[1][mod(I,length(faces(bz)[1])) + 1]
    i2 = faces(bz)[1][mod(I+1,length(faces(bz)[1])) + 1]
    # find the first and next corner
    c1 = corners(bz)[i1]
    c2 = corners(bz)[i2]

    # return the specific type
    return newReciprocalPoint(
        R,
        (c1.+c2)./2,
        "M\'",
        "M\'"
    )
end

# M point (Based on BZ object) in 3D
function getReciprocalPoint(
            :: Type{R},
            unitcell   :: U,
            identifier :: Val{:M},
            instance   :: Val{I}
        ) :: R where {D,I,N,LS,LB, R<:AbstractReciprocalPoint{3}, S<:AbstractSite{LS,3}, B<:AbstractBond{LB,N},U<:AbstractUnitcell{S,B}}

    # get the bz
    bz = getBrillouinZone(getReciprocalUnitcell(unitcell))

    # get the faceindex
    faceindex = mod(div(I, 10), length(faces(bz))) + 1
    face = faces(bz)[faceindex]
    # first and second index
    cornerindex = mod(I, 10)
    i1 = face[mod(cornerindex-1, length(face)) + 1]
    i2 = face[mod(cornerindex,   length(face)) + 1]
    # find the first and next corner
    c1 = corners(bz)[i1]
    c2 = corners(bz)[i2]

    # return the specific type
    return newReciprocalPoint(
        R,
        (c1.+c2)./2,
        "M",
        "M"
    )
end

# M' point (Based on BZ object) in 3D
function getReciprocalPoint(
            :: Type{R},
            unitcell   :: U,
            identifier :: Val{:Mp},
            instance   :: Val{I}
        ) :: R where {D,I,N,LS,LB, R<:AbstractReciprocalPoint{3}, S<:AbstractSite{LS,3}, B<:AbstractBond{LB,N},U<:AbstractUnitcell{S,B}}

    # get the bz
    bz = getBrillouinZone(getReciprocalUnitcell(unitcell))

    # get the faceindex
    faceindex = mod(div(I, 10), length(faces(bz))) + 1
    face = faces(bz)[faceindex]
    # first and second index
    cornerindex = mod(I, 10)
    i1 = face[mod(cornerindex, length(face)) + 1]
    i2 = face[mod(cornerindex+1,   length(face)) + 1]
    # find the first and next corner
    c1 = corners(bz)[i1]
    c2 = corners(bz)[i2]

    # return the specific type
    return newReciprocalPoint(
        R,
        (c1.+c2)./2,
        "M\'",
        "M\'"
    )
end

#########################
#
#	X POINT (center of face)
#
#########################

# X point (Based on BZ object) in 3D
function getReciprocalPoint(
            :: Type{R},
            unitcell   :: U,
            identifier :: Val{:X},
            instance   :: Val{I}
        ) :: R where {D,I,N,LS,LB, R<:AbstractReciprocalPoint{3}, S<:AbstractSite{LS,3}, B<:AbstractBond{LB,N},U<:AbstractUnitcell{S,B}}

    # get the bz
    bz = getBrillouinZone(getReciprocalUnitcell(unitcell))

    # get the faceindex
    faceindex = mod(I-1, length(faces(bz))) + 1
    face = faces(bz)[faceindex]
    # find the center
    center = [0.0, 0.0, 0.0]
    for c in face
        center .+= corners(bz)[c]
    end
    center ./= length(face)

    # return the specific type
    return newReciprocalPoint(
        R,
        center,
        "X",
        "X"
    )
end


"""
Tensor{StoreT,IndsT}

A plain old tensor (with order independent
interface and no assumption of labels)
"""
struct Tensor{ElT,N,StoreT<:TensorStorage,IndsT} <: AbstractArray{ElT,N}
  storage::StoreT
  inds::IndsT

  """
      Tensor{ElT,N,StoreT,IndsT}(inds, store::StorageType)

  Internal constructor for creating a Tensor from the
  storage and indices.

  The Tensor is a view of the tensor storage.

  For normal usage, use the Tensor(store::TensorStorage, inds)
  and tensor(store::TensorStorage, inds) constructors.
  """
  function Tensor{ElT,N,StoreT,IndsT}(
    ::AllowAlias, storage::TensorStorage, inds::Tuple
  ) where {ElT,N,StoreT<:TensorStorage,IndsT}
    return new{ElT,N,StoreT,IndsT}(storage, inds)
  end
end

function Tensor{ElT,N,StoreT,IndsT}(
  ::NeverAlias, storage::TensorStorage, inds
) where {ElT,N,StoreT<:TensorStorage,IndsT}
  return Tensor{ElT,N,StoreT,IndsT}(AllowAlias(), copy(storage), inds)
end

# Allow the storage and indices to be input in opposite ordering
(T::Type{<:Tensor})(as::AliasStyle, inds, storage::TensorStorage) = T(as, storage, inds)

"""
    Tensor(storage::TensorStorage, inds)

Construct a Tensor from a tensor storage and indices.
The Tensor holds a copy of the storage data.

The indices `inds` will be converted to a `Tuple`.
"""
function Tensor(as::AliasStyle, storage::TensorStorage, inds::Tuple)
  return Tensor{eltype(storage),length(inds),typeof(storage),typeof(inds)}(
    as, storage, inds
  )
end

# Automatically convert to Tuple if the indices are not a Tuple
# already (like a Vector). In the future this may be lifted
# to allow for very large tensor orders in which case Tuple
# operations may become too slow.
function Tensor(as::AliasStyle, storage::TensorStorage, inds)
  return Tensor(as, storage, Tuple(inds))
end

tensor(args...; kwargs...) = Tensor(AllowAlias(), args...; kwargs...)
Tensor(storage::TensorStorage, inds) = Tensor(NeverAlias(), storage, inds)
Tensor(inds, storage::TensorStorage) = Tensor(storage, inds)

storage(T::Tensor) = T.storage

# TODO: deprecate
store(T::Tensor) = storage(T)

data(T::Tensor) = data(storage(T))

datatype(T::Tensor) = datatype(storage(T))

indstype(::Type{<:Tensor{<:Any,<:Any,<:Any,IndsT}}) where {IndsT} = IndsT
indstype(T::Tensor) = indstype(typeof(T))

storagetype(::Type{<:Tensor{<:Any,<:Any,StoreT}}) where {StoreT} = StoreT
storagetype(T::Tensor) = storagetype(typeof(T))

# TODO: deprecate
storetype(args...) = storagetype(args...)

inds(T::Tensor) = T.inds

ind(T::Tensor, j::Integer) = inds(T)[j]

eachindex(T::Tensor) = CartesianIndices(dims(inds(T)))

eachblock(T::Tensor) = eachblock(inds(T))

eachdiagblock(T::Tensor) = eachdiagblock(inds(T))

eltype(::Tensor{ElT}) where {ElT} = ElT

strides(T::Tensor) = dim_to_strides(inds(T))

setstorage(T, nstore) = tensor(nstore, inds(T))

setinds(T, ninds) = tensor(storage(T), ninds)

#
# Generic Tensor functions
#

# The size is obtained from the indices
dims(T::Tensor) = dims(inds(T))
dim(T::Tensor) = dim(inds(T))
dim(T::Tensor, i::Int) = dim(inds(T), i)
maxdim(T::Tensor) = maxdim(inds(T))
mindim(T::Tensor) = mindim(inds(T))
diaglength(T::Tensor) = mindim(T)
size(T::Tensor) = dims(T)
size(T::Tensor, i::Int) = dim(T, i)

# Needed for passing Tensor{T,2} to BLAS/LAPACK
# TODO: maybe this should only be for DenseTensor?
function unsafe_convert(::Type{Ptr{ElT}}, T::Tensor{ElT}) where {ElT}
  return unsafe_convert(Ptr{ElT}, storage(T))
end

copy(T::Tensor) = setstorage(T, copy(storage(T)))

copyto!(R::Tensor, T::Tensor) = (copyto!(storage(R), storage(T)); R)

complex(T::Tensor) = setstorage(T, complex(storage(T)))

Base.real(T::Tensor) = setstorage(T, real(storage(T)))

Base.imag(T::Tensor) = setstorage(T, imag(storage(T)))

# Define Base.similar in terms of NDTensors.similar
Base.similar(T::Tensor, args...) = similar(T, args...)

#
# Necessary to overload since the generic fallbacks are
# slow
#

LinearAlgebra.norm(T::Tensor) = norm(storage(T))

conj(as::AliasStyle, T::Tensor) = setstorage(T, conj(as, storage(T)))
conj(T::Tensor) = conj(AllowAlias(), T)

randn!!(T::Tensor) = (randn!(T); T)
Random.randn!(T::Tensor) = (randn!(storage(T)); T)

LinearAlgebra.rmul!(T::Tensor, α::Number) = (rmul!(storage(T), α); T)
scale!(T::Tensor, α::Number) = rmul!(storage(T), α)

fill!!(T::Tensor, α::Number) = fill!(T, α)
fill!(T::Tensor, α::Number) = (fill!(storage(T), α); T)

-(T::Tensor) = setstorage(T, -storage(T))

#function similar(::Type{<:Tensor{ElT,N,StoreT}},dims) where {ElT,N,StoreT}
#  return tensor(similar(StoreT,dim(dims)),dims)
#end

# TODO: make sure these are implemented correctly
#similar(T::Type{<:Tensor},::Type{S}) where {S} = tensor(similar(storage(T),S),inds(T))
#similar(T::Type{<:Tensor},::Type{S},dims) where {S} = tensor(similar(storage(T),S),dims)

similar(T::Tensor) = setstorage(T, similar(storage(T)))

# TODO: for BlockSparse, this needs to include the offsets
# TODO: for Diag, the storage is not just the total dimension
similar(T::Tensor, dims::Tuple) = _similar_from_dims(T, dims)
function similar(::Type{TensorT}, dims::Tuple) where {TensorT<:Tensor}
  return _similar_from_dims(TensorT, dims)
end
function similar(::Type{TensorT}, dims::Tuple{}) where {TensorT<:Tensor}
  return _similar_from_dims(TensorT, dims)
end

# To handle method ambiguity with AbstractArray
#similar(T::Tensor,dims::Dims) = _similar_from_dims(T,dims)

similar(T::Tensor, ::Type{S}) where {S} = setstorage(T, similar(storage(T), S))

function similar(::Type{TensorT}, ::Type{S}, dims) where {TensorT<:Tensor,S<:Number}
  return _similar_from_dims(TensorT, S, dims)
end

similar(T::Tensor, ::Type{S}, dims) where {S<:Number} = _similar_from_dims(T, S, dims)

# To handle method ambiguity with AbstractArray
similar(T::Tensor, ::Type{S}, dims::Dims) where {S<:Number} = _similar_from_dims(T, S, dims)

_similar_from_dims(T::Tensor, dims) = tensor(similar(storage(T), dim(dims)), dims)

function _similar_from_dims(T::Tensor, ::Type{S}, dims) where {S<:Number}
  return tensor(similar(storage(T), S, dim(dims)), dims)
end

function _similar_from_dims(::Type{TensorT}, dims) where {TensorT<:Tensor}
  return _similar_from_dims(TensorT, eltype(TensorT), dims)
end

function _similar_from_dims(
  ::Type{TensorT}, ::Type{S}, dims
) where {TensorT<:Tensor,S<:Number}
  return tensor(similar(storagetype(TensorT), S, dim(dims)), dims)
end

function convert(
  ::Type{<:Tensor{<:Number,N,StoreR,Inds}}, T::Tensor{<:Number,N,<:Any,Inds}
) where {N,Inds,StoreR}
  return setstorage(T, convert(StoreR, storage(T)))
end

function zeros(TensorT::Type{<:Tensor{ElT,N,StoreT}}, inds) where {ElT,N,StoreT}
  return error("zeros(::Type{$TensorT}, inds) not implemented yet")
end

function promote_rule(
  ::Type{<:Tensor{ElT1,N1,StoreT1,IndsT1}}, ::Type{<:Tensor{ElT2,N2,StoreT2,IndsT2}}
) where {ElT1,ElT2,N1,N2,StoreT1,StoreT2,IndsT1,IndsT2}
  StoreR = promote_type(StoreT1, StoreT2)
  ElR = eltype(StoreR)
  return Tensor{ElR,N3,StoreR,IndsR} where {N3,IndsR}
end

function promote_rule(
  ::Type{<:Tensor{ElT1,N,StoreT1,Inds}}, ::Type{<:Tensor{ElT2,N,StoreT2,Inds}}
) where {ElT1,ElT2,N,StoreT1,StoreT2,Inds}
  StoreR = promote_type(StoreT1, StoreT2)
  ElR = eltype(StoreR)
  return Tensor{ElR,N,StoreR,Inds}
end

# Convert the tensor type to the closest dense
# type
function dense(::Type{<:Tensor{ElT,NT,StoreT,IndsT}}) where {ElT,NT,StoreT,IndsT}
  return Tensor{ElT,NT,dense(StoreT),IndsT}
end

dense(T::Tensor) = setstorage(T, dense(storage(T)))

function similartype(
  ::Type{<:Tensor{ElT,<:Any,StoreT,<:Any}}, ::Type{IndsR}
) where {ElT,StoreT,IndsR}
  return Tensor{ElT,length(IndsR),StoreT,IndsR}
end

function similartype(
  ::Type{<:Tensor{ElT,<:Any,StoreT,<:Any}}, ::Type{IndsR}
) where {ElT,StoreT,IndsR<:NTuple{NR}} where {NR}
  return Tensor{ElT,NR,StoreT,IndsR}
end

# Convert to Array, avoiding copying if possible
array(T::Tensor) = array(dense(T))
matrix(T::Tensor{<:Number,2}) = array(T)
vector(T::Tensor{<:Number,1}) = array(T)

isempty(T::Tensor) = isempty(storage(T))

#
# Helper functions for BlockSparse-type storage
#

"""
nzblocks(T::Tensor)

Return a vector of the non-zero blocks of the BlockSparseTensor.
"""
nzblocks(T::Tensor) = nzblocks(storage(T))

eachnzblock(T::Tensor) = eachnzblock(storage(T))

blockoffsets(T::Tensor) = blockoffsets(storage(T))
nnzblocks(T::Tensor) = nnzblocks(storage(T))
nnz(T::Tensor) = nnz(storage(T))
nblocks(T::Tensor) = nblocks(inds(T))
blockdims(T::Tensor, block) = blockdims(inds(T), block)
blockdim(T::Tensor, block) = blockdim(inds(T), block)

"""
offset(T::Tensor, block::Block)

Get the linear offset in the data storage for the specified block.
If the specified block is not non-zero structurally, return nothing.

offset(T::Tensor,pos::Int)

Get the offset of the block at position pos
in the block-offsets list.
"""
offset(T::Tensor, block) = offset(storage(T), block)

"""
isblocknz(T::Tensor,
          block::Block)

Check if the specified block is non-zero
"""
isblocknz(T::Tensor, block) = isblocknz(storage(T), block)

function blockstart(T::Tensor{<:Number,N}, block) where {N}
  start_index = @MVector ones(Int, N)
  for j in 1:N
    ind_j = ind(T, j)
    for block_j in 1:(block[j] - 1)
      start_index[j] += blockdim(ind_j, block_j)
    end
  end
  return Tuple(start_index)
end

function blockend(T::Tensor{<:Number,N}, block) where {N}
  end_index = @MVector zeros(Int, N)
  for j in 1:N
    ind_j = ind(T, j)
    for block_j in 1:block[j]
      end_index[j] += blockdim(ind_j, block_j)
    end
  end
  return Tuple(end_index)
end

#
# Some generic getindex and setindex! functionality
#

@propagate_inbounds @inline setindex!!(T::Tensor, x, I...) = setindex!(T, x, I...)

insertblock!!(T::Tensor, block) = insertblock!(T, block)

"""
getdiagindex

Get the specified value on the diagonal
"""
function getdiagindex(T::Tensor{<:Number,N}, ind::Int) where {N}
  return getindex(T, CartesianIndex(ntuple(_ -> ind, Val(N))))
end

"""
setdiagindex!

Set the specified value on the diagonal
"""
function setdiagindex!(T::Tensor{<:Number,N}, val, ind::Int) where {N}
  setindex!(T, val, CartesianIndex(ntuple(_ -> ind, Val(N))))
  return T
end

#
# Some generic contraction functionality
#

function zero_contraction_output(
  T1::TensorT1, T2::TensorT2, indsR::IndsR
) where {TensorT1<:Tensor,TensorT2<:Tensor,IndsR}
  return zeros(contraction_output_type(TensorT1, TensorT2, IndsR), indsR)
end

#
# Broadcasting
#

BroadcastStyle(::Type{T}) where {T<:Tensor} = Broadcast.ArrayStyle{T}()

function Base.similar(
  bc::Broadcast.Broadcasted{Broadcast.ArrayStyle{T}}, ::Type{ElT}
) where {T<:Tensor,ElT}
  A = find_tensor(bc)
  return similar(A, ElT)
end

"`A = find_tensor(As)` returns the first Tensor among the arguments."
find_tensor(bc::Broadcast.Broadcasted) = find_tensor(bc.args)
find_tensor(args::Tuple) = find_tensor(find_tensor(args[1]), Base.tail(args))
find_tensor(x) = x
find_tensor(a::Tensor, rest) = a
find_tensor(::Any, rest) = find_tensor(rest)

function summary(io::IO, T::Tensor)
  for (dim, ind) in enumerate(inds(T))
    println(io, "Dim $dim: ", ind)
  end
  println(io, typeof(storage(T)))
  return println(io, " ", Base.dims2string(dims(T)))
end

#
# Printing
#

print_tensor(io::IO, T::Tensor) = Base.print_array(io, T)
print_tensor(io::IO, T::Tensor{<:Number,1}) = Base.print_array(io, reshape(T, (dim(T), 1)))


on_windows = @windows ? true : false
on_osx = @osx ? true : false

oldwdir = pwd()

nemo_pkgdir = Pkg.dir("Nemo")
nemo_wdir = Pkg.dir("Nemo", "deps")
nemo_vdir = Pkg.dir("Nemo", "local")

pkgdir = Pkg.dir("Hecke")
wdir = Pkg.dir("Hecke", "deps")
vdir = Pkg.dir("Hecke", "local")

if !ispath(Pkg.dir("Hecke", "local"))
    mkdir(Pkg.dir("Hecke", "local"))
end
if !ispath(Pkg.dir("Hecke", "local", "lib"))
    mkdir(Pkg.dir("Hecke", "local", "lib"))
end

function download_dll(url_string, location_string)
   try
      run(`curl -o $(location_string) -L $(url_string)`)
   catch
      download(url_string, location_string)
   end
end

LDFLAGS = "-Wl,-rpath,$nemo_vdir/lib -Wl,-rpath,\$\$ORIGIN/../share/julia/site/v$(VERSION.major).$(VERSION.minor)/Nemo/local/lib -Wl,-rpath,$vdir/lib -Wl,-rpath,\$\$ORIGIN/../share/julia/site/v$(VERSION.major).$(VERSION.minor)/Hecke/local/lib"
DLCFLAGS = "-fPIC -fno-common"

cd(wdir)

if on_windows
   if Int == Int64
      download_dll("http://www.mathematik.uni-kl.de/~thofmann/hecke/bin/libhecke.dll", joinpath(vdir, "lib", "libhecke.dll"))
   else
      println("There is no libhecke for 32 bit Windows")
   end
else
   cd("$wdir/hecke")
   withenv("LD_LIBRARY_PATH"=>"$vdir/lib:$nemo_vdir/lib", "LDFLAGS"=>LDFLAGS) do
      run(`./configure --prefix=$vdir --disable-static --enable-shared --with-mpir=$nemo_vdir --with-mpfr=$nemo_vdir --with-flint=$nemo_vdir`)
      run(`make -j4`)
      run(`make install`)
   end
end

push!(Libdl.DL_LOAD_PATH, Pkg.dir("Hecke", "local", "lib"))

cd(oldwdir)