export Material, UniformMaterial, TexturedMaterial, AATexturedMaterial
export getcolor

abstract type Material end

"""
    struct UniformMaterial <: Material

A uniform-looking material.

# Initialization

    UniformMaterial(r, g, b)

Initialize the material with a color whose red, green, and blue components
are `r`, `g`, `b`. All the three values must be in the range ``[0, 1]``.

# Examples

```julia
texture = SolidState.UniformMaterial(0.5, 0.5, 0.1)
```
"""
struct UniformMaterial <: Material
    color::RGB{Float64}

    UniformMaterial(r, g, b) = new(RGB{Float64}(r, g, b))
end

getcolor(mat::UniformMaterial, u, v) = mat.color

################################################################################

"""
    struct TexturedMaterial <: Material

A textured material, mapping a bitmap to a ``(u, v)`` grid in the domain
``[0, 1] \times [0, 1]``.

If you are looking for an anti-aliased textured material, use
`AATexturedMaterial`: it produce good-looking results even for
low-resolution bitmaps, at the expense of more calculations.

# Initialization

    TexturedMaterial(img::Array{RGB{Float64}, 2})

Initialize the texture with a bitmap.

    TexturedMaterial(fname::AbstractString)

Initialize the texture with a bitmap loaded from a file with name
`fname`. All the image types implemented in the
[Images.jl](https://github.com/JuliaImages/Images.jl) package are
recognized.

# Examples

```julia
texture = SolidState.TexturedMaterial("picture.jpg")
```
"""
struct TexturedMaterial <: Material
    texture::Array{RGB{Float64}, 2}

    TexturedMaterial(img::Array{RGB{Float64}, 2}) = new(img)
    TexturedMaterial(fname::AbstractString) = new(convert.(RGB{Float64},
                                                           Images.load(fname)))
end

function getcolor(mat::TexturedMaterial, u, v)
    height, width = size(mat.texture)
    mat.texture[round(Int, height - v * (height - 1)),
                round(Int, u * (width - 1) + 1)]
end

################################################################################

interp_texture(img) = LinearInterpolation((1:size(img)[1], 1:size(img)[2]), img)

"""
    struct AATexturedMaterial <: Material

An anti-aliased textured material, mapping a bitmap to a ``(u, v)`` grid in
the domain ``[0, 1] \times [0, 1]``.

Use `TexturedMaterial` if your textures are high-resolution, as it is faster.
Anti-aliasing only helps when the resolution of textures are so low that the
rendered object looks pixelated.

# Initialization

    AATexturedMaterial(img::Array{RGB{Float64}, 2})

Initialize the texture with a bitmap.

    AATexturedMaterial(fname::AbstractString)

Initialize the texture with a bitmap loaded from a file with name
`fname`. All the image types implemented in the
[Images.jl](https://github.com/JuliaImages/Images.jl) package are
recognized.

# Examples

```julia
texture = SolidState.AATexturedMaterial("picture.jpg")
```
"""
struct AATexturedMaterial <: Material
    texture

    AATexturedMaterial(img::Array{RGB{Float64}, 2}) = new(interp_texture(img))
    AATexturedMaterial(fname::AbstractString) = new(interp_texture(convert.(RGB{Float64},
                                                                            Images.load(fname))))
end

function getcolor(mat::AATexturedMaterial, u, v)
    height, width = size(mat.texture)
    mat.texture(height - v * (height - 1), u * (width - 1) + 1)
end


import Base.Iterators:flatten

using CodecLz4;
using ZfpCompression;

width = 14
height = 6
precision = 8

pixels = Float32(100.0) * randn(Float32, width, height)
mask = map(x -> x > -90, pixels)

compressed_pixels = zfp_compress(pixels, precision=precision)
compressed_mask = transcode(LZ4HCCompressor, collect(flatten(UInt8.(mask))))

println("compressed pixels size:", length(compressed_pixels))
println("compressed mask size:", length(compressed_mask))

compressed_mask = lz4_hc_compress(collect(flatten(UInt8.(mask))))
println("compressed mask size:", length(compressed_mask))

decomp = zfp_decompress(compressed_pixels)

display(pixels)
display(decomp)

# 1D spectrum compression
depth = 768
spectrum = Float32(100.0) * randn(Float32, depth)
precision = 24

compressed_spectrum = zfp_compress(spectrum, precision=precision)
println("compressed spectrum size: ", length(compressed_spectrum))

decomp = zfp_decompress(compressed_spectrum)

display(spectrum)
display(decomp)


function show_table(app, prop)
    callback!(
        app,
        Output("show_$prop", "children"),
        Output("store_$prop", "data"),
        Input("upload_$prop", "contents"),
        State("upload_$prop", "filename"),
        State("upload_$prop", "last_modified"),
        prevent_initial_call = true
    ) do contents, filename, last_modified
        df = parse_contents(contents, filename)
        df_show = show_table(df, filename, last_modified)
        df_json = JSON.json(df)
        return df_show, df_json
    end
end

function run_simulation(app)
    callback!(
        app,
        Output("sim_complete", "displayed"),
        Output("sim_complete", "message"),
        Output("build_and_run_msg", "children"),
        Output("store_inv_on_hand", "data"),
        Output("store_inv_level", "data"),
        Output("store_inv_position", "data"),
        Output("store_ech_position", "data"),
        Output("store_inv_pipeline", "data"),
        Output("store_market_demand", "data"),
        Output("store_replenishments", "data"),
        Input("build_and_run", "n_clicks"),
        State("store_bill_of_materials", "data"),
        State("store_lead_times", "data"),
        State("store_demand", "data"),
        State("store_policy", "data"),
        State("policy_variable","value"),
        State("policy_type","value"),
        State("mode","value"),
        State("num_periods","value"),
        prevent_initial_call = true
    ) do n_clicks, bom_json, lt_json, demand_json, policy_json, policy_variable, policy_type, op_mode, num_periods
        #initialize outputs
        msg = html_div("")
        msg_txt = ""
        show_msg = false
        inv_on_hand = DataFrame()
        inv_level = DataFrame()
        inv_position = DataFrame()
        ech_position = DataFrame()
        inv_pipeline = DataFrame()
        demand = DataFrame()
        replenishments = DataFrame()

        #trigger function call
        if num_periods > 0
            try
                #parse inputs
                bom_df = DataFrame(JSON.parse(bom_json))
                lt_df = DataFrame(JSON.parse(lt_json))
                demand_df = DataFrame(JSON.parse(demand_json))
                policy_df = DataFrame(JSON.parse(policy_json))
                backlog = op_mode == "backlog"
                policy_variable = Symbol(policy_variable)
                policy_type = Symbol(policy_type)

                #build and run model
                net = build_network(lt_df)
                build_bom!(net, bom_df)
                build_demand!(net, demand_df)
                env = run_policy!(net, policy_df, policy_variable, policy_type, num_periods, backlog)

                #competion message
                msg_txt = "Simulation Complete!"
                current_time = Libc.strftime(Libc.TmStruct(Libc.TimeVal().sec))
                msg = html_div("Simulation completed at $(current_time)", style = (color = "green",))
                show_msg = true

                #prepare results
                node_dict = get_prop(net, :node_dictionary)
                node_names = Dict(val => key for (key,val) in node_dict)
                inv_on_hand = env.inv_on_hand
                inv_level = env.inv_level
                inv_position = env.inv_position
                ech_position = env.ech_position
                demand = env.demand
                inv_pipeline = env.inv_pipeline
                replenishments = env.replenishments
                for df in [inv_on_hand, inv_level, inv_position, ech_position, demand]
                    transform!(df,
                        :node => ByRow(i -> node_names[i]) => :node
                    )
                end
                for df in [inv_pipeline, replenishments]
                    transform!(df,
                        :arc => ByRow(i -> "$(node_names[i[1]]) => $(node_names[i[2]])") => :arc
                    )
                end
            catch e
                current_time = Libc.strftime(Libc.TmStruct(Libc.TimeVal().sec))
                msg = html_div("Simulation failed at $(current_time) due to an error: $e.", style = (color = "red",))
                throw(e)
            end
        else
            msg = html_div("Enter a positive integer for the number of periods to run.", style = (color = "red",))
        end

        return show_msg, msg_txt, msg,
            JSON.json(inv_on_hand), JSON.json(inv_level),
            JSON.json(inv_position), JSON.json(ech_position),
            JSON.json(inv_pipeline), JSON.json(demand),
            JSON.json(replenishments)
    end
end

function download_results(app)
    callback!(
        app,
        Output("inv_onhand_result", "data"),
        Output("inv_level_result", "data"),
        Output("inv_position_result", "data"),
        Output("ech_position_result", "data"),
        Output("inv_pipeline_result", "data"),
        Output("market_demand_result", "data"),
        Output("replenishments_result", "data"),
        Input("download", "n_clicks"),
        State("store_inv_on_hand", "data"),
        State("store_inv_level", "data"),
        State("store_inv_position", "data"),
        State("store_ech_position", "data"),
        State("store_inv_pipeline", "data"),
        State("store_market_demand", "data"),
        State("store_replenishments", "data"),
        prevent_initial_call = true
    ) do n_clicks, inv_onhand_json, inv_level_json, inv_position_json, ech_position_json, inv_pipeline_json, market_demand_json, replenishments_json
        #parse results data
        inv_onhand = DataFrame(JSON.parse(inv_onhand_json))
        inv_level = DataFrame(JSON.parse(inv_level_json))
        inv_position = DataFrame(JSON.parse(inv_position_json))
        ech_position = DataFrame(JSON.parse(ech_position_json))
        inv_pipeline = DataFrame(JSON.parse(inv_pipeline_json))
        market_demand = DataFrame(JSON.parse(market_demand_json))
        replenishments = DataFrame(JSON.parse(replenishments_json))
        replace!(replenishments.reallocated, nothing => missing)
        replace!(inv_onhand.level, nothing => Inf)
        replace!(inv_level.level, nothing => Inf)
        replace!(inv_position.level, nothing => Inf)
        replace!(ech_position.level, nothing => Inf)

        return dcc_send_string(CSV.write, inv_onhand, "inv_onhand.csv"),
            dcc_send_string(CSV.write, inv_level, "inv_level.csv"),
            dcc_send_string(CSV.write, inv_position, "inv_position.csv"),
            dcc_send_string(CSV.write, ech_position, "ech_position.csv"),
            dcc_send_string(CSV.write, inv_pipeline, "inv_pipeline.csv"),
            dcc_send_string(CSV.write, market_demand, "market_demand.csv"),
            dcc_send_string(CSV.write, replenishments, "replenishments.csv")
    end
end

function build_callbacks(app)
    show_table(app, "bill_of_materials")
    show_table(app, "lead_times")
    show_table(app, "demand")
    show_table(app, "policy")
    run_simulation(app)
    download_results(app)
end


using DataStructures: CircularBuffer
using SHA

mutable struct TierrianOrganism
    key::UInt64

    a::UInt16
    b::UInt16
    c::UInt16
    d::UInt16

    stack::CircularBuffer{UInt16}

    ip::UInt16

    error_flag::Bool

    start_address::UInt16
    length::UInt16

    has_daughter::Bool
    daughter_address::UInt16
    daughter_length::UInt16

    hash::String
    parent_hash::String

    daughters::Vector{TierrianOrganism}

    time_birth::UInt64
    age::UInt16

    function TierrianOrganism(start_address::UInt16, length::UInt16)
        new(0, 0, 0, 0, 0, CircularBuffer{UInt16}(10), start_address, false, start_address, length, false, 0, 0, "", "", [], 0, 0)
    end
end

function _address_within(organism, address)
    mod(address - organism.start_address, organism.length) + organism.start_address
end


#! format: off
get_variable_multiplier(_, ::Type{<:PSY.RenewableGen}, ::AbstractRenewableFormulation) = 1.0
get_expression_type_for_reserve(::ActivePowerReserveVariable, ::Type{<:PSY.RenewableGen}, ::Type{<:PSY.Reserve{PSY.ReserveUp}}) = ActivePowerRangeExpressionUB
get_expression_type_for_reserve(::ActivePowerReserveVariable, ::Type{<:PSY.RenewableGen}, ::Type{<:PSY.Reserve{PSY.ReserveDown}}) = ActivePowerRangeExpressionLB
########################### ActivePowerVariable, RenewableGen #################################

get_variable_binary(::ActivePowerVariable, ::Type{<:PSY.RenewableGen}, ::AbstractRenewableFormulation) = false

get_variable_lower_bound(::ActivePowerVariable, d::PSY.RenewableGen, ::AbstractRenewableFormulation) = 0.0
get_variable_upper_bound(::ActivePowerVariable, d::PSY.RenewableGen, ::AbstractRenewableFormulation) = PSY.get_max_active_power(d)

########################### ReactivePowerVariable, RenewableGen #################################

get_variable_binary(::ReactivePowerVariable, ::Type{<:PSY.RenewableGen}, ::AbstractRenewableFormulation) = false

get_multiplier_value(::TimeSeriesParameter, d::PSY.RenewableGen, ::FixedOutput) = PSY.get_max_active_power(d)
get_multiplier_value(::TimeSeriesParameter, d::PSY.RenewableGen, ::AbstractRenewableFormulation) = PSY.get_max_active_power(d)

########################Objective Function##################################################
objective_function_multiplier(::ActivePowerVariable, ::AbstractRenewableDispatchFormulation)=OBJECTIVE_FUNCTION_NEGATIVE

variable_cost(::Nothing, ::ActivePowerVariable, ::PSY.RenewableDispatch, ::AbstractRenewableDispatchFormulation)=0.0
variable_cost(cost::PSY.OperationalCost, ::ActivePowerVariable, ::PSY.RenewableDispatch, ::AbstractRenewableDispatchFormulation)=PSY.get_variable(cost)

#! format: on

get_initial_conditions_device_model(
    ::OperationModel,
    ::DeviceModel{T, <:AbstractRenewableFormulation},
) where {T <: PSY.RenewableGen} = DeviceModel(T, RenewableFullDispatch)

get_initial_conditions_device_model(
    ::OperationModel,
    ::DeviceModel{T, FixedOutput},
) where {T <: PSY.RenewableGen} = DeviceModel(T, FixedOutput)

function get_min_max_limits(
    device,
    ::Type{ReactivePowerVariableLimitsConstraint},
    ::Type{<:AbstractRenewableFormulation},
)
    return PSY.get_reactive_power_limits(device)
end

function get_default_time_series_names(
    ::Type{<:PSY.RenewableGen},
    ::Type{<:Union{FixedOutput, AbstractRenewableFormulation}},
)
    return Dict{Type{<:TimeSeriesParameter}, String}(
        ActivePowerTimeSeriesParameter => "max_active_power",
        ReactivePowerTimeSeriesParameter => "max_active_power",
    )
end

function get_default_attributes(
    ::Type{<:PSY.RenewableGen},
    ::Type{<:Union{FixedOutput, AbstractRenewableFormulation}},
)
    return Dict{String, Any}()
end

####################################### Reactive Power constraint_infos #########################

function add_constraints!(
    container::OptimizationContainer,
    T::Type{<:ReactivePowerVariableLimitsConstraint},
    U::Type{<:ReactivePowerVariable},
    devices::IS.FlattenIteratorWrapper{V},
    model::DeviceModel{V, W},
    X::Type{<:PM.AbstractPowerModel},
) where {V <: PSY.RenewableGen, W <: AbstractDeviceFormulation}
    add_range_constraints!(container, T, U, devices, model, X)
    return
end

"""
Reactive Power Constraints on Renewable Gen Constant power_factor
"""
function add_constraints!(
    container::OptimizationContainer,
    ::Type{<:ReactivePowerVariableLimitsConstraint},
    ::Type{<:ReactivePowerVariable},
    devices::IS.FlattenIteratorWrapper{V},
    ::DeviceModel{V, W},
    ::Type{<:PM.AbstractPowerModel},
) where {V <: PSY.RenewableGen, W <: RenewableConstantPowerFactor}
    names = [PSY.get_name(d) for d in devices]
    time_steps = get_time_steps(container)
    p_var = get_variable(container, ActivePowerVariable(), V)
    q_var = get_variable(container, ReactivePowerVariable(), V)
    jump_model = get_jump_model(container)
    constraint =
        add_constraints_container!(container, EqualityConstraint(), V, names, time_steps)
    for t in time_steps, d in devices
        name = PSY.get_name(d)
        pf = sin(acos(PSY.get_power_factor(d)))
        constraint[name, t] =
            JuMP.@constraint(jump_model, q_var[name, t] == p_var[name, t] * pf)
    end
end

function add_constraints!(
    container::OptimizationContainer,
    T::Type{ActivePowerVariableLimitsConstraint},
    U::Type{<:Union{VariableType, ExpressionType}},
    devices::IS.FlattenIteratorWrapper{V},
    model::DeviceModel{V, W},
    X::Type{<:PM.AbstractPowerModel},
) where {V <: PSY.RenewableGen, W <: AbstractRenewableDispatchFormulation}
    add_parameterized_upper_bound_range_constraints(
        container,
        ActivePowerVariableTimeSeriesLimitsConstraint,
        U,
        ActivePowerTimeSeriesParameter,
        devices,
        model,
        X,
    )
    return
end

##################################### renewable generation cost ############################
function objective_function!(
    container::OptimizationContainer,
    devices::IS.FlattenIteratorWrapper{T},
    ::DeviceModel{T, U},
    ::Type{<:PM.AbstractPowerModel},
) where {T <: PSY.RenewableGen, U <: AbstractRenewableDispatchFormulation}
    add_variable_cost!(container, ActivePowerVariable(), devices, U())
    return
end


@testset "environment.jl" begin

    @testset "CPEnv{TS}" begin
        graph = Matrix(adjacency_matrix(random_regular_graph(6, 3)))
        ts = SeaPearl.DefaultTrajectoryState(GeometricFlux.FeaturedGraph(graph; nf=rand(3, 6)), 1)
        env = SeaPearl.CPEnv{SeaPearl.DefaultTrajectoryState}(
            .0,
            false,
            ts,
            collect(1:4),
            [1, 4],
            [true, false, false, true]
        )

        Random.seed!(0)
        actions = Set([agent(env) for I = 1:10])

        @test actions == Set([1,4])
    end
end


# Copyright (c) 2017: Miles Lubin and contributors
# Copyright (c) 2017: Google Inc.
#
# Use of this source code is governed by an MIT-style license that can be found
# in the LICENSE.md file or at https://opensource.org/licenses/MIT.

function _precompile_()
    ccall(:jl_generating_output, Cint, ()) == 1 || return nothing
    Base.precompile(
        Tuple{
            typeof(MOI.eval_hessian_lagrangian),
            NLPEvaluator,
            SubArray{Float64,1,Vector{Float64},Tuple{UnitRange{Int64}},true},
            Vector{Float64},
            Float64,
            SubArray{Float64,1,Vector{Float64},Tuple{UnitRange{Int64}},true},
        },
    )
    Base.precompile(
        Tuple{
            typeof(MOI.eval_objective_gradient),
            NLPEvaluator,
            Vector{Float64},
            Vector{Float64},
        },
    )
    Base.precompile(Tuple{typeof(MOI.initialize),NLPEvaluator,Vector{Symbol}})
    Base.precompile(Tuple{typeof(MOI.features_available),NLPEvaluator})
    Base.precompile(
        Tuple{
            typeof(MOI.eval_constraint_jacobian),
            NLPEvaluator,
            SubArray{Float64,1,Vector{Float64},Tuple{UnitRange{Int64}},true},
            Vector{Float64},
        },
    )
    Base.precompile(
        Tuple{
            typeof(MOI.eval_constraint),
            NLPEvaluator,
            SubArray{Float64,1,Vector{Float64},Tuple{UnitRange{Int64}},true},
            Vector{Float64},
        },
    )
    Base.precompile(Tuple{typeof(MOI.jacobian_structure),NLPEvaluator})
    Base.precompile(
        Tuple{typeof(MOI.eval_objective),NLPEvaluator,Vector{Float64}},
    )
    return
end

_precompile_()


using HTTP: get
using Gumbo: parsehtml
using DataFrames

"""
Fetches the codon usage table of species from http://www.kazusa.or.jp/codon/.
`species` is the ID of the species according to TODO.
`aa` speficies the genetic code with the values corresponding to:
    1: Standard
    2: Vertebrate Mitochondrial
    3: Yeast Mitochondrial
    4: Mold, Protozoan, Coelenterate Mitochondrial and Mycoplasma/Spiroplasma
    5: Invertebrate Mitochondrial
    6: Ciliate Macronuclear and Dasycladacean
    9: Echinoderm Mitochondrial
    10: ALternative Ciliate Macronuclear
    11: Eubacterial
    12: Alternative Yeast
    13: Ascidian Mitochondrial
    14: Flatworm Mitochondrial
    15: Blepharisma Nuclear Code

Returns a DataFrame containing aminoacids, their codons and codon-fraction, but no stop codons
"""
function get_codon_usage(; species = 9606, aa=1)
    r = HTTP.get("http://www.kazusa.or.jp/codon/cgi-bin/showcodon.cgi?species=$species&aa=$aa&style=N")
    table_string = parsehtml(String(r.body)).root[2][7][1].text
    df = DataFrame()
    for match in eachmatch(r"[AUGC]{3} [A-Z] \d\.\d\d [\d\s]\d\.\d \(\d+\)", table_string)
        entries = split(match.match)
        push!(df,
            (Codon = RNACodon(entries[1]), AminoAcid = AminoAcid(only(entries[2])), fraction = parse(Float64, entries[3]))
        )
    end
    return df
end


push!(LOAD_PATH, "/Users/megorov/Desktop/projects/stanford/sniper/egorov/sniperRepo/new-version")

using UrbanMap_
using Collada_
using SniperMDP_
using ValueIterations
using Simulator
using SimServers

mapPath = "/Users/megorov/Desktop/projects/stanford/sniper/egorov/sniperRepo/new-version/maps/demo_map_border2D.dae"
coll = ColladaObjects(mapPath)
map = UrbanMap(coll, 15, 15) # 15x15 map

mdp = SniperMDP(map, nSnipers = 1, nMonitors = 1, targets=[(2.0,14.0)])

vi = SerialSolver(maxIterations = 15)

policy = solve(vi, mdp, verbose=true)

#policy = load_policy("policies/demo-mdp-1.policy")


#This implementation is very similar to original implementation in https://github.com/stanfordnlp/mac-network
using JSON,Knet,Images,HDF5,Plots
#import AutoGrad: cat1d
using Printf,Random
include("model.jl")
savemodel(filename,m,mrun,o) = Knet.save(filename,"m",m,"mrun",mrun,"o",o)

function loadmodel(filename;onlywrun=false)
    d = Knet.load(filename)
    if onlywrun
        mrun=d["mrun"];o=d["o"]
        m=nothing
    else
        m=d["m"];o=d["o"];mrun=d["mrun"];
    end
    return m,mrun,o;
end

function getQdata(dhome,set)
    JSON.parsefile(dhome*set*".json")
end

function invert(vocab)
       int2tok = Array{String}(undef,length(vocab))
       for (k,v) in vocab; int2tok[v] = k; end
       return int2tok
end

function getDicts(dhome,dicfile)
    dic  = JSON.parsefile(dhome*dicfile*".json")
    qvoc = dic["word_dic"]
    avoc = dic["answer_dic"]
    i2w  = invert(qvoc)
    i2a  = invert(avoc)
    return qvoc,avoc,i2w,i2a
end

function loadFeatures(dhome,set;h5=false)
    if h5
        return h5open(dhome*set*".hdf5","r")["data"]
    else
        feats = reinterpret(Float32,read(open(dhome*set*".bin")))
        return reshape(feats,(14,14,1024,div(length(feats),200704)))
    end
end

function miniBatch(data;shfl=true,srtd=false,B=32)
    L = length(data)
    shfl && shuffle!(data)
    srtd && sort!(data;by=x->length(x[2]))
    batchs = [];
    for i=1:B:L
        b         = min(L-i+1,B)
        questions = Any[]
        answers   = zeros(Int,b)
        images    = Any[]
        families  = zeros(Int,b)

        for j=1:b
            crw = data[i+j-1]
            push!(questions,reverse(Array{Int}(crw[2])))
            push!(images,parse(Int,crw[1][end-9:end-4])+1)
            answers[j]  = crw[3]
            families[j] = crw[4]
        end

        lngths     = length.(questions);
        srtindices = sortperm(lngths;rev=true)

        lngths     = lngths[srtindices]
        Tmax       = lngths[1]
        questions  = questions[srtindices]
        answers    = answers[srtindices]
        images     = images[srtindices]
        families   = families[srtindices]

        qs = Int[];
        batchSizes = Int[];
        pads = falses(b,Tmax)

        for k=1:b
           pads[k,lngths[k]+1:Tmax].=true
        end

        if sum(pads)==0
           pads=nothing
        end

        while true
            batch = 0
            for j=1:b
                if length(questions[j]) > 0
                    batch += 1
                    push!(qs,pop!(questions[j]))
                end
            end
            if batch != 0
                push!(batchSizes,batch)
            else
                break;
            end
        end
        push!(batchs,(images,qs,answers,batchSizes,pads,families))
    end
    return batchs
end

function loadTrainingData(dhome="data/";h5=false)
    !h5 && println("Loading pretrained features for train&val sets.
                It requires minimum 70GB RAM!!!")
    trnfeats = loadFeatures(dhome,"train";h5=h5)
    valfeats = loadFeatures(dhome,"val";h5=h5)
    println("Loading questions ...")
    trnqstns = getQdata(dhome,"train")
    valqstns = getQdata(dhome,"val")
    println("Loading dictionaries ... ")
    qvoc,avoc,i2w,i2a = getDicts(dhome,"dic")
    return (trnfeats,valfeats),(trnqstns,valqstns),(qvoc,avoc,i2w,i2a)
end

function loadDemoData(dhome="data/demo/")
    println("Loading demo features ...")
    feats = loadFeatures(dhome,"demo")
    println("Loading demo questions ...")
    qstns = getQdata(dhome,"demo")
    println("Loading dictionaries ...")
    dics = getDicts(dhome,"dic")
    return feats,qstns,dics
end

function modelrun(M,data,feats,o,Mrun=nothing;train=false)
    getter(id) = view(feats,:,:,:,id)
    cnt=total=0.0; L=length(data);
    Mparams   = params(M)
    Rparams   = Mrun !== nothing ? params(Mrun) : nothing
    # results   = similar(Array{Float32},200704*48) #uncomment for INPLACE
    println("Timer Starts");
    for i in 1:L
        ids,questions,answers,batchSizes,pad,families = data[i]
        B    = batchSizes[1]
        xB   = atype(ones(Float32,1,B))
        #x    = inplace_batcher(results,feats,ids) #uncomment for INPLACE
        x    = batcher1(feats,ids) #comment for INPLACE
        xS   = atype(x)
        #xS   = atype(reshape(cat1d(map(getter,ids)...),14,14,1024,B))
        xP   = pad==nothing ? nothing : atype(pad*Float32(1e22))
        if train
            J = @diff M(questions,batchSizes,xS,xB,xP;answers=answers,p=o[:p],selfattn=o[:selfattn],gating=o[:gating])
            cnt += value(J)*B; total += B;
            for w in Mparams
                update!(w.value,grad(J,w),w.opt)
            end
            if Mrun != nothing
                for (wr,wi) in zip(Rparams,Mparams);
                    axpy!(1.0f0-o[:ema],wi.value-wr.value,wr.value);
                end
            end
        else
            preds  = M(questions,batchSizes,xS,xB,xP;p=o[:p],selfattn=o[:selfattn],gating=o[:gating])
            cnt   += sum(preds.==answers)
            total += B
        end
        i % 1000 == 0 && println(@sprintf("%.2f Accuracy|Loss", train ? cnt/total : 100cnt/total))
    end
    train && savemodel(o[:prefix]*".jld2",M,Mrun,o);
end

function train!(M,Mrun,sets,feats,o)
    @info "Training Starts...."
    setoptim!(M,o)
    for i=1:o[:epochs]
        println("Epoch $(i) starts...")
        modelrun(M,sets[1],feats[1],o,Mrun;train=true)
        modelrun(Mrun,sets[2],feats[2],o;train=false)
    end
    return M,Mrun;
end

function train(sets,feats,o)
     if o[:mfile]==nothing
         M    = MACNetwork(o);
         Mrun = deepcopy(M)
     else
         M,Mrun,o = loadmodel(o[:mfile])
     end
     train!(M,Mrun,sets,feats,o)
     return M,Mrun;
end

function train(dhome="data/",o=nothing)
     if o==nothing
         o=Dict(:h5=>false,:mfile=>nothing,:epochs=>10,
                :lr=>0.0001,:p=>12,:ema=>0.999f0,:batchsize=>32,
                :selfattn=>false,:gating=>false,:d=>512,
                :shuffle=>true,:sorted=>false,:prefix=>string(now())[1:10],
                :vocab_size=>90,:embed_size=>300, :dhome=>"data/", :loadresnet=>false)
     end
     feats,qdata,dics = loadTrainingData(dhome;h5=o[:h5])
     sets = []
     for q in qdata; push!(sets,miniBatch(q;shfle=o[:shuffle],srtd=o[:sorted])); end
     qdata = nothing; #gc();
     M,Mrun = train(sets,feats,o)
     return M,Mrun,sets,feats,dics;
end

function validate(Mrun,valset,valfeats,o)
     modelrun(Mrun,valset,valfeats,o;train=false)
end

function validate(mfile,valset,valfeats,o)
     _,Mrun,_ = loadmodel(mfile)
     modelrun(Mrun,valset,valfeats;train=false)
     return Mrun
end

function validate(mfile,dhome,o)
     _,Mrun,_,o   = loadmodel(mfile)
     valfeats     = loadFeatures(dhome,"val")
     qdata        = getQdata(dhome,"val")
     dics         = getDicts(dhome,"dic")
     valset       = miniBatch(qdata;shfle=o[:shuffle],srtd=o[:sorted])
     modelrun(Mrun,valset,valfeats,o;train=false)
     return Mrun,valset,valfeats
end

function singlerun(Mrun,feat,question;p=12,selfattn=false,gating=false)
    results        = Dict{String,Any}("cnt"=>1)
    batchSizes     = ones(Int,length(question))
    xB             = atype(ones(Float32,1,1))
    outputs = Mrun(question,batchSizes,feat,xB,nothing;tap=results,p=p,selfattn=selfattn,gating=gating,allsteps=true)
    prediction = argmax(results["y"])
    return results,prediction,outputs
end

function visualize(img,results;p=12)
    s_y,s_x = size(img)./14
    for k=1:p
        α = results["w_attn_$(k)"][:]
        wrds    = i2w[question]
        p = bar(α;xticks=(collect(1:length(wrds)),wrds),xrotation=90,bar_width = 1,
            xtickfont = font(8, "Courier"),yticks=0:.1:(maximum(α)+.1),
            legend=false,size=(600,100+400*(maximum(α))),aspect_ratio=10)
        savefig(p,"plots/$(k).png")
        display(RGB.(load("plots/$(k).png")))
        println("Image Attention Map: ")
        flush(stdout)
        hsvimg = HSV.(img);
        attn = results["KB_attn_$(k)"]
        for i=1:14,j=1:14
            rngy          = floor(Int,(i-1)*s_y+1):floor(Int,min(i*s_y,320))
            rngx          = floor(Int,(j-1)*s_x+1):floor(Int,min(j*s_x,480))
            hsvimg[rngy,rngx]  = scalepixel.(hsvimg[rngy,rngx],attn[LinearIndices((1:14,1:14))[i,j]])
        end
        display(hsvimg)
    end
end

function scalepixel(pixel,scaler)
     return HSV(pixel.h,pixel.s,min(1.0,pixel.v+5*scaler))
end

Courses/World of machine learning/visualize/src/main.jl


instr = "invalid value"
inhandle = 999%Clong
@test_throws ErrorException set_reference_stateS("Water", instr);
@test_throws ErrorException HAPropsSI(instr, "T", 298.15, "P", 101325, "R", 0.5)
@test_throws ErrorException HAPropsSI("H", instr, 298.15, "P", 101325, "R", 0.5)
@test_throws ErrorException PropsSI(instr, "rhomolar_critical")
@test_throws ErrorException PropsSI("n-Butane", instr)
@test_throws ErrorException PropsSI(instr, "T", 300, "P", 101325, "n-Butane")
@test_throws ErrorException PropsSI("D", "T", 300, "P", 101325, instr)
@test_throws ErrorException PhaseSI(instr, 300, "Q", 1, "Water")
@test_throws ErrorException PhaseSI("T", 300, "Q", 1, instr)
@test_throws ErrorException set_reference_stateS(instr, "DEF")
@test_throws ErrorException set_reference_stateS("Water", instr)
@test_throws ErrorException get_global_param_string(instr)
@test_throws ErrorException get_parameter_information_string(instr)
@test_throws ErrorException get_parameter_information_string("HMOLAR", instr)
@test_throws ErrorException get_fluid_param_string(instr, "aliases")
@test_throws ErrorException get_fluid_param_string("Water", instr)
@test_throws ErrorException get_param_index(instr)
@test_throws ErrorException get_input_pair_index(instr)
@test_throws ErrorException AbstractState_factory(instr, "R245fa")
@test_throws ErrorException AbstractState_factory("HEOS", instr)
@test_throws ErrorException AbstractState_free(inhandle)
@test_throws ErrorException AbstractState_set_fractions(inhandle, [0.0])
@test_throws ErrorException AbstractState_update(inhandle, "PQ_INPUTS", 101325, 0)
handle = AbstractState_factory("HEOS", "Water&Ethanol")
@test_throws ErrorException AbstractState_update(handle, instr, 101325, 0)
@test_throws ErrorException AbstractState_update(handle, inhandle, 101325, 0)
@test_throws ErrorException AbstractState_output(handle, instr)
@test_throws ErrorException AbstractState_output(inhandle, "T")
@test_throws ErrorException AbstractState_keyed_output(inhandle, 1%Clong)
@test_throws ErrorException AbstractState_keyed_output(handle, inhandle)
@test_throws ErrorException AbstractState_specify_phase(inhandle, "phase_gas")
@test_throws ErrorException AbstractState_specify_phase(handle, instr)
@test_throws ErrorException AbstractState_unspecify_phase(inhandle)
pq_inputs = get_input_pair_index("PQ_INPUTS")
T = [0.0]; p = [0.0]; rhomolar = [0.0]; hmolar = [0.0]; smolar = [0.0]
@test_throws ErrorException AbstractState_update_and_common_out(inhandle, pq_inputs, [101325.0], [0.0], 1, T, p, rhomolar, hmolar, smolar)
@test_throws ErrorException AbstractState_update_and_common_out(handle, instr, [101325.0], [0.0], 1, T, p, rhomolar, hmolar, smolar)
out = [0.0]
@test_throws ErrorException AbstractState_update_and_1_out(inhandle, pq_inputs, [101325.0], [0.0], 1, 1%Clong, out)
@test_throws ErrorException AbstractState_update_and_1_out(handle, instr, [101325.0], [0.0], 1, "T", out)
@test_throws ErrorException AbstractState_update_and_5_out(inhandle, pq_inputs, [101325.0], [0.0], 1, [1%Clong, 1%Clong, 1%Clong, 1%Clong, 1%Clong], out, out, out, out, out)
@test_throws ErrorException AbstractState_update_and_5_out(handle, instr, [101325.0], [0.0], 1, ["T", "T", "T", "T", "T"], out, out, out, out, out)
@test_throws ErrorException AbstractState_set_binary_interaction_double(inhandle, 0, 1, "betaT", 0.987)
@test_throws ErrorException AbstractState_set_binary_interaction_double(handle, 0, 1, instr, 0.987)
AbstractState_free(handle)
handle = AbstractState_factory("SRK", "Ethanol");
@test_throws ErrorException AbstractState_set_fluid_parameter_double(inhandle, 1, "c", 0.0)
@test_throws ErrorException AbstractState_set_fluid_parameter_double(handle, 0, instr, 0.0)
if (branchname == "nightly")
  @test_throws ErrorException AbstractState_set_cubic_alpha_C(inhandle, 0, "TWU", 0.0, 0.0, 0.0)
  @test_throws ErrorException AbstractState_set_cubic_alpha_C(handle, 0, instr, 0.0, 0.0, 0.0)
end
if (haskey(ENV, "includelocalwrapper") && ENV["includelocalwrapper"]=="on")
  @test_throws ErrorException saturation_ancillary(instr, "I", 1, "T", 300.0)
  @test_throws ErrorException saturation_ancillary("R410A", instr, 1, "T", 300.0)
  AbstractState_free(handle)
  handle = AbstractState_factory("HEOS", "Water")
  AbstractState_update(handle, "PQ_INPUTS", 15e5, 0)
  @test_throws ErrorException AbstractState_first_saturation_deriv(inhandle, 1%Clong, 2%Clong)
  @test_throws ErrorException AbstractState_first_saturation_deriv(handle, inhandle, 2%Clong)
  @test_throws ErrorException AbstractState_first_partial_deriv(inhandle, 1%Clong, 2%Clong, 3%Clong)
  @test_throws ErrorException AbstractState_first_partial_deriv(handle, inhandle, 2%Clong, 3%Clong)
  @test_throws ErrorException AbstractState_build_phase_envelope(inhandle, "none")
  len=100;t=zeros(len);p=zeros(len);x=zeros(2*len);y=zeros(2*len);rhomolar_vap=zeros(len);rhomolar_liq=zeros(len);
  @test_throws ErrorException AbstractState_get_phase_envelope_data(inhandle, len, t, p, rhomolar_vap, rhomolar_liq, x, y)
  @test_throws ErrorException AbstractState_build_spinodal(inhandle)
  rhomolar=zeros(len); stable=zeros(Clong, len)
  @test_throws ErrorException AbstractState_all_critical_points(inhandle, 2, t, p, rhomolar, stable)
  tau=zeros(len);delta=zeros(len);m1=zeros(len)
  @test_throws ErrorException AbstractState_get_spinodal_data(inhandle, len, tau, delta, m1)
  AbstractState_free(handle)
end


#!/usr/bin/env julia

function packagelt(a::String, b::String)
    a == "julia" && b != "julia" && return true
    a != "julia" && b == "julia" && return false
    return lowercase(a) < lowercase(b)
end

≲(v::VersionNumber, t::NTuple{0,Int}) = true
≲(v::VersionNumber, t::NTuple{1,Int}) = v.major ≤ t[1]
≲(v::VersionNumber, t::NTuple{2,Int}) = v.major < t[1] ||
                                        v.major ≤ t[1] && v.minor ≤ t[2]
≲(v::VersionNumber, t::NTuple{3,Int}) = v.major < t[1] ||
                                        v.major ≤ t[1] && v.minor < t[2] ||
                                        v.major ≤ t[1] && v.minor ≤ t[2] && v.patch ≤ t[3]

≲(t::NTuple{0,Int}, v::VersionNumber) = true
≲(t::NTuple{1,Int}, v::VersionNumber) = t[1] ≤ v.major
≲(t::NTuple{2,Int}, v::VersionNumber) = t[1] < v.major ||
                                        t[1] ≤ v.major && t[2] ≤ v.minor
≲(t::NTuple{3,Int}, v::VersionNumber) = t[1] < v.major ||
                                        t[1] ≤ v.major && t[2] < v.minor ||
                                        t[1] ≤ v.major && t[2] ≤ v.minor && t[3] ≤ v.patch

function compress_versions(inc::Vector{VersionNumber}, from::Vector{VersionNumber})
    issorted(inc) || (inc = sort(inc))
    exc = sort!(setdiff(from, inc))
    pairs = []
    if isempty(exc)
        lo, hi = first(inc), last(inc)
        push!(pairs, (lo.major, lo.minor) => (hi.major, hi.minor))
    else
        for v in inc
            t = (v.major, v.minor)
            if any(t ≲ w ≲ t for w in exc)
                t = (v.major, v.minor, v.patch)
            end
            if isempty(pairs) || any(pairs[end][1] ≲ w ≲ t for w in exc)
                push!(pairs, t => t) # need a new interval
            else
                pairs[end] = pairs[end][1] => t # can be merged with last
            end
        end
    end
    @assert all(any(p[1] ≲ v ≲ p[2] for p ∈ pairs) for v ∈ inc)
    @assert all(!any(p[1] ≲ v ≲ p[2] for p ∈ pairs) for v ∈ exc)
    return pairs
end
compress_versions(f::Function, from::Vector{VersionNumber}) =
    compress_versions(filter(f, from), from)
compress_versions(vi::VersionInterval, from::Vector{VersionNumber}) =
    compress_versions(v->v in vi, from)
compress_versions(inc, from) = compress_versions(inc, collect(from))

versions_string(p::Pair) = versions_string(p...)
versions_string(a::Tuple{}, b::Tuple{}) = "*"
versions_string(a::NTuple{m,Int}, b::NTuple{n,Int}) where {m,n} =
    a == b ? join(a, '.') : "$(join(a, '.'))-$(join(b, '.'))"

versions_repr(x) = repr(versions_string(x))
versions_repr(v::Vector) = length(v) == 1 ? repr(versions_string(v[1])) :
    "[" * join(map(repr∘versions_string, v), ", ") * "]"

## compress per-version data ##

function compress_versions_data(
    d::Dict{VersionNumber,Dict{String,String}},
    versions::Vector{VersionNumber},
)
    kvs = Dict{Pair{String,String},Vector{VersionNumber}}()
    for (ver, x) in d, (k, v) in x
        push!(get!(kvs, k => v, VersionNumber[]), ver)
    end
    tx = Tuple{Any,String,String}[]
    for (kv, vs) in kvs
        for v in compress_versions(sort!(vs), versions)
            push!(tx, (v, kv...))
        end
    end
    sort!(tx, by=t->(t[1][1], t[1][2], t[2], t[3]))
    return Tuple{String,String,String}[
        (versions_string(t[1]), t[2], t[3]) for t in tx
    ]
end

## dict utility functions ##

function invert_map(fwd::Dict{K,V}) where {K,V}
    rev = Dict{V,Vector{K}}()
    for (k, v) in fwd
        push!(get!(rev, v, K[]), k)
    end
    return rev
end

function invert_map(fwd::Dict{Vector{K},V}) where {K,V}
    rev = Dict{V,Vector{K}}()
    for (k, v) in fwd
        append!(get!(rev, v, K[]), k)
    end
    return rev
end

flatten_keys(d::Dict{Vector{K},V}) where {K,V} =
    isempty(d) ? Dict{K,V}() : Dict{K,V}(k => v for (ks, v) in d for k in ks)


using Showoff
using Compat
using Compat.Test
using Compat.Dates

@testset "Internals" begin
    @test Showoff.@grisu_ccall(1, 2, 3) === nothing
    @test Showoff.grisu(1.0, Base.Grisu.SHORTEST, 2) == (1, 1, false, Base.Grisu.DIGITS)

    let x = [1.0, Inf, 2.0, NaN]
        @test Showoff.concrete_minimum(x) == 1.0
        @test Showoff.concrete_maximum(x) == 2.0
    end

    @test_throws ArgumentError Showoff.concrete_minimum([])
    @test_throws ArgumentError Showoff.concrete_maximum([])

    let x = [1.12345, 4.5678]
        @test Showoff.plain_precision_heuristic(x) == 5
        @test Showoff.scientific_precision_heuristic(x) == 6
    end
end

@testset "Formatting" begin
    @test Showoff.format_fixed(-10.0, 0) == "-10"
    @test Showoff.format_fixed(0.012345, 3) == "0.012"
    @test Showoff.format_fixed(Inf, 1) == "∞"
    @test Showoff.format_fixed(-Inf, 1) == "-∞"
    @test Showoff.format_fixed(NaN, 1) == "NaN"
    @test Showoff.format_fixed_scientific(0.0, 1, false) == "0"
    @test Showoff.format_fixed_scientific(Inf, 1, false) == "∞"
    @test Showoff.format_fixed_scientific(-Inf, 1, false) == "-∞"
    @test Showoff.format_fixed_scientific(NaN, 1, false) == "NaN"
    @test Showoff.format_fixed_scientific(0.012345678, 4, true) == "12.34568×10⁻³"
    @test Showoff.format_fixed_scientific(0.012345678, 4, false) == "1.234568×10⁻²"
    @test Showoff.format_fixed_scientific(-10.0, 4, false) == "-1.000×10¹"
end

@testset "Showoff" begin
    x = [1.12345, 4.5678]
    @test showoff(x) == ["1.12345", "4.56780"]
    @test showoff([0.0, 50000.0]) == ["0", "5×10⁴"]
    @test showoff(x, :plain) == ["1.12345", "4.56780"]
    @test showoff(x, :scientific) == ["1.12345×10⁰", "4.56780×10⁰"]
    @test showoff(x, :engineering) == ["1.12345×10⁰", "4.56780×10⁰"]
    @test showoff([DateTime("2017-04-11", "yyyy-mm-dd")]) == ["Apr 11, 2017"]
    @test showoff(["a", "b"]) == ["\"a\"", "\"b\""]
    @test_throws ArgumentError showoff(x, :nevergonnagiveyouup)
    @test_throws ArgumentError showoff([Inf, Inf, NaN])
end


using Random

function RLCore.Experiment(::Val{:JuliaRL}, ::Val{:Minimax}, ::Val{:OpenSpiel}, game;)
    env = OpenSpielEnv(string(game))
    agents = (
        Agent(policy = MinimaxPolicy(), role = 0),
        Agent(policy = MinimaxPolicy(), role = 1),
    )
    hooks = (TotalRewardPerEpisode(), TotalRewardPerEpisode())
    description = """
      # Play `$game` in OpenSpiel with Minimax
      """
    Experiment(agents, env, StopAfterEpisode(1), hooks, description)
end

function RLCore.Experiment(
    ::Val{:JuliaRL},
    ::Val{:TabularCFR},
    ::Val{:OpenSpiel},
    game;
    n_iter = 300,
    seed = 123,
)
    env = OpenSpielEnv(
        game;
        default_state_style = RLBase.Information{String}(),
        is_chance_agent_required = true,
    )
    rng = StableRNG(seed)
    π = TabularCFRPolicy(; rng = rng)

    description = """
      # Play `$game` in OpenSpiel with TabularCFRPolicy
      """
    Experiment(π, env, StopAfterStep(300), EmptyHook(), description)
end

function RLCore.Experiment(
    ::Val{:JuliaRL},
    ::Val{:DeepCFR},
    ::Val{:OpenSpiel},
    game;
    n_iter = 300,
    seed = 123,
)
    @assert game == "leduc_poker"
    env = OpenSpielEnv(
        "leduc_poker";
        default_state_style = RLBase.Information{Array}(),
        is_chance_agent_required = true,
    )

    #= using CxxWrap =#

    p = DeepCFR(
        Π = NeuralNetworkApproximator(
            model = Chain(Dense(30, 64, relu), Dense(64, 64, relu), Dense(64, 3)) |> gpu,
            optimizer = ADAM(0.001),
        ),
        MΠ = ReservoirTrajectory(
            3_000_000,
            :I => Any, #= CxxWrap.StdLib.StdVectorAllocated{Float64}, =#
            :t => Int,
            :σ => Vector{Float32},
            :m => Vector{Bool},
        ),
        V = Dict(
            p => NeuralNetworkApproximator(
                model = Chain(Dense(30, 64, relu), Dense(64, 64, relu), Dense(64, 3)) |> gpu,
                optimizer = ADAM(0.001),
            ) for p in get_players(env) if p != get_chance_player(env)
        ),
        MV = Dict(
            p => ReservoirTrajectory(
                3_000_000,
                :I => Any,#=CxxWrap.StdLib.StdVectorAllocated{Float64}=#
                :t => Int,
                :r̃ => Vector{Float32},
                :m => Vector{Bool},
            ) for p in get_players(env) if p != get_chance_player(env)
        ),
        K = 1500,
        n_training_steps_V = 750,
        n_training_steps_Π = 2000,
        batch_size_V = 2048,
        batch_size_Π = 2048,
        initializer = glorot_normal(CUDA.CURAND.default_rng()),
    )
    # nash_conv ≈ 0.23
    Experiment(p, env, StopAfterStep(500), EmptyHook(), "# run DeepcCFR on leduc_poker")
end


export AbstractBlock, PrimitiveBlock, CompositeBlock, AbstractContainer, TagBlock

"""
    AbstractBlock

Abstract type for quantum circuit blocks.
"""
abstract type AbstractBlock{N} end

"""
    PrimitiveBlock{N} <: AbstractBlock{N}

Abstract type that all primitive block will subtype from. A primitive block
is a concrete block who can not be decomposed into other blocks. All composite
block can be decomposed into several primitive blocks.

!!! note

    subtype for primitive block with parameter should implement `hash` and `==`
    method to enable key value cache.
"""
abstract type PrimitiveBlock{N} <: AbstractBlock{N} end

"""
    CompositeBlock{N} <: AbstractBlock{N}

Abstract supertype which composite blocks will inherit from. Composite blocks
are blocks composited from other [`AbstractBlock`](@ref)s, thus it is a `AbstractBlock`
as well.
"""
abstract type CompositeBlock{N} <: AbstractBlock{N} end

"""
    AbstractContainer{BT, N} <: CompositeBlock{N}

Abstract type for container block. Container blocks are blocks contain a single
block. Container block should have a
"""
abstract type AbstractContainer{BT<:AbstractBlock,N} <: CompositeBlock{N} end

"""
    TagBlock{BT, N} <: AbstractContainer{BT, N}

`TagBlock` is a special kind of Container block, it forwards most of the methods
but tag the block with some extra information.
"""
abstract type TagBlock{BT,N} <: AbstractContainer{BT,N} end

"""
    content(x)

Returns the content of `x`.
"""
@interface content

"""
    chcontent(x, blk)

Create a similar block of `x` and change its content to blk.
"""
@interface chcontent

"""
    apply!(register, block)

Apply a block (of quantum circuit) to a quantum register.
"""

@interface apply!

"""
    occupied_locs(x)

Return a tuple of occupied locations of `x`.
"""
@interface occupied_locs

"""
    subblocks(x)

Returns an iterator of the sub-blocks of a composite block. Default is empty.
"""
@interface subblocks

"""
    chsubblocks(composite_block, itr)

Change the sub-blocks of a [`CompositeBlock`](@ref) with given iterator `itr`.
"""
@interface chsubblocks

@interface print_block

"""
    mat([T=ComplexF64], blk)

Returns the matrix form of given block.
"""
@interface mat

# parameters
"""
    getiparams(block)

Returns the intrinsic parameters of node `block`, default is an empty tuple.
"""
@interface getiparams

"""
    setiparams!(block, itr)
    setiparams!(block, params...)

Set the parameters of `block`.
"""
@interface setiparams!

"""
    parameters(block)

Returns all the parameters contained in block tree with given root `block`.
"""
@interface parameters

"""
    nparameters(block) -> Int

Return number of parameters in `block`. See also [`niparams`](@ref).
"""
@interface nparameters

"""
    niparam(block) -> Int

Return number of intrinsic parameters in `block`. See also [`nparameters`](@ref).
"""
@interface niparams

"""
    iparams_eltype(block)

Return the element type of [`getiparams`](@ref).
"""
@interface iparams_eltype

"""
    parameters_eltype(x)

Return the element type of [`parameters`](@ref).
"""
@interface parameters_eltype

"""
    dispatch!(x::AbstractBlock, collection)

Dispatch parameters in collection to block tree `x`.

!!! note

    it will try to dispatch the parameters in collection first.
"""
@interface dispatch!

"""
    render_params(r::AbstractBlock, params)

`params` can be a number or a symbol like `:zero` and `:random`.
This function renders the input parameter to a consumable type to `r`.
"""
@interface render_params

"""
    expect(op::AbstractBlock, reg) -> Vector
    expect(op::AbstractBlock, reg => circuit) -> Vector
    expect(op::AbstractBlock, density_matrix) -> Vector

Get the expectation value of an operator, the second parameter can be a register `reg` or a pair of input register and circuit `reg => circuit`.

expect'(op::AbstractBlock, reg=>circuit) -> Pair
expect'(op::AbstractBlock, reg) -> AbstracRegister

Obtain the gradient with respect to registers and circuit parameters.
For pair input, the second return value is a pair of `gψ=>gparams`,
with `gψ` the gradient of input state and `gparams` the gradients of circuit parameters.
For register input, the return value is a register.

!!! note

    For batched register, `expect(op, reg=>circuit)` returns a vector of size number of batch as output. However, one can not differentiate over a vector loss, so `expect'(op, reg=>circuit)` accumulates the gradient over batch, rather than returning a batched gradient of parameters.
"""
@interface expect

"""
    operator_fidelity(b1::AbstractBlock, b2::AbstractBlock) -> Number

Operator fidelity defined as

```math
F^2 = \\frac{1}{d^2}\\left[{\\rm Tr}(b1^\\dagger b2)\\right]
```

Here, `d` is the size of the Hilbert space. Note this quantity is independant to global phase.
See arXiv: 0803.2940v2, Equation (2) for reference.
"""
@interface operator_fidelity

"""
    apply_back!((ψ, ∂L/∂ψ*), circuit::AbstractBlock, collector) -> AbstractRegister

back propagate and calculate the gradient ∂L/∂θ = 2*Re(∂L/∂ψ*⋅∂ψ*/∂θ), given ∂L/∂ψ*.
`ψ` is the output register, ∂L/∂ψ* should also be register type.

Note: gradients are stored in `Diff` blocks, it can be access by either `diffblock.grad` or `gradient(circuit)`.
Note2: now `apply_back!` returns the inversed gradient!
"""
@interface apply_back!

"""
    mat_back!(T, rb::AbstractBlock, adjy, collector)

Back propagate the matrix gradients.
"""
@interface mat_back!


struct SRTM <: RasterDataSource end

# SRTM Mirror with 5x5 degree tiles
const SRTM_URI = URI(scheme = "https", host = "srtm.csi.cgiar.org", path = "/wp-content/uploads/files/srtm_5x5/TIFF")

const HAS_SRTM_TILE = BitArray([
    0  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  0  1  1  0  0  0  0  0  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  0  0
    0  1  1  1  1  0  0  0  0  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  0  0  0  0  0  0  0  0  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  0  1  1  1  1  1  1
    0  0  0  0  0  0  0  0  0  0  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  0  0  0  0  0  0  0  0  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  0  0  0  0
    0  0  0  0  0  0  0  0  0  0  0  1  1  1  1  1  1  1  1  1  1  1  1  1  1  0  0  0  0  0  0  0  0  0  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  0  0  0  0  0  0
    0  0  0  0  0  0  0  0  0  0  0  1  1  1  1  1  1  1  1  1  1  1  0  0  0  0  0  0  0  1  1  1  0  0  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  0  0  0  0  0  0  0
    0  0  0  0  0  0  0  0  0  0  0  1  1  1  1  1  1  1  1  1  1  0  0  1  0  0  0  0  0  0  0  0  1  0  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  0  0  0  0  0  0  0
    1  1  0  0  0  0  0  0  0  0  0  0  1  1  1  1  1  1  1  1  1  0  0  0  0  0  0  0  0  0  0  0  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  0  1  0  0  0  0  0  0  0
    0  0  1  1  1  0  0  0  0  0  0  0  1  1  1  1  1  1  1  1  1  1  0  0  0  0  0  0  0  0  0  0  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  0  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  0  0  0  0  0
    0  0  1  0  1  1  0  0  0  0  0  0  0  1  1  1  1  1  1  1  1  1  1  1  0  0  0  0  0  0  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  0  0  1  1  1  1  1  1  1  1  1  1  1  0  0  0  0  1  0  0  0  1  0  0
    0  0  0  0  0  0  0  0  0  0  0  0  0  0  1  0  0  1  1  1  1  1  1  1  1  0  0  0  0  0  0  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  0  0  0  1  1  1  0  1  1  1  1  1  1  1  1  0  1  1  1  0  0  1  1  1  0
    0  0  0  1  0  0  0  0  0  0  0  0  0  0  0  0  0  0  1  1  1  1  1  1  1  1  0  0  0  0  0  0  0  1  1  1  1  1  1  1  1  1  1  1  1  1  1  0  0  0  1  1  1  0  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  0
    1  0  0  1  1  0  0  0  0  0  0  0  0  0  0  0  0  1  1  1  1  1  1  1  1  1  1  0  0  0  0  0  0  0  1  1  0  1  1  1  1  1  1  1  1  1  0  0  0  0  1  0  0  0  0  1  1  1  1  1  1  1  1  0  0  0  1  0  0  1  1  0
    0  1  0  0  0  1  0  0  0  0  0  0  0  0  0  0  0  1  1  1  1  1  1  1  1  1  1  1  1  1  0  0  0  0  0  0  0  1  1  1  1  1  1  1  1  0  1  1  0  0  1  0  0  0  0  1  1  1  1  1  1  1  1  1  1  1  1  1  0  1  1  1
    0  1  0  1  1  1  0  1  1  0  0  0  0  0  0  0  0  0  0  1  1  1  1  1  1  1  1  1  1  1  0  0  0  1  0  0  0  0  1  1  1  1  1  1  0  1  1  1  0  0  1  0  0  0  0  0  1  1  1  1  1  1  1  1  1  1  1  1  1  1  0  1
    1  1  1  1  0  1  1  1  1  0  0  0  0  0  0  0  0  0  0  0  1  1  1  1  1  1  1  1  1  0  0  0  0  0  0  0  0  0  1  1  1  1  1  1  1  1  1  1  0  0  0  0  0  0  0  1  0  1  0  1  1  1  1  1  1  1  1  1  1  1  1  1
    1  1  1  1  1  1  1  1  1  0  0  0  0  0  0  0  0  0  0  0  1  1  1  1  1  1  1  1  1  0  0  0  0  0  1  0  0  0  1  1  1  1  1  1  1  1  1  1  1  0  0  0  0  0  0  0  0  0  0  1  1  1  1  1  1  1  1  1  1  1  1  1
    1  1  0  0  1  1  1  1  1  1  1  1  0  0  0  0  0  0  0  0  0  1  1  1  1  1  1  1  0  0  1  0  0  0  0  0  0  0  1  1  1  1  1  1  1  1  0  1  0  0  0  0  0  0  0  0  0  0  1  1  1  1  1  1  1  1  1  1  1  1  1  0
    1  0  0  0  0  0  0  1  0  1  0  0  0  0  1  0  0  0  0  1  1  1  1  1  1  1  1  0  0  0  0  0  0  0  0  0  0  0  1  1  1  1  1  0  1  1  0  0  0  0  0  0  0  0  0  0  0  0  1  1  1  1  1  1  1  1  1  0  0  1  0  0
    1  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  1  1  1  1  1  1  1  0  0  0  0  0  0  0  0  0  0  0  0  0  1  1  1  1  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  1  1  1  1  1  1  1  1  1  1  0  0  1  0
    0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  1  1  1  1  0  0  0  0  0  0  0  0  1  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  1  0  0  0  0  0  0  0  1  0  0  0  1  1  1  1  0  0  0  1  1
    1  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  1  1  1  1  0  0  0  0  0  0  0  0  0  1  1  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  1  1  0  0  0  1  1  1
    0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  1  1  1  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  1  0  0  1  0  0  1  1  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  1  1  1
    0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  1  1  1  1  1  0  0  0  1  0  0  0  0  0  0  0  1  0  0  0  0  0  0  0  0  0  0  0  0  1  1  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  1  0  1  0  0
    0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  1  1  0  0  0  0  0  0  1  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  1  0  0  0  0
])

function _raster_tile_stem(tile_index::CartesianIndex)
    y, x = tile_index.I
    "srtm_$(lpad(x, 2, '0'))_$(lpad(y, 2, '0'))"
end

_rastername(::Type{SRTM}, tile_index::CartesianIndex{2}) = _raster_tile_stem(tile_index) * ".tif"
_rasterpath(T::Type{SRTM}, tile_index::CartesianIndex{2}) = joinpath(rasterpath(), "SRTM", _rastername(T, tile_index))

_zipname(::Type{SRTM}, tile_index::CartesianIndex{2}) = _raster_tile_stem(tile_index) * ".zip"
_zipurl(T::Type{SRTM}, tile_index::CartesianIndex{2}) = joinpath(SRTM_URI, _zipname(T, tile_index))
_zippath(T::Type{SRTM}, tile_index::CartesianIndex{2}) = joinpath(rasterpath(), "SRTM", "zips", _zipname(T, tile_index))

function _getraster(T::Type{SRTM}, tile_index::CartesianIndex{2})
    raster_path = _rasterpath(T, tile_index)
    if !isfile(raster_path)
        @info "Note: not all oceanic tiles exist in the SRTM dataset."
        zip_path = _zippath(T, tile_index)
        _maybe_download(_zipurl(T, tile_index), zip_path)
        mkpath(dirname(raster_path))
        raster_name = _rastername(T, tile_index)
        zf = ZipFile.Reader(zip_path)
        write(raster_path, read(_zipfile_to_read(raster_name, zf)))
        close(zf)
    end
    return raster_path
end

# Adapted from https://github.com/centreborelli/srtm4/blob/master/src/srtm4.c#L87-L117
function _wgs84_to_tile_x(x)
    # tiles longitude indexes go from 1 to 72,
    # covering the range from -180 to +180
    tile_x = (1 + floor(Int, (x + 180) / 5)) % 72
    return tile_x == 0 ? 72 : tile_x
end

function _wgs84_to_tile_y(y)
    y = clamp(y, -60, 60)
    tile_y = 1 + floor(Int, (60 - y) / 5)
    return tile_y == 25 ? 24 : tile_y
end

function bounds_to_tile_indices(::Type{SRTM}, bounds::NTuple{4,Real})
    bounds_to_tile_indices(SRTM, ((bounds[1], bounds[3]), (bounds[2], bounds[4])))
end
function bounds_to_tile_indices(::Type{SRTM}, (xs, ys)::NTuple{2,NTuple{2,Real}})
    _check_order(xs)
    _check_order(ys)
    t_xs = _wgs84_to_tile_x.(xs)
    t_ys = reverse(_wgs84_to_tile_y.(ys))
    @show t_xs t_ys
    return CartesianIndices((t_ys[1]:(t_ys[2]), t_xs[1]:(t_xs[2])))
end

_check_order((a, b)) = a > b && throw(ArgumentError("Upper bound $b less than lower bound $a"))

for op in (:getraster, :rastername, :rasterpath, :zipname, :zipurl, :zippath)
    _op = Symbol('_', op) # Name of internal function
    @eval begin
        # Broadcasting function dispatch
        function $_op(T::Type{SRTM}, tile_index::CartesianIndices)
            broadcast(tile_index) do I
                HAS_SRTM_TILE[I] ? $_op(T, I) : missing
            end
        end
        # Bounds to tile indices dispatch
        $_op(T::Type{SRTM}, bounds::Tuple) = $_op(T, bounds_to_tile_indices(T, bounds))

        # Public function definition with key-word arguments
        function $op(T::Type{SRTM}; bounds=nothing, tile_index=nothing)
            if isnothing(bounds) & isnothing(tile_index)
                :op === :getraster || return joinpath(rasterpath(), "SRTM")
                throw(ArgumentError("One of `bounds` or `tile_index` kwarg must be specified"))
            elseif !isnothing(bounds) & !isnothing(tile_index)
                throw(ArgumentError("Only one of `bounds` or `tile_index` should be specified. " *
                                    "found `bounds`=$bounds and `tile_index`=$tile_index"))
            else
                # Call the internal function without key-word arguments
                return $_op(T, isnothing(tile_index) ? bounds : tile_index)
            end
        end
    end
end


using Base.Test

using DataFrames

import Celeste: GalsimBenchmark

# Ensure GalSim test images are available.
wd = pwd()
cd(joinpath(Pkg.dir("Celeste"), "benchmark", "galsim"))
run(`make fetch`)
cd(wd)

if test_long_running
    GALSIM_CASES_EXERCISED = [
        "simple_star",
        "star_with_noise",
        "angle_and_axis_ratio_1",
        "galaxy_with_all",
        "galaxy_with_noise",
    ]
else
    GALSIM_CASES_EXERCISED = [
        "star_with_noise",
    ]
end

function assert_estimates_are_close(benchmark_results)
    for row in eachrow(benchmark_results)
        if ismissing(row[:truth])
            continue
        end
        if row[:variable] == :is_star
            maximum_error = 0.1
        elseif row[:variable] == :gal_frac_dev
            maximum_error = 0.2
        elseif row[:variable] == :gal_angle_deg
            maximum_error = 5
        else
            maximum_error = 0.2 * abs(row[:truth])
        end
        @test isapprox(row[:truth], row[:estimate],
                       atol=maximum_error)
    end
end

@testset "galsim benchmarks" begin
    @testset "single-source inference" begin
        truth, results = GalsimBenchmark.run_benchmarks(
            test_case_names=GALSIM_CASES_EXERCISED,
            joint_inference=false
        )
        assert_estimates_are_close(
            GalsimBenchmark.truth_comparison_df(truth, results))
    end

    @testset "GalSim benchmark tests, joint inference" begin
        truth, results = GalsimBenchmark.run_benchmarks(
            test_case_names=GALSIM_CASES_EXERCISED,
            joint_inference=true
        )
        assert_estimates_are_close(
            GalsimBenchmark.truth_comparison_df(truth, results))
    end
end


export absolute_genebodycoverage

"""
    absolute_genebodycoverage(path_bam::String, path_bed12::String;
							output_prefix::String = "", bin_size::Int=100)

Calculates absolute gene body coverage.

# Arguments
---------
- `path_bam::String`: Path to a BAM file.
- `path_bed12::String`: Path to a reference gene model file with BED12 format.
- `output_prefix::String`: Prefix for output files. If this keyword is not specified or set to "" (defalut), no output files are saved.
- `bin_size::Int`: Bin size for transcripts (Defalut: 100).
"""
function absolute_genebodycoverage(path_bam::String, path_bed12::String;
									output_prefix::String = "", bin_size::Int=100)
	if output_prefix != ""
		path_out = output_prefix * ".absoluteGeneBodyCoverage.txt"
		path_out_plot = output_prefix * ".absoluteGeneBodyCoverage.pdf"
		println(@sprintf "Start calculating absolute gene body coverage...\n- bam: %s\n- bed12: %s\n- output: %s\n          %s\n- bin_size: %d\n" path_bam path_bed12 path_out path_out_plot bin_size)
	end

	# File check
	if !isfile(path_bam)
		error(@sprintf "No such file: %s\n" path_bam)
	end
	if !isfile(path_bed12)
		error(@sprintf "No such file: %s\n" path_bed12)
	end
	if output_prefix != ""
		if !uv_access_writable(dirname(output_prefix))
			error(@sprintf "Output files are not writable to: %s\n" (dirname(output_prefix) != "" ? dirname(output_prefix) : "."))
		end
	end

	# Load transcripts information (BED12)
	transcripts = load_transcript(path_bed12)
	println(@sprintf "transcripts: %d" length(transcripts))

	# Calculate L_max (Maximum length of transcript)
	L_max = maximum(sum(BED.blocksizes(t)) for t in transcripts)
	println(@sprintf("L_max: %d", L_max))

	# Calculate K
	K = Int(ceil(L_max/bin_size))
	println(@sprintf("K: %d", K))

	# Calculate the number of transcripts in each bin
	N_transcript = zeros(Int, K)

	# Generate a L_max-length array for read coverage
	coverage = zeros(L_max)

	# Generate a K-length array for absolute coverage
	abcov = zeros(K)

	# Open BAM file
	open(BAM.Reader, path_bam, index=path_bam*".bai") do bam_reader

		# For each transcript
		println("Calculate absolute read coverage for each transcript...")

		for t in transcripts
			# Skip the transcript whose region does not exist in BAM
			if ! exists_region_bam(bam_reader, BED.chrom(t), BED.chromstart(t), BED.chromend(t))
				continue
			end

			# Get blockSizes (= exon lengths) and blockStarts (=start position of exons relative to chromStart of t)
			blockSizes = BED.blocksizes(t)
			L = sum(blockSizes)
			blockStarts = BED.blockstarts(t)

			# Count the number of transcripts whose 5'-end in each bin
			N_transcript[Int(ceil(sum(blockSizes)/bin_size))] += 1

			# Strand
			BED.strand(t)

			# Initialize coverage
			for p in 1:length(coverage)
				coverage[p] = 0
			end

			# Get the read coverage on each exon and save it to `coverage`
			# Note: Read coverage is saved in `coverage` in a 'Left justified' manner,
			#       and left is 5'-end.
			# If the strand of a transcript is '-' STRAND_NEG, read coverage was reversed.
			p_start = 1
			p_end = 0
			exon_start = BED.chromstart(t) # Returns 1-based coordinate(though 0-based in BED12 format)
			exon_end = 0
			for i in 1:length(blockSizes) # For each exon
				# Calculate start and end positions of an exon on the genome
				exon_start = blockStarts[i] - 1 + exon_start  # Returns 1-based coordinate(though 0-based in BED12 format)
				exon_end = exon_start + blockSizes[i] - 1

				# Calculate start and end positions of an exon on mRNA
				p_start = p_end + 1
				p_end = p_start + blockSizes[i] - 1

				# Get read coverage and save it
				if BED.strand(t) == STRAND_POS
					pos_cov = p_start:p_end
				elseif BED.strand(t) == STRAND_NEG
					pos_cov = (L-p_start+1):-1:(L-p_end+1)
				else # STRAND_NA, STRAND_BOTH
					continue
				end

				coverage[pos_cov] = readcoverage_bam_base(bam_reader, BED.chrom(t), exon_start, exon_end)
			end

			# Calculate count per bin
			# (Distance from 3'-end is binned.)
			# (k-th bin: [(k-1)*b+1, k*b])
			for k in 1:K # for each bin
				if k*bin_size > L # Case when k-th bin exceeds boundary of transcript
					if sum(coverage[1:(L-((k-1)*bin_size+1)+1)]) > 0
						abcov[k] += 1
						break
					end
				else
					if sum(coverage[(L-k*bin_size+1):(L-((k-1)*bin_size+1)+1)]) > 0
						abcov[k] += 1
					end
				end
			end
		end
	end

	# Calculate number of transcripts in each bin
	for k in (K-1):-1:1
		N_transcript[k] += N_transcript[k+1]
	end

	# Divide count by the number of transcripts for each bin
	for k in 1:K
		abcov[k] = abcov[k] / N_transcript[k]
	end

	if output_prefix != ""
		# Write output (TSV) (Sample_ID, Bin, Coverage)
		println("Write results...")
		bin_numbers = collect(1:K)
		bin_starts = (bin_numbers .- 1) .* bin_size .+ 1
		bin_ends = bin_numbers .* bin_size
		open(path_out, "w") do io
			writedlm(io, [bin_numbers bin_starts bin_ends abcov N_transcript], '\t')
		end

		# Save plot
		plot_absolute_coverage(abcov, out_path=path_out_plot)

		# Message
		println(@sprintf "Finished! Check output files:\n- %s\n- %s" path_out path_out_plot)
	end

    return(abcov)
end