# nextpow2(W) = vshl(one(W), vsub_fast(8sizeof(W), leading_zeros(vsub_fast(W, one(W)))))

# @inline _pick_vector(::StaticInt{W}, ::Type{T}) where {W,T} = Vec{W,T}
# @inline pick_vector(::Type{T}) where {T} = _pick_vector(pick_vector_width(T), T)
# @inline function pick_vector(::Val{N}, ::Type{T}) where {N, T}
#     _pick_vector(smin(nextpow2(StaticInt{N}()), pick_vector_width(T)), T)
# end
# pick_vector(N::Int, ::Type{T}) where {T} = pick_vector(Val(N), T)

@inline MM(::Union{Val{W},StaticInt{W}}) where {W} = MM{W}(0)
@inline MM(::Union{Val{W},StaticInt{W}}, i) where {W} = MM{W}(i)
@inline MM(::Union{Val{W},StaticInt{W}}, i::AbstractSIMDVector{W}) where {W} = i
@inline MM(::StaticInt{W}, i, ::StaticInt{X}) where {W,X} = MM{W,X}(i)
@inline gep(ptr::Ptr, i::MM) = gep(ptr, data(i))

@inline Base.one(::Type{MM{W,X,I}}) where {W,X,I} = one(I)
@inline staticm1(i::MM{W,X,I}) where {W,X,I} = MM{W,X}(vsub_fast(data(i), one(I)))
@inline staticp1(i::MM{W,X,I}) where {W,X,I} = MM{W,X}(vadd_fast(data(i), one(I)))
@inline vadd_fast(i::MM{W,X}, j::Integer) where {W,X} = MM{W,X}(vadd_fast(data(i), j))
@inline vadd_fast(i::Integer, j::MM{W,X}) where {W,X} = MM{W,X}(vadd_fast(i, data(j)))
@inline vadd_fast(i::MM{W,X}, ::StaticInt{j}) where {W,X,j} = MM{W,X}(vadd_fast(data(i), StaticInt{j}()))
@inline vadd_fast(::StaticInt{i}, j::MM{W,X}) where {W,X,i} = MM{W,X}(vadd_fast(StaticInt{i}(), data(j)))
@inline vadd_fast(i::MM{W,X}, ::StaticInt{0}) where {W,X} = i
@inline vadd_fast(::StaticInt{0}, j::MM{W,X}) where {W,X} = j
@inline vsub_fast(i::MM{W,X}, j::Integer) where {W,X} = MM{W,X}(vsub_fast(data(i), j))
@inline vsub_fast(i::MM{W,X}, ::StaticInt{j}) where {W,X,j} = MM{W,X}(vsub_fast(data(i), StaticInt{j}()))
@inline vsub_fast(i::MM{W,X}, ::StaticInt{0}) where {W,X} = i

@inline vadd(i::MM{W,X}, j::Integer) where {W,X} = MM{W,X}(vadd_fast(data(i), j))
@inline vadd(i::Integer, j::MM{W,X}) where {W,X} = MM{W,X}(vadd_fast(i, data(j)))
@inline vadd(i::MM{W,X}, ::StaticInt{j}) where {W,X,j} = MM{W,X}(vadd_fast(data(i), StaticInt{j}()))
@inline vadd(::StaticInt{i}, j::MM{W,X}) where {W,X,i} = MM{W,X}(vadd_fast(StaticInt{i}(), data(j)))
@inline vsub(i::MM{W,X}, j::Integer) where {W,X} = MM{W,X}(vsub_fast(data(i), j))
@inline vsub(i::MM{W,X}, ::StaticInt{j}) where {W,X,j} = MM{W,X}(vsub_fast(data(i), StaticInt{j}()))
@inline vsub(i::MM{W,X}, ::StaticInt{0}) where {W,X} = i
@inline vsub(i::MM) = i * StaticInt{-1}()
@inline vsub_fast(i::MM) = i * StaticInt{-1}()
@inline vmul(::StaticInt{M}, i::MM{W,X}) where {M,W,X} = MM{W}(vmul_fast(data(i), StaticInt{M}()), StaticInt{X}() * StaticInt{M}())
@inline vmul(i::MM{W,X}, ::StaticInt{M}) where {M,W,X} = MM{W}(vmul_fast(data(i), StaticInt{M}()), StaticInt{X}() * StaticInt{M}())
@inline vrem(i::MM{W,X,I}, ::Type{I}) where {W,X,I<:IntegerTypesHW} = i
@inline vrem(i::MM{W,X}, ::Type{I}) where {W,X,I<:IntegerTypesHW} = MM{W,X}(data(i) % I)
@inline veq(::AbstractIrrational, ::MM{W,<:Integer}) where {W} = zero(Mask{W})
@inline veq(x::AbstractIrrational, i::MM{W}) where {W} = x == Vec(i)
@inline veq(::MM{W,<:Integer}, ::AbstractIrrational) where {W} = zero(Mask{W})
@inline veq(i::MM{W}, x::AbstractIrrational) where {W} = Vec(i) == x

@inline function vsub(i::NativeTypes, j::MM{W,X}) where {W,X}
    MM(StaticInt{W}(), vsub(i, data(j)), -StaticInt{X}())
end
@inline function vsub_fast(i::NativeTypes, j::MM{W,X}) where {W,X}
    MM(StaticInt{W}(), vsub_fast(i, data(j)), -StaticInt{X}())
end

@inline function Base.in(m::MM{W,X,<:Integer}, r::AbstractUnitRange) where {W,X}
    vm = Vec(m)
    (vm ≥ first(r)) & (vm ≤ last(r))
end

@inline function pick_vector_width_shift(args::Vararg{Any,K}) where {K}
    W = pick_vector_width(args...)
    W, intlog2(W)
end


"""
Example usage of Agtor which distributes runs across available processors.
"""

using Dates, DataFrames, CSV
using Agtor
using Distributed, FileIO, Glob
using JLD2

using Profile, BenchmarkTools, OwnTime, Logging

addprocs(2, exeflags="--project=.")

@everywhere begin

    using Dates, DataFrames, CSV

    using Agtor

    # Start month/day of growing season
    # Represents when water allocations are announced.
    const gs_start = (5, 1)

    """
    Allocation callback for example.

    Any interactions with other models could be defined in these callbacks
    """
    function allocation_callback!(zone, dt_i)
        # Resetting allocations for example run
        if monthday(dt_i) == gs_start
            # Example annual water allocations
            allocs = (surface_water=150.0, groundwater=40.0)

            update_available_water!(zone, allocs)
        end
    end

end

"""
Model setup for example run.
"""
function setup_zone(data_dir::String="test/data/")
    zone_spec_dir::String = "$(data_dir)zones"
    zone_params::Dict{Symbol, Dict} = load_spec(zone_spec_dir)

    collated_specs::Array = []
    agparams = collect_agparams!(zone_params[:Zone_1], collated_specs; ignore=[:crop_spec])

    climate_data::String = "$(data_dir)climate/farm_climate_data.csv"
    climate::Climate = load_climate(climate_data)

    return create(zone_params[:Zone_1], climate), agparams
end

"""
Run example scenarios (for a single farm area) by distributing across available cores.
Results are saved to a file on completion, based on scenario id.
"""
function test_scenario_run(data_dir::String="test/data/", result_dir::String="")::Nothing
    z1, agparams = setup_zone(data_dir)
    z1.manager = BaseManager("test")

    scen_data = joinpath(data_dir, "scenarios", "sampled_params.csv")
    samples = DataFrame!(CSV.File(scen_data))

    # Each row represents a scenario to run.
    # We distribute these across available cores.
    @sync @distributed (hcat) for row_id in 1:nrow(samples)
        tmp_z = deepcopy(z1)
        update_model!(tmp_z, samples[row_id, :])
        res = run_model(tmp_z, run_timestep!, allocation_callback!)

        pth = joinpath(result_dir, "sampled_params_batch_run_distributed_$(row_id).jld2")
        save_results!(pth, string(row_id), res)
    end

    # Collate all results into a single JLD2 file
    fn_pattern = joinpath(result_dir, "sampled_params_batch_run_distributed_*.jld2")
    collate_results!(fn_pattern, "dist_collated.jld2")

    return
end

@btime test_scenario_run()

# Loading and displaying results
saved_results = load("dist_collated.jld2")
@info saved_results


"""
This function creates the model for a full thermal dispatch formulation depending on combination of devices, device_formulation and system_formulation
"""
function PSI.construct_device!(
    optimization_container::PSI.OptimizationContainer,
    sys::PSY.System,
    model::PSI.DeviceModel{T, ThermalStandardUCOutages},
    ::Type{S},
) where {T <: PSY.ThermalGen, S <: PM.AbstractPowerModel}
    devices = PSI.get_available_components(T, sys)

    if !PSI.validate_available_devices(T, devices)
        return
    end

    # Variables
    PSI.add_variables!(
        optimization_container,
        PSI.ActivePowerVariable,
        devices,
        ThermalStandardUCOutages(),
    )
    PSI.add_variables!(
        optimization_container,
        PSI.ReactivePowerVariable,
        devices,
        ThermalStandardUCOutages(),
    )
    PSI.add_variables!(
        optimization_container,
        PSI.OnVariable,
        devices,
        ThermalStandardUCOutages(),
    )
    PSI.add_variables!(
        optimization_container,
        PSI.StartVariable,
        devices,
        ThermalStandardUCOutages(),
    )
    PSI.add_variables!(
        optimization_container,
        PSI.StopVariable,
        devices,
        ThermalStandardUCOutages(),
    )

    # Initial Conditions
    PSI.initial_conditions!(optimization_container, devices, ThermalStandardUCOutages())

    # Constraints
    PSI.add_constraints!(
        optimization_container,
        PSI.RangeConstraint,
        PSI.ActivePowerVariable,
        devices,
        model,
        S,
        PSI.get_feedforward(model),
    )
    PSI.add_constraints!(
        optimization_container,
        PSI.RangeConstraint,
        PSI.ReactivePowerVariable,
        devices,
        model,
        S,
        PSI.get_feedforward(model),
    )
    PSI.commitment_constraints!(
        optimization_container,
        devices,
        model,
        S,
        PSI.get_feedforward(model),
    )
    outage_constraints!(
        optimization_container,
        devices,
        model,
        S,
        PSI.get_feedforward(model),
    )
    PSI.ramp_constraints!(
        optimization_container,
        devices,
        model,
        S,
        PSI.get_feedforward(model),
    )
    PSI.time_constraints!(
        optimization_container,
        devices,
        model,
        S,
        PSI.get_feedforward(model),
    )
    PSI.feedforward!(optimization_container, devices, model, PSI.get_feedforward(model))

    # Cost Function
    PSI.cost_function!(
        optimization_container,
        devices,
        model,
        S,
        PSI.get_feedforward(model),
    )

    return
end

"""
This function creates the model for a full thermal dispatch formulation depending on combination of devices, device_formulation and system_formulation
"""
function PSI.construct_device!(
    optimization_container::PSI.OptimizationContainer,
    sys::PSY.System,
    model::PSI.DeviceModel{T, ThermalStandardUCOutages},
    ::Type{S},
) where {T <: PSY.ThermalGen, S <: PM.AbstractActivePowerModel}
    devices = get_available_components(T, sys)

    if !PSI.validate_available_devices(T, devices)
        return
    end

    # Variables
    PSI.add_variables!(
        optimization_container,
        PSI.ActivePowerVariable,
        devices,
        ThermalStandardUCOutages(),
    )
    PSI.add_variables!(
        optimization_container,
        PSI.OnVariable,
        devices,
        ThermalStandardUCOutages(),
    )
    PSI.add_variables!(
        optimization_container,
        PSI.StartVariable,
        devices,
        ThermalStandardUCOutages(),
    )
    PSI.add_variables!(
        optimization_container,
        PSI.StopVariable,
        devices,
        ThermalStandardUCOutages(),
    )

    # Initial Conditions
    PSI.initial_conditions!(optimization_container, devices, ThermalStandardUCOutages())

    # Constraints
    PSI.add_constraints!(
        optimization_container,
        PSI.RangeConstraint,
        PSI.ActivePowerVariable,
        devices,
        model,
        S,
        PSI.get_feedforward(model),
    )
    PSI.commitment_constraints!(
        optimization_container,
        devices,
        model,
        S,
        PSI.get_feedforward(model),
    )
    outage_constraints!(
        optimization_container,
        devices,
        model,
        S,
        PSI.get_feedforward(model),
    )
    PSI.ramp_constraints!(
        optimization_container,
        devices,
        model,
        S,
        PSI.get_feedforward(model),
    )
    PSI.time_constraints!(
        optimization_container,
        devices,
        model,
        S,
        PSI.get_feedforward(model),
    )
    PSI.feedforward!(optimization_container, devices, model, PSI.get_feedforward(model))

    # Cost Function
    PSI.cost_function!(
        optimization_container,
        devices,
        model,
        S,
        PSI.get_feedforward(model),
    )

    return
end

function construct_device!(
    optimization_container::OptimizationContainer,
    sys::PSY.System,
    model::DeviceModel{T, ThermalDispatchOutages},
    ::Type{S},
) where {
    T <: PSY.ThermalGen,
    S <: PM.AbstractPowerModel,
}
    devices = get_available_components(T, sys)

    if !validate_available_devices(T, devices)
        return
    end

    # Variables
    add_variables!(optimization_container, ActivePowerVariable, devices, ThermalDispatchOutages())
    add_variables!(optimization_container, ReactivePowerVariable, devices, ThermalDispatchOutages())

    # Initial Conditions

    # Constraints
    add_constraints!(
        optimization_container,
        RangeConstraint,
        ActivePowerVariable,
        devices,
        model,
        S,
        get_feedforward(model),
    )
    add_constraints!(
        optimization_container,
        RangeConstraint,
        ReactivePowerVariable,
        devices,
        model,
        S,
        get_feedforward(model),
    )
    outage_constraints!(
        optimization_container,
        devices,
        model,
        S,
        PSI.get_feedforward(model),
    )
    feedforward!(optimization_container, devices, model, get_feedforward(model))

    # Cost Function
    cost_function!(optimization_container, devices, model, S, get_feedforward(model))

    return
end

function construct_device!(
    optimization_container::OptimizationContainer,
    sys::PSY.System,
    model::DeviceModel{T, ThermalDispatchOutages},
    ::Type{S},
) where {
    T <: PSY.ThermalGen,
    S <: PM.AbstractActivePowerModel,
}
    devices = get_available_components(T, sys)

    if !validate_available_devices(T, devices)
        return
    end

    # Variables
    add_variables!(optimization_container, ActivePowerVariable, devices, ThermalDispatchOutages())

    # Initial Conditions

    # Constraints
    add_constraints!(
        optimization_container,
        RangeConstraint,
        ActivePowerVariable,
        devices,
        model,
        S,
        get_feedforward(model),
    )
    outage_constraints!(
        optimization_container,
        devices,
        model,
        S,
        PSI.get_feedforward(model),
    )
    feedforward!(optimization_container, devices, model, get_feedforward(model))

    # Cost Function
    cost_function!(optimization_container, devices, model, S, get_feedforward(model))

    return
end


# MIT license
# Copyright (c) Microsoft Corporation. All rights reserved.
# See LICENSE in the project root for full license information.

# This file contains average properties of the human eye and optical human eye models

# luminance (cd/m2)	Multiple	Value	Item
# 10−6	µcd/m2	1 µcd/m2	Absolute threshold of vision[1]
# 10−5
# 10−4		400 µcd/m2	Darkest sky[2]
# 10−3	mcd/m2	1 mcd/m2	Night sky[3]
# 1.4   mcd/m2	Typical photographic scene lit by full moon[4]
# 5     mcd/m2	Approximate scotopic/mesopic threshold[5]
# 10−2		40 mcd/m2	Phosphorescent markings on a watch dial after 1 h in the dark[6][7]
# 10−1
# 100	cd/m2	2 cd/m2	Floodlit buildings, monuments, and fountains[8]
# 5     cd/m2	Approximate mesopic/photopic threshold[5]
# 101		25 cd/m2	Typical photographic scene at sunrise or sunset[4]
# 30    cd/m2	Green electroluminescent source[2]
# 102		250 cd/m2	Peak luminance of a typical LCD monitor[10][11]
# 700   cd/m2	Typical photographic scene on overcast day[4][8][11]
# 103	kcd/m2	2 kcd/m2	Average cloudy sky[2]
# 5     kcd/m2	Typical photographic scene in full sunlight[4][8]

"""
# Pupil diameter as a function of scene luminance
https://jov.arvojournals.org/article.aspx?articleid=2279420
https://en.wikipedia.org/wiki/Orders_of_magnitude_(luminance)

Pupil diameter is approximately 2.8mm at 100cd/m^2. A typical overcast day is 700cd/m^2
"""

"""computes pupil diameter as a function of scene luminance `L`, in cd/m², and the angular area, `a`, over which this luminance is presented to the eye."""
𝐃sd(L,a) = 7.75 - 5.75 * ((L * a / 846)^.41) / ((L * a / 846)^.41 + 2) # the first letter of this function name is \bfD not D.
export 𝐃sd

eyeradius() = 12mm
export eyeradius

"""Posterior focal length, i.e., optical distance from entrance pupil to the retina. Focal length will change depending on accomodation. This value is for focus at ∞. When the eye is focused at 25cm focal length will be ≈ 22mm. Because the index of refraction of the vitreous humor is approximately 1.33 the physical distance from the entrance pupil to the retina will be 24mm/1.33 = 18mm."""
eyefocallength() = 24mm
export eyefocallength

vc_epupil() = 3mm #distance from vertex of cornea to entrance pupil

""" distance from vertex of cornea to center of rotation"""
cornea_to_eyecenter() = 13.5mm
export cornea_to_eyecenter

entrancepupil_to_retina() = 22.9mm

"""distance from entrance pupil to center of rotation."""
entrancepupil_to_eyecenter() = entrancepupil_to_retina() - eyeradius()
export entrancepupil_to_eyecenter

"""average angle, in degrees, the eye will rotate before users will turn their head"""
comfortable_eye_rotation_angle() = 20°

"""Average one sided translation of the entrance pupil associated with comfortable eye rotation. If you are using this to define an eyebox multiply this value by 2"""
comfortable_entrance_pupil_translation() = sin(comfortable_eye_rotation_angle())*entrancepupil_to_eyecenter()
export comfortable_entrance_pupil_translation


#####
##### BaseCase methods
#####

function free_convection_windspeed(surf::SurfaceBase, grid, state, gm::GridMeanVariables, param_set, ::BaseCase)
    prog_gm = center_prog_grid_mean(state)
    aux_tc = center_aux_turbconv(state)
    zi = get_inversion(grid, state, param_set, surf.Ri_bulk_crit)
    wstar = get_wstar(surf.bflux, zi) # yair here zi in TRMM should be adjusted
    surf.windspeed = sqrt(surf.windspeed * surf.windspeed + (1.2 * wstar) * (1.2 * wstar))
    return
end

#####
##### Default SurfaceBase behavior
#####

free_convection_windspeed(surf::SurfaceBase, grid, state, gm::GridMeanVariables, param_set) =
    free_convection_windspeed(surf, grid, state, gm, param_set, BaseCase())

#####
##### SurfaceNone
#####

function update(surf::SurfaceBase{SurfaceNone}, grid, state, gm::GridMeanVariables, param_set)
    # JH: assigning small fluxed so that simulation won"t crash when computing mixing length
    kc_surf = kc_surface(grid)
    surf.windspeed = 0.0001
    surf.zrough = 1e-4
    surf.bflux = 1e-4
    surf.ustar = compute_friction_velocity(param_set, surf.windspeed, surf.bflux, surf.zrough, grid.zc[kc_surf])
    return
end
free_convection_windspeed(surf::SurfaceBase{SurfaceNone}, grid, state, gm::GridMeanVariables, param_set) = nothing

function update(surf::SurfaceBase{SurfaceFixedFlux}, grid, state, gm::GridMeanVariables, param_set)
    kc_surf = kc_surface(grid)
    kf_surf = kf_surface(grid)
    zb = grid.zc[kc_surf]
    aux_gm = center_aux_grid_mean(state)
    prog_gm = center_prog_grid_mean(state)
    p0_f_surf = face_ref_state(state).p0[kf_surf]
    ρ0_f_surf = face_ref_state(state).ρ0[kf_surf]
    α0_f_surf = face_ref_state(state).α0[kf_surf]
    u_gm_surf = prog_gm.u[kc_surf]
    v_gm_surf = prog_gm.v[kc_surf]
    q_tot_gm_surf = prog_gm.q_tot[kc_surf]
    θ_liq_ice_gm_surf = prog_gm.θ_liq_ice[kc_surf]
    T_gm_surf = aux_gm.T[kc_surf]

    ts = thermo_state_pθq(param_set, p0_f_surf, surf.Tsurface, surf.qsurface)
    rho_tflux = surf.shf / TD.cp_m(ts)

    surf.windspeed = sqrt(u_gm_surf^2 + v_gm_surf^2)
    surf.rho_qtflux = surf.lhf / TD.latent_heat_vapor(param_set, surf.Tsurface)

    ts = thermo_state_pθq(param_set, p0_f_surf, θ_liq_ice_gm_surf, q_tot_gm_surf)
    Π = TD.exner(ts)
    surf.rho_hflux = rho_tflux / Π
    surf.bflux = buoyancy_flux(param_set, surf.shf, surf.lhf, surf.Tsurface, surf.qsurface, α0_f_surf, ts)

    if !surf.ustar_fixed
        # Correction to windspeed for free convective cases (Beljaars, QJRMS (1994), 121, pp. 255-270)
        # Value 1.2 is empirical, but should be O(1)
        if surf.windspeed < 0.1  # Limit here is heuristic
            if surf.bflux > 0.0
                free_convection_windspeed(surf, grid, state, gm, param_set)
            else
                print("WARNING: Low windspeed + stable conditions, need to check ustar computation")
                print("surf.bflux ==>", surf.bflux)
                print("surf.shf ==>", surf.shf)
                print("surf.lhf ==>", surf.lhf)
                print("u_gm_surf ==>", u_gm_surf)
                print("v_gm_surf ==>", v_gm_surf)
                print("q_tot_gm_surf ==>", q_tot_gm_surf)
                print("α0_f_surf ==>", α0_f_surf)
                error("Terminating execution.")
            end
        end

        surf.ustar = compute_friction_velocity(param_set, surf.windspeed, surf.bflux, surf.zrough, zb)
    end

    von_karman_const = CPSGS.von_karman_const(param_set)
    surf.obukhov_length = obukhov_length(param_set, surf.ustar, surf.bflux)
    surf.rho_uflux = -ρ0_f_surf * surf.ustar^2 / surf.windspeed * u_gm_surf
    surf.rho_vflux = -ρ0_f_surf * surf.ustar^2 / surf.windspeed * v_gm_surf
    return
end

function update(surf::SurfaceBase{SurfaceFixedCoeffs}, grid, state, gm::GridMeanVariables, param_set)
    kc_surf = kc_surface(grid)
    kf_surf = kf_surface(grid)
    p0_f_surf = face_ref_state(state).p0[kf_surf]
    ρ0_f_surf = face_ref_state(state).ρ0[kf_surf]
    α0_f_surf = face_ref_state(state).α0[kf_surf]
    aux_gm = center_aux_grid_mean(state)
    prog_gm = center_prog_grid_mean(state)
    u_gm_surf = prog_gm.u[kc_surf]
    v_gm_surf = prog_gm.v[kc_surf]
    T_gm_surf = aux_gm.T[kc_surf]
    q_tot_gm_surf = prog_gm.q_tot[kc_surf]
    θ_liq_ice_gm_surf = prog_gm.θ_liq_ice[kc_surf]

    ts = thermo_state_pθq(param_set, p0_f_surf, surf.Tsurface, surf.qsurface)
    surf.windspeed = max(sqrt(u_gm_surf^2 + v_gm_surf^2), 0.01)
    cp_ = TD.cp_m(ts)
    lv = TD.latent_heat_vapor(ts)

    surf.rho_qtflux = -surf.cq * surf.windspeed * (q_tot_gm_surf - surf.qsurface) * ρ0_f_surf
    surf.lhf = lv * surf.rho_qtflux

    ts = thermo_state_pθq(param_set, p0_f_surf, θ_liq_ice_gm_surf, q_tot_gm_surf)
    Π = TD.exner(ts)
    surf.rho_hflux = -surf.ch * surf.windspeed * (θ_liq_ice_gm_surf - surf.Tsurface / Π) * ρ0_f_surf
    surf.shf = cp_ * surf.rho_hflux

    surf.bflux = buoyancy_flux(param_set, surf.shf, surf.lhf, surf.Tsurface, surf.qsurface, α0_f_surf, ts)

    surf.ustar = friction_velocity_given_windspeed(surf.cm, surf.windspeed)
    surf.obukhov_length = obukhov_length(param_set, surf.ustar, surf.bflux)

    surf.rho_uflux = -ρ0_f_surf * surf.ustar^2 / surf.windspeed * u_gm_surf
    surf.rho_vflux = -ρ0_f_surf * surf.ustar^2 / surf.windspeed * v_gm_surf
    return
end

function update(surf::SurfaceBase{SurfaceMoninObukhov}, grid, state, gm::GridMeanVariables, param_set)
    g = CPP.grav(param_set)
    kc_surf = kc_surface(grid)
    kf_surf = kf_surface(grid)
    zb = grid.zc[kc_surf]
    p0_f_surf = face_ref_state(state).p0[kf_surf]
    p0_c_surf = center_ref_state(state).p0[kc_surf]
    ρ0_f_surf = face_ref_state(state).ρ0[kf_surf]
    α0_f_surf = face_ref_state(state).α0[kf_surf]
    u_gm_surf = prog_gm.u[kc_surf]
    v_gm_surf = prog_gm.v[kc_surf]
    q_tot_gm_surf = prog_gm.q_tot[kc_surf]
    θ_liq_ice_gm_surf = prog_gm.θ_liq_ice[kc_surf]
    T_gm_surf = aux_gm.T[kc_surf]
    Pg = surf.ref_params.Pg

    pvg = TD.saturation_vapor_pressure(param_set, TD.PhaseEquil, surf.Tsurface)
    surf.qsurface = TD.q_vap_saturation_from_density(param_set, surf.Tsurface, ρ0_f_surf, pvg)
    lv = TD.latent_heat_vapor(param_set, T_gm_surf)

    phase_part = TD.PhasePartition(surf.qsurface, 0.0, 0.0)
    h_star = TD.liquid_ice_pottemp_given_pressure(param_set, surf.Tsurface, Pg, phase_part)

    ts_g = thermo_state_pθq(param_set, Pg, h_star, surf.qsurface)
    θ_virt_g = TD.virtual_pottemp(ts_g)

    ts_b = thermo_state_pθq(param_set, p0_c_surf, θ_liq_ice_gm_surf, surf.qsurface)
    θ_virt_b = TD.virtual_pottemp(ts_b)

    surf.windspeed = sqrt(u_gm_surf^2 + v_gm_surf^2)
    Nb2 = g / θ_virt_g * (θ_virt_b - θ_virt_g) / zb
    Ri = Nb2 * zb * zb / (surf.windspeed * surf.windspeed)

    surf.cm, surf.ch, surf.obukhov_length = exchange_coefficients_byun(param_set, Ri, zb, surf.zrough)
    surf.rho_uflux = -surf.cm * surf.windspeed * u_gm_surf * ρ0_f_surf
    surf.rho_vflux = -surf.cm * surf.windspeed * v_gm_surf * ρ0_f_surf

    surf.rho_hflux = -surf.ch * surf.windspeed * (θ_liq_ice_gm_surf - h_star) * ρ0_f_surf
    surf.rho_qtflux = -surf.ch * surf.windspeed * (q_tot_gm_surf - surf.qsurface) * ρ0_f_surf
    surf.lhf = lv * surf.rho_qtflux

    ts = thermo_state_pθq(param_set, p0_f_surf, surf.Tsurface, surf.qsurface)
    surf.shf = TD.cp_m(ts) * surf.rho_hflux

    surf.bflux = buoyancy_flux(param_set, surf.shf, surf.lhf, surf.Tsurface, surf.qsurface, α0_f_surf, ts)
    surf.ustar = friction_velocity_given_windspeed(surf.cm, surf.windspeed)

    return
end

function update(surf::SurfaceBase{SurfaceMoninObukhovDry}, grid, state, gm::GridMeanVariables, param_set)
    g = CPP.grav(param_set)
    kc_surf = kc_surface(grid)
    kf_surf = kf_surface(grid)

    zb = grid.zc[kc_surf]
    p0_f_surf = face_ref_state(state).p0[kf_surf]
    p0_c_surf = center_ref_state(state).p0[kc_surf]
    ρ0_f_surf = face_ref_state(state).ρ0[kf_surf]
    α0_f_surf = face_ref_state(state).α0[kf_surf]
    prog_gm = center_prog_grid_mean(state)
    aux_gm = center_aux_grid_mean(state)
    u_gm_surf = prog_gm.u[kc_surf]
    v_gm_surf = prog_gm.v[kc_surf]
    q_tot_gm_surf = prog_gm.q_tot[kc_surf]
    θ_liq_ice_gm_surf = prog_gm.θ_liq_ice[kc_surf]
    T_gm_surf = aux_gm.T[kc_surf]
    Pg = surf.ref_params.Pg

    pvg = TD.saturation_vapor_pressure(param_set, TD.PhaseEquil, surf.Tsurface)
    surf.qsurface = TD.q_vap_saturation_from_density(param_set, surf.Tsurface, ρ0_f_surf, pvg)
    lv = TD.latent_heat_vapor(param_set, T_gm_surf)

    phase_part = TD.PhasePartition(surf.qsurface, 0.0, 0.0)
    h_star = TD.liquid_ice_pottemp_given_pressure(param_set, surf.Tsurface, Pg, phase_part)

    ts_g = thermo_state_pθq(param_set, Pg, h_star, surf.qsurface)
    θ_virt_g = TD.virtual_pottemp(ts_g)

    ts_b = thermo_state_pθq(param_set, p0_c_surf, θ_liq_ice_gm_surf, surf.qsurface)
    θ_virt_b = TD.virtual_pottemp(ts_b)

    surf.windspeed = sqrt(u_gm_surf^2 + v_gm_surf^2)
    Nb2 = g / θ_virt_g * (θ_virt_b - θ_virt_g) / zb
    Ri = Nb2 * zb * zb / (surf.windspeed * surf.windspeed)

    surf.cm, surf.ch, surf.obukhov_length = exchange_coefficients_byun(param_set, Ri, zb, surf.zrough)
    surf.rho_uflux = -surf.cm * surf.windspeed * u_gm_surf * ρ0_f_surf
    surf.rho_vflux = -surf.cm * surf.windspeed * v_gm_surf * ρ0_f_surf

    surf.rho_hflux = -surf.ch * surf.windspeed * (θ_liq_ice_gm_surf - h_star) * ρ0_f_surf
    surf.rho_qtflux = -surf.ch * surf.windspeed * (q_tot_gm_surf - surf.qsurface) * ρ0_f_surf
    surf.lhf = lv * 0.0

    ts = thermo_state_pθq(param_set, p0_f_surf, surf.Tsurface, surf.qsurface)
    surf.shf = TD.cp_m(ts) * surf.rho_hflux

    surf.bflux = buoyancy_flux(param_set, surf.shf, surf.lhf, surf.Tsurface, surf.qsurface, α0_f_surf, ts)
    surf.ustar = friction_velocity_given_windspeed(surf.cm, surf.windspeed)

    return
end

function update(surf::SurfaceBase{SurfaceSullivanPatton}, grid, state, gm::GridMeanVariables, param_set)
    g = CPP.grav(param_set)
    R_d = CPP.R_d(param_set)
    kc_surf = kc_surface(grid)
    kf_surf = kf_surface(grid)

    zb = grid.zc[kc_surf]

    p0_f_surf = face_ref_state(state).p0[kf_surf]
    p0_c_surf = center_ref_state(state).p0[kc_surf]
    ρ0_f_surf = face_ref_state(state).ρ0[kf_surf]
    α0_c_surf = center_ref_state(state).α0[kc_surf]
    u_gm_surf = prog_gm.u[kc_surf]
    v_gm_surf = prog_gm.v[kc_surf]
    q_tot_gm_surf = prog_gm.q_tot[kc_surf]
    θ_liq_ice_gm_surf = prog_gm.θ_liq_ice[kc_surf]
    T_gm_surf = aux_gm.T[kc_surf]
    Pg = surf.ref_params.Pg

    ts = thermo_state_pθq(param_set, p0_f_surf, surf.Tsurface, surf.qsurface)
    lv = TD.latent_heat_vapor(param_set, T_gm_surf)
    T0 = p0_c_surf * α0_c_surf / R_d # TODO: can we use a thermo state here?

    θ_flux = 0.24
    phase_part = TD.PhasePartition(q_tot_gm_surf, 0.0, 0.0)
    Π = TD.exner_given_pressure(param_set, p0_f_surf, phase_part)
    surf.bflux = g * θ_flux * Π / T0

    pvg = TD.saturation_vapor_pressure(param_set, TD.PhaseEquil, surf.Tsurface)
    surf.qsurface = TD.q_vap_saturation_from_density(param_set, surf.Tsurface, ρ0_f_surf, pvg)
    h_star = TD.liquid_ice_pottemp_given_pressure(param_set, surf.Tsurface, Pg, phase_part)

    ts_g = thermo_state_pθq(param_set, Pg, h_star, surf.qsurface)
    θ_virt_g = TD.virtual_pottemp(ts_g)

    ts_b = thermo_state_pθq(param_set, p0_c_surf, θ_liq_ice_gm_surf, surf.qsurface)
    θ_virt_b = TD.virtual_pottemp(ts_b)

    surf.windspeed = sqrt(u_gm_surf^2 + v_gm_surf^2)
    Nb2 = g / θ_virt_g * (θ_virt_b - θ_virt_g) / zb
    Ri = Nb2 * zb * zb / (surf.windspeed * surf.windspeed)

    surf.cm, surf.ch, surf.obukhov_length = exchange_coefficients_byun(param_set, Ri, zb, surf.zrough)
    surf.rho_uflux = -surf.cm * surf.windspeed * u_gm_surf * ρ0_f_surf
    surf.rho_vflux = -surf.cm * surf.windspeed * v_gm_surf * ρ0_f_surf

    surf.rho_hflux = -surf.ch * surf.windspeed * (θ_liq_ice_gm_surf - h_star) * ρ0_f_surf
    surf.rho_qtflux = -surf.ch * surf.windspeed * (q_tot_gm_surf - surf.qsurface) * ρ0_f_surf
    surf.lhf = lv * surf.rho_qtflux
    surf.shf = TD.cp_m(ts) * surf.rho_hflux

    surf.ustar = friction_velocity_given_windspeed(surf.cm, surf.windspeed)
    return
end


import Base.length
import Base.reshape
import AxisArrays.AxisArray
import JuMP.value
using AxisArrays
using JuMP

function emptySetException(sets, values, floatbool=false)
    try
        return parameter(sets, values)
    catch
        if floatbool
            return Float64[]
        else
            return Int64[]
        end
    end
end

#TODO Get rid of union types
struct Parameter
     Tech::Array{String,1}
     Load::Array{String,1}
     TechClass::Array{String,1}
     TechIsGrid::AxisArray{Int64,1,Array{Int64,1},Tuple{Axis{:row,Array{String,1}}}}
     TechToLoadMatrix::AxisArray{Int64,2,Array{Int64,2},Tuple{Axis{:row,Array{String,1}},Axis{:col,Array{String,1}}}}
     TurbineDerate::AxisArray{Float64,1,Array{Float64,1},Tuple{Axis{:row,Array{String,1}}}}
     TechToTechClassMatrix::AxisArray{Int64,2,Array{Int64,2},Tuple{Axis{:row,Array{String,1}},Axis{:col,Array{String,1}}}}
     NMILRegime::Array{String,1}
     r_tax_owner::Float64
     r_tax_offtaker::Float64
     pwf_om::Float64
     pwf_e::Float64
     pwf_prod_incent::AxisArray{Float64,1,Array{Float64,1},Tuple{Axis{:row,Array{String,1}}}}
     LevelizationFactor::AxisArray{Float64,1,Array{Float64,1},Tuple{Axis{:row,Array{String,1}}}}
     LevelizationFactorProdIncent::AxisArray{Float64,1,Array{Float64,1},Tuple{Axis{:row,Array{String,1}}}}
     StorageCostPerKW::Float64
     StorageCostPerKWH::Float64
     OMperUnitSize::AxisArray{Float64,1,Array{Float64,1},Tuple{Axis{:row,Array{String,1}}}}
     CapCostSlope::Union{AxisArray{Float64,2,Array{Float64,2},Tuple{Axis{:row,Array{String,1}},Axis{:col,UnitRange{Int64}}}},AxisArray{Int64,2,Array{Int64,2},Tuple{Axis{:row,Array{String,1}},Axis{:col,UnitRange{Int64}}}}}
     CapCostYInt::Union{AxisArray{Float64,2,Array{Float64,2},Tuple{Axis{:row,Array{String,1}},Axis{:col,UnitRange{Int64}}}},AxisArray{Int64,2,Array{Int64,2},Tuple{Axis{:row,Array{String,1}},Axis{:col,UnitRange{Int64}}}}}
     CapCostX::AxisArray{Float64,2,Array{Float64,2},Tuple{Axis{:row,Array{String,1}},Axis{:col,UnitRange{Int64}}}}
     ProdIncentRate::AxisArray{Float64,2,Array{Float64,2},Tuple{Axis{:row,Array{String,1}},Axis{:col,Array{String,1}}}}
     MaxProdIncent::AxisArray{Float64,1,Array{Float64,1},Tuple{Axis{:row,Array{String,1}}}}
     MaxSizeForProdIncent::AxisArray{Float64,1,Array{Float64,1},Tuple{Axis{:row,Array{String,1}}}}
     two_party_factor::Float64
     analysis_years::Int64
     AnnualElecLoad::Float64
     LoadProfile::AxisArray{Float64,2,Array{Float64,2},Tuple{Axis{:row,Array{String,1}},Axis{:col,UnitRange{Int64}}}}
     ProdFactor::AxisArray{Float64,3,Array{Float64,3},Tuple{Axis{:row,Array{String,1}},Axis{:col,Array{String,1}},Axis{:page,UnitRange{Int64}}}}
     StorageMinChargePcent::Float64
     EtaStorIn::AxisArray{Float64,2,Array{Float64,2},Tuple{Axis{:row,Array{String,1}},Axis{:col,Array{String,1}}}}
     EtaStorOut::AxisArray{Float64,1,Array{Float64,1},Tuple{Axis{:row,Array{String,1}}}}
     InitSOC::Float64
     MaxSize::AxisArray{Float64,1,Array{Float64,1},Tuple{Axis{:row,Array{String,1}}}}
     MinStorageSizeKW::Float64
     MaxStorageSizeKW::Float64
     MinStorageSizeKWH::Float64
     MaxStorageSizeKWH::Float64
     TechClassMinSize::AxisArray{Float64,1,Array{Float64,1},Tuple{Axis{:row,Array{String,1}}}}
     MinTurndown::AxisArray{Float64,1,Array{Float64,1},Tuple{Axis{:row,Array{String,1}}}}
     FuelRate::AxisArray{Float64,3,Array{Float64,3},Tuple{Axis{:row,Array{String,1}},Axis{:col,UnitRange{Int64}},Axis{:page,UnitRange{Int64}}}}
     FuelAvail::AxisArray{Float64,2,Array{Float64,2},Tuple{Axis{:row,Array{String,1}},Axis{:col,UnitRange{Int64}}}}
     FixedMonthlyCharge::Float64
     AnnualMinCharge::Float64
     MonthlyMinCharge::Float64
     ExportRates::AxisArray{Float64,3,Array{Float64,3},Tuple{Axis{:row,Array{String,1}},Axis{:col,Array{String,1}},Axis{:page,UnitRange{Int64}}}}
     TimeStepRatchetsMonth::AxisArray{Array{Int64,1},1,Array{Array{Int64,1},1},Tuple{Axis{:row,UnitRange{Int64}}}}
     DemandRatesMonth::AxisArray{Float64,2,Array{Float64,2},Tuple{Axis{:row,UnitRange{Int64}},Axis{:col,UnitRange{Int64}}}}
     DemandLookbackPercent::Float64
     MaxDemandInTier::Array{Float64,1}
     MaxDemandMonthsInTier::Array{Float64,1}
     MaxUsageInTier::Array{Float64,1}
     FuelBurnRateM::AxisArray{Float64,3,Array{Float64,3},Tuple{Axis{:row,Array{String,1}},Axis{:col,Array{String,1}},Axis{:page,UnitRange{Int64}}}}
     FuelBurnRateB::AxisArray{Float64,3,Array{Float64,3},Tuple{Axis{:row,Array{String,1}},Axis{:col,Array{String,1}},Axis{:page,UnitRange{Int64}}}}
     NMILLimits::AxisArray{Float64,1,Array{Float64,1},Tuple{Axis{:row,Array{String,1}}}}
     TechToNMILMapping::AxisArray{Int64,2,Array{Int64,2},Tuple{Axis{:row,Array{String,1}},Axis{:col,Array{String,1}}}}
     DemandRates::Union{Array{Float64,1},AxisArray{Any,2,Array{Any,2},Tuple{Axis{:row,UnitRange{Int64}},Axis{:col,UnitRange{Int64}}}},AxisArray{Float64,2,Array{Float64,2},Tuple{Axis{:row,UnitRange{Int64}},Axis{:col,UnitRange{Int64}}}}}
     TimeStepRatchets::Union{Array{Int64,1},AxisArray{Array{Any,1},1,Array{Array{Any,1},1},Tuple{Axis{:row,UnitRange{Int64}}}},AxisArray{Array{Int64,1},1,Array{Array{Int64,1},1},Tuple{Axis{:row,UnitRange{Int64}}}}}
     DemandLookbackMonths::Array{Any,1}
     CapCostSegCount::Int64
     FuelBinCount::Int64
     DemandBinCount ::Int64
     DemandMonthsBinCount::Int64
     TimeStepCount::Int64
     Seg::UnitRange{Int64}
     Points::UnitRange{Int64}
     Month::UnitRange{Int64}
     Ratchets::UnitRange{Int64}
     FuelBin::UnitRange{Int64}
     DemandBin::UnitRange{Int64}
     DemandMonthsBin::UnitRange{Int64}
     TimeStep::UnitRange{Int64}
     TimeStepBat::UnitRange{Int64}
     TimeStepScaling::Float64
     OMcostPerUnitProd::AxisArray{Float64,1,Array{Float64,1},Tuple{Axis{:row,Array{String,1}}}}
end

function build_param(args...;
          Tech,
          Load,
          TechClass,
          TechIsGrid,
          TechToLoadMatrix,
          TurbineDerate,
          TechToTechClassMatrix,
          NMILRegime,
          r_tax_owner,
          r_tax_offtaker,
          pwf_om,
          pwf_e,
          pwf_prod_incent,
          LevelizationFactor,
          LevelizationFactorProdIncent,
          StorageCostPerKW,
          StorageCostPerKWH,
          OMperUnitSize,
          CapCostSlope,
          CapCostYInt,
          CapCostX,
          ProdIncentRate,
          MaxProdIncent,
          MaxSizeForProdIncent,
          two_party_factor,
          analysis_years,
          AnnualElecLoad,
          LoadProfile,
          ProdFactor,
          StorageMinChargePcent,
          EtaStorIn,
          EtaStorOut,
          InitSOC,
          MaxSize,
          MinStorageSizeKW,
          MaxStorageSizeKW,
          MinStorageSizeKWH,
          MaxStorageSizeKWH,
          TechClassMinSize,
          MinTurndown,
          FuelRate,
          FuelAvail,
          FixedMonthlyCharge,
          AnnualMinCharge,
          MonthlyMinCharge,
          ExportRates,
          TimeStepRatchetsMonth,
          DemandRatesMonth,
          DemandLookbackPercent,
          MaxDemandInTier,
          MaxDemandMonthsInTier,
          MaxUsageInTier,
          FuelBurnRateM,
          FuelBurnRateB,
          NMILLimits,
          TechToNMILMapping,
          DemandRates,
          TimeStepRatchets,
          DemandLookbackMonths,
          CapCostSegCount,
          FuelBinCount,
          DemandBinCount ,
          DemandMonthsBinCount,
          TimeStepCount,
          NumRatchets,
          TimeStepScaling,
          OMcostPerUnitProd,
          kwargs...
    )

    Seg = 1:CapCostSegCount
    Points = 0:CapCostSegCount
    Month = 1:12
    Ratchets = 1:NumRatchets
    FuelBin = 1:FuelBinCount
    DemandBin = 1:DemandBinCount
    DemandMonthsBin = 1:DemandMonthsBinCount
    TimeStep=1:TimeStepCount
    TimeStepBat=0:TimeStepCount

    TechIsGrid = parameter(Tech, TechIsGrid)
    TechToLoadMatrix = parameter((Tech, Load), TechToLoadMatrix)
    TurbineDerate = parameter(Tech, TurbineDerate)
    TechToTechClassMatrix = parameter((Tech, TechClass), TechToTechClassMatrix)
    pwf_prod_incent = parameter(Tech, pwf_prod_incent)
    LevelizationFactor = parameter(Tech, LevelizationFactor)
    LevelizationFactorProdIncent = parameter(Tech, LevelizationFactorProdIncent)
    OMperUnitSize = parameter(Tech, OMperUnitSize)
    CapCostSlope = parameter((Tech, Seg), CapCostSlope)
    CapCostYInt = parameter((Tech, Seg), CapCostYInt)
    CapCostX = parameter((Tech, Points), CapCostX)
    ProdIncentRate = parameter((Tech, Load), ProdIncentRate)
    MaxProdIncent = parameter(Tech, MaxProdIncent)
    MaxSizeForProdIncent = parameter(Tech, MaxSizeForProdIncent)
    LoadProfile = parameter((Load, TimeStep), LoadProfile)
    ProdFactor = parameter((Tech, Load, TimeStep), ProdFactor)
    EtaStorIn = parameter((Tech, Load), EtaStorIn)
    EtaStorOut = parameter(Load, EtaStorOut)
    MaxSize = parameter(Tech, MaxSize)
    TechClassMinSize = parameter(TechClass, TechClassMinSize)
    MinTurndown = parameter(Tech, MinTurndown)
    TimeStepRatchets = emptySetException(Ratchets, TimeStepRatchets)
    DemandRates = emptySetException((Ratchets, DemandBin), DemandRates, true)
    FuelRate = parameter((Tech, FuelBin, TimeStep), FuelRate)
    FuelAvail = parameter((Tech, FuelBin), FuelAvail)
    ExportRates = parameter((Tech, Load, TimeStep), ExportRates)
    TimeStepRatchetsMonth = parameter(Month, TimeStepRatchetsMonth)
    DemandRatesMonth = parameter((Month, DemandMonthsBin), DemandRatesMonth)
    MaxDemandInTier = parameter(DemandBin, MaxDemandInTier)
    MaxDemandMonthsInTier = parameter(DemandMonthsBin, MaxDemandMonthsInTier)
    MaxUsageInTier = parameter(FuelBin, MaxUsageInTier)
    FuelBurnRateM = parameter((Tech, Load, FuelBin), FuelBurnRateM)
    FuelBurnRateB = parameter((Tech, Load, FuelBin), FuelBurnRateB)
    NMILLimits = parameter(NMILRegime, NMILLimits)
    TechToNMILMapping = parameter((Tech, NMILRegime), TechToNMILMapping)
    OMcostPerUnitProd = parameter(Tech, OMcostPerUnitProd)

    param = Parameter(Tech,
                      Load,
                      TechClass,
                      TechIsGrid,
                      TechToLoadMatrix,
                      TurbineDerate,
                      TechToTechClassMatrix,
                      NMILRegime,
                      r_tax_owner,
                      r_tax_offtaker,
                      pwf_om,
                      pwf_e,
                      pwf_prod_incent,
                      LevelizationFactor,
                      LevelizationFactorProdIncent,
                      StorageCostPerKW,
                      StorageCostPerKWH,
                      OMperUnitSize,
                      CapCostSlope,
                      CapCostYInt,
                      CapCostX,
                      ProdIncentRate,
                      MaxProdIncent,
                      MaxSizeForProdIncent,
                      two_party_factor,
                      analysis_years,
                      AnnualElecLoad,
                      LoadProfile,
                      ProdFactor,
                      StorageMinChargePcent,
                      EtaStorIn,
                      EtaStorOut,
                      InitSOC,
                      MaxSize,
                      MinStorageSizeKW,
                      MaxStorageSizeKW,
                      MinStorageSizeKWH,
                      MaxStorageSizeKWH,
                      TechClassMinSize,
                      MinTurndown,
                      FuelRate,
                      FuelAvail,
                      FixedMonthlyCharge,
                      AnnualMinCharge,
                      MonthlyMinCharge,
                      ExportRates,
                      TimeStepRatchetsMonth,
                      DemandRatesMonth,
                      DemandLookbackPercent,
                      MaxDemandInTier,
                      MaxDemandMonthsInTier,
                      MaxUsageInTier,
                      FuelBurnRateM,
                      FuelBurnRateB,
                      NMILLimits,
                      TechToNMILMapping,
                      DemandRates,
                      TimeStepRatchets,
                      DemandLookbackMonths,
                      CapCostSegCount,
                      FuelBinCount,
                      DemandBinCount ,
                      DemandMonthsBinCount,
                      TimeStepCount,
                      Seg,
                      Points,
                      Month,
                      Ratchets,
                      FuelBin,
                      DemandBin,
                      DemandMonthsBin,
                      TimeStep,
                      TimeStepBat,
                      TimeStepScaling,
                      OMcostPerUnitProd)

    return param

end

# Code for paramter() function
function paramDataFormatter(setTup::Tuple, data::AbstractArray)
    reverseTupleAxis = Tuple([length(set) for set in setTup][end:-1:1])
    shapedData = reshape(data, reverseTupleAxis)
    reverseDataAxis = [length(setTup)+1 - n for n in 1:length(setTup)]
    shapedDataT = permutedims(shapedData, reverseDataAxis)
    return AxisArray(shapedDataT, setTup)
end

function parameter(setTup::Tuple, data::AbstractArray)
    #data = retype(nondata)
    try
        formattedParam = paramDataFormatter(setTup, data)
        return formattedParam
    catch
        correctLength = prod([length(x) for x in setTup])
        if length(data) < correctLength
            let x = 1
                for set in setTup
                    x = x * length(set)
                end
                numZeros = x - length(data)
                for zero in 1:numZeros
                    append!(data, 0)
                end
                formattedParam = paramDataFormatter(setTup, data)
                return formattedParam
            end
        else
            data = data[1:correctLength]
            formattedParam = paramDataFormatter(setTup, data)
            return formattedParam
        end
    end
end

function parameter(set::UnitRange{Int64}, data::Float64)
    return [data]
end

function parameter(setTup::Tuple{Array{Symbol,1}, UnitRange{Int64}}, data::Number)
    newTup = ([setTup[1][1], :FAKE], 1:2)
    return AxisArray(fill(data, 2, 2), newTup)
end

function parameter(set::Symbol, data::Int64)
    return AxisArray([data], set)
end

function parameter(set, data)
    shapedData = reshape(data, length(set))
    return AxisArray(shapedData, set)
end

# Additional dispatches to make things easier
function length(::Symbol)
    return 1
end

function reshape(data::Number, axes::Int64)
    return data
end

function AxisArray(data::Number, index::Array{Symbol, 1})
    return AxisArray([float(data)], index)
end

function JuMP.value(::Val{false})
    return 0.0
end

function JuMP.value(x::Float64)
    return x
end


function get_irnss_orbital_elements(obj::IRNSSPropagator)
    return jcall(obj, "getIRNSSOrbitalElements", IRNSSOrbitalElements, ())
end


#generate amino acid alignments
function simAlignments(numFiles,numSeqs,parentdir)
	infiles = Vector{String}(undef, numFiles)
	for f in 1:numFiles
		file, io = mktemp(parentdir)
    lenSeq = rand(100:1000)
		infiles[f] = file
    w = FASTX.FASTA.Writer(io)
		for n in 1:numSeqs
			seq = BioSequences.randaaseq(lenSeq) # all seqs need to have same length
			rec = FASTX.FASTA.Record(string(n),seq)
			write(w, rec)
		end
    close(io)
	end
	return(infiles)
end


const TEST_FLOAT32 = false
const TEST_DOUBLE64 = false
const TEST_STATIC_SIZED = false

using Manifolds
using ManifoldsBase
using ManifoldsBase: number_of_coordinates

using LinearAlgebra
using Distributions
using DoubleFloats
using ForwardDiff
using Quaternions
using Random
using ReverseDiff
using StaticArrays
using Statistics
using StatsBase
using Test
using LightGraphs
using SimpleWeightedGraphs

function include_test(path)
    @info "Testing $path"
    @time include(path)  # show basic timing, (this will print a newline at end)
end

function our_base_ambiguities()
    ambigs = Test.detect_ambiguities(Base)
    modules_we_care_about =
        [Base, LinearAlgebra, Manifolds, ManifoldsBase, StaticArrays, Statistics, StatsBase]
    our_ambigs = filter(ambigs) do (m1, m2)
        we_care = m1.module in modules_we_care_about && m2.module in modules_we_care_about
        return we_care && (m1.module === Manifolds || m2.module === Manifolds)
    end
    return our_ambigs
end


#-------------------------------------------------------------------
#* EMSO Model Library (EML) Copyright (C) 2004 - 2007 ALSOC.
#*
#* This LIBRARY is free software; you can distribute it and/or modify
#* it under the therms of the ALSOC FREE LICENSE as available at
#* http://www.enq.ufrgs.br/alsoc.
#*
#* EMSO Copyright (C) 2004 - 2007 ALSOC, original code
#* from http://www.rps.eng.br Copyright (C) 2002-2004.
#* All rights reserved.
#*
#* EMSO is distributed under the therms of the ALSOC LICENSE as
#* available at http://www.enq.ufrgs.br/alsoc.
#*--------------------------------------------------------------------
#* Author: Gerson Balbueno Bicca
#* $Id: DoublePipeIncr.mso								$
#*------------------------------------------------------------------
type Properties_Wall
	Properties_Wall()=begin
		N=outers.N
		new(
			DanaInteger(Dict{Symbol,Any}(
				:Brief=>"Number of zones",
				:Default=>2
			)),
			fill(viscosity(Dict{Symbol,Any}(
				:Brief=>"Stream Incremental Viscosity",
				:Default=>1,
				:Lower=>1e-5,
				:Upper=>1e5,
				:Symbol=>"\\mu"
			)),(N)),
			fill(temperature(Dict{Symbol,Any}(
				:Brief=>"Incremental Wall Temperature",
				:Lower=>50
			)),(N)),
			[:N,],
			[:Mu,:Twall,]
		)
	end
	N::DanaInteger
	Mu::Array{viscosity}
	Twall::Array{temperature}
	parameters::Array{Symbol,1}
	variables::Array{Symbol,1}
	attributes::Dict{Symbol,Any}
end
export Properties_Wall
function atributes(in::Properties_Wall,_::Dict{Symbol,Any})
	fields::Dict{Symbol,Any}=Dict{Symbol,Any}()
	fields[:Pallete]=false
	fields[:Brief]="Incremental Physical properties of the streams at wall temperature."
	fields[:Info]="to be documented."
	drive!(fields,_)
	return fields
end
Properties_Wall(_::Dict{Symbol,Any})=begin
	newModel=Properties_Wall()
	newModel.attributes=atributes(newModel,_)
	newModel
end


using LinearAlgebra

function plus(a, b)
	return a + b
end

minus(a, b) = a - b

# TODO: Overloading is not implemented in the core yet, this should be reviewed or
# implemented ad-hoc in Julia loader when resolving the calls in the invoke
function plus(a)
	return a+1
end

function mul(a,b)
	return a*b
end


module ITensorParallel

using Distributed
using MPI
using ITensors

import ITensors: product, position!, noiseterm, lproj, rproj

include("default_in_partition.jl")
include("opsum_sum.jl")
include("threaded_projmposum.jl")
include("distributed_projmposum.jl")
include("mpi_projmposum.jl")

export ThreadedProjMPOSum, opsum_sum, DistributedSum, MPISum

end


using Weber
using Base.Test
include("find_timing.jl")

const check_timing = get(ENV,"WEBER_TIMING_TESTS","Yes") != "No"
const moment_eps = 2.5e-3

# warm up JIT...
find_timing() do
  addtrial(moment(50ms,() -> record(:a)),
           moment(50ms,() -> record(:b)) >> moment(100ms,() -> record(:d)),
           moment(100ms,() -> record(:c)),
           moment(100ms,() -> record(:e)))
end

comp_events,comp_times,_ = find_timing() do
  addtrial(moment(50ms,() -> record(:a)),
           moment(50ms,() -> record(:b)) >> moment(100ms,() -> record(:d)),
           moment(100ms,() -> record(:c)),
           moment(100ms,() -> record(:e)))
end

@testset "Compound Moments" begin
  @test comp_events == [:a,:b,:c,:d,:e]
  if check_timing
    comp_diff = maximum(diff(comp_times) - 0.05)
    @test comp_diff < moment_eps
  end
end


using Luxor, Colors
using Printf
using JSON
using ArgParse

include("strange.jl")

function parse_commandline()
    settings = ArgParseSettings()

    @add_arg_table settings begin
        "--infile"
            help = "parameter input filename"
            arg_type = String
            default = "params.json"
        "--outfile"
            help = "strange image output filename"
            default = "regenerate.png"
	"--n"
	    help = "generate n variations of random colors on input file"
	    default = 1
	    arg_type = Int
	"--red"
	    help = "red midpoint"
	    default = 1.0
	    arg_type = Float64
	"--blue"
	    help = "blue midpoint"
	    default = 1.0
	    arg_type = Float64
	"--green"
	    help = "green midpoint"
	    default = 1.0
	    arg_type = Float64
	"--redvar"
	    help = "red midpoint"
	    default = .50
	    arg_type = Float64
	"--bluevar"
	    help = "blue midpoint"
	    default = .50
	    arg_type = Float64
	"--greenvar"
	    help = "green midpoint"
	    default = .50
	    arg_type = Float64
    end

    return parse_args(settings)
end

# generate n png images

function main()
	parsed_args = parse_commandline()

	infile = parsed_args["infile"]
	outfile = parsed_args["outfile"]

	n = parsed_args["n"]

	red_range = parsed_args["red"]
	green_range = parsed_args["green"]
	blue_range = parsed_args["blue"]

	red_var = parsed_args["redvar"]
	green_var = parsed_args["greenvar"]
	blue_var = parsed_args["bluevar"]

	println("input ctrl file: $infile")

	sp = JSON.parsefile(infile)

	ctrl_params = Dict([
			    :a => sp["ctrl_params"]["a"],
			    :b => sp["ctrl_params"]["b"],
			    :c => sp["ctrl_params"]["c"],
			    :d => sp["ctrl_params"]["d"],
			    :e => sp["ctrl_params"]["e"]])

	dot_size = get(sp, "dot_size", .50)
	res = get(sp, "res", 1000)

	color_profile = Dict([
			      :r => sp["color_profile"]["r"],
			      :g => sp["color_profile"]["g"],
			      :b => sp["color_profile"]["b"]])
	flip = get(sp, "flip", false)
	cmode = get(sp, "cmode", "rgb")

	params = @sprintf("Params: a=%-2.16f, b=%-2.16f, c=%-2.16f, d=%-2.16f, e=%-2.16f, res=%-2.2d, dotsize = %-2.16f flip = %d, cmode = %s\n",
			  ctrl_params[:a],
			  ctrl_params[:b],
			  ctrl_params[:c],
			  ctrl_params[:d],
			  ctrl_params[:e],
			  res,
			  dot_size,
			  flip,
			  cmode)
	println("$params")

	if n > 1

		# override cmode if generating more than one pic
		cmode = "rgb"
		for i in 1:n

			imagefile = @sprintf("%s_%-2.2d.png", outfile, i)
			jsonfile = @sprintf("%s_%-2.2d.json", outfile, i)

			println("outputfile: $imagefile")
			color_profile = Dict([
					      :r => red_range + rand()*rs()*red_range*red_var,
					      :g => green_range + rand()*rs()*green_range*green_var,
					      :b => blue_range + rand()*rs()*blue_range*blue_var
					     ])

			cf = open(jsonfile, "w")
                	sset = StrangeSet(ctrl_params, dot_size, res, color_profile, imagefile, flip, cmode)
                	JSON.print(cf, JSON.parse(JSON.json(sset)), 4)
                	close(cf)

			strange(imagefile, flip, cmode, dot_size, res, ctrl_params, color_profile)
		end
	else
		println("outputfile: $outfile")
		strange(outfile, flip, cmode, dot_size, res, ctrl_params, color_profile)
	end
	println("finished generating $outfile")
end

main()


mutable struct MCVBCallback <: AbstractCallback
  every::Int64
  dkl::Float64
  aval::Float64
  # biasing observables
  Afunc::Function
  #Bfunc::Function
  # force Malliavin weights
  mvy::Array{Float64}
  mvydot::Array{Float64}
  gradf::Array{Float64}
  # value baseline Malliavin weights
  mvz::Array{Float64}
  mvzdot::Array{Float64}
  gradv::Array{Float64}
  # value baseline function
  vbl::AbstractValueBaseline
  # extra memory for computing some stuff
  em::Array{Float64}
  # IO stuff for writing things to files
  #avgsio::Union{IO,nothing}
  function MCVBCallback(Afunc::Function,model::AbstractPotential,vbl::AbstractValueBaseline)
    mvy::Array{Float64} = zeros(Float64,gradshape(model))
    mvydot::Array{Float64} = zeros(Float64,jacshape(model))
    gradf::Array{Float64} = zeros(Float64,gradshape(model))
    mvz::Array{Float64} = zeros(Float64,gradshape(vbl))
    mvzdot::Array{Float64} = zeros(Float64,gradshape(vbl))
    gradv::Array{Float64} = zeros(Float64,gradshape(vbl))
    em = zeros(Float64,getdimensionality(model))
    new(1,0.0,0.0,Afunc,mvy,mvydot,gradf,mvz,mvzdot,gradv,vbl,em)
  end
end

"""
Function that zeros all the quantities for restarting trajectories
"""
function initialize!(cb::MCVBCallback)
  cb.mvy .= 0.0
  cb.mvydot .= 0.0
  cb.mvz .= 0.0
  cb.mvzdot .= 0.0
end

function initializeavgs!(cb::MCVBCallback)
  cb.dkl = 0.0
  cb.aval = 0.0
  cb.gradf .= 0.0
  cb.gradv .= 0.0
end

function initializeall!(cb::MCVBCallback)
  cb.dkl = 0.0
  cb.aval = 0.0
  cb.mvy .= 0.0
  cb.mvydot .= 0.0
  cb.gradf .= 0.0
  cb.mvz .= 0.0
  cb.mvzdot .= 0.0
  cb.gradv .= 0.0
end

"""
Function that averages all the quantities by the number of trajectories
"""
function average!(ntraj::Int64,cb::MCVBCallback)
  cb.dkl /= ntraj
  cb.aval /= ntraj
  cb.gradf /= ntraj
  cb.gradv /= ntraj
end

"""
Implementation of the Monte Carlo Value Baseline algorithm
"""
function callback(cb::MCVBCallback,system::AbstractThermostattedSystem,mm::MixedModel,args...)
  # extract some arguments
  ind::Int64 = args[1]
  dt::Float64 = args[2]

  # compute gradient of variable force and update ydot
  jacobian!(cb.mvydot,system,mm.potentials[length(mm.potentials)])
  cb.mvydot .*= (sqrt(dt)/system.thermostat.scale)
  cb.mvy .+= (cb.mvydot*system.thermostat.rands)

  # calculate value baseline function
  vval::Float64 = callvbl(system,cb.vbl)

  # compute gradient of value baseline
  gradient!(cb.mvzdot,system,cb.vbl)
  cb.mvz .+= (dt .* cb.mvzdot)

  ### compute the instantaneous return
  # action difference - positive term
  cb.em .= ((system.thermostat.scale/sqrt(dt)) .* system.thermostat.rands).^2
  # action difference - negative term
  cb.em .-= (mm.potentials[length(mm.potentials)].f .+ (system.thermostat.scale/sqrt(dt)) .* system.thermostat.rands).^2
  # compute return
  rval::Float64 = sum(cb.em) / (2*system.thermostat.scale^2)

  # add return to KL divergence
  cb.dkl += rval*dt

  # observable bias
  aval::Float64 = cb.Afunc(system,mm,ind,dt)
  rval += aval# + Bfunc(system,mm,ind,dt)
  cb.aval += aval

  # update the gradients of the force parameters and the value baseline parameters
  cb.gradf .+= (rval .* cb.mvy) .- ((vval/dt) .* (cb.mvydot*system.thermostat.rands))
  cb.gradv .+= (rval .* cb.mvz) .- (vval .* cb.mvzdot)
end

#"""
#Implementation of the Monte Carlo Value Baseline algorithm for an active particle simulation
#"""
#function callback(cb::MCVBCallback,system::AbstractActiveSystem,mm::MixedModel,args...)
#  # extract some arguments
#  ind::Int64 = args[1]
#  dt::Float64 = args[2]
#
#  # compute gradient of variable force and update ydot
#  jacobian!(cb.mvydot,system,mm.potentials[length(mm.potentials)])
#  for i in 1:system.dim
#    @. @views cb.mvydot[:,i] .*= (sqrt(dt) / system.thermostat.scale[i])
#  end
#  cb.mvy .+= (cb.mvydot*system.thermostat.rands)
#
#  # calculate value baseline function
#  vval::Float64 = callvbl(system,cb.vbl)
#
#  # compute gradient of value baseline
#  gradient!(cb.mvzdot,system,cb.vbl)
#  cb.mvz .+= (dt .* cb.mvzdot)
#
#  ### compute the instantaneous return
#  # action difference - positive term
#  #cb.em .= ((system.thermostat.scale/sqrt(dt)) .* system.thermostat.rands).^2
#  # action difference - negative term
#  cb.em .= -(mm.potentials[length(mm.potentials)].f .+ (system.thermostat.scale/sqrt(dt)) .* system.thermostat.rands).^2
#  #cb.em .-= (mm.potentials[length(mm.potentials)].f .+ (system.thermostat.scale/sqrt(dt)) .* system.thermostat.rands).^2
#  # compute return
#  rval::Float64 = sum(cb.em ./ (2 .* system.thermostat.scale.^2))
#
#  # add return to KL divergence
#  cb.dkl += rval*dt
#
#  # observable bias
#  aval::Float64 = cb.Afunc(system,mm,ind,dt)
#  rval += aval# + Bfunc(system,mm,ind,dt)
#  cb.aval += aval
#
#  # update the gradients of the force parameters and the value baseline parameters
#  cb.gradf .+= (rval .* cb.mvy) .- ((vval/dt) .* (cb.mvydot*system.thermostat.rands))
#  cb.gradv .+= (rval .* cb.mvz) .- (vval .* cb.mvzdot)
#
#end
#


module BSR

using MATLAB

export
    gpb,
    gpb_owt_ucm,
    gpb_owt_ucm_strength

include("matutil.jl")
include("bsrfun.jl")

end


# Fix ambiguities on julia 0.4

*(a::ResElem{fmpz}, b::fmpz) = parent(a)(data(a) * b)

*(a::fmpz, b::ResElem{fmpz}) = b*a

+(a::ResElem{fmpz}, b::fmpz) = parent(a)(data(a) + b)

+(a::fmpz, b::ResElem{fmpz}) = b + a

-(a::ResElem{fmpz}, b::fmpz) = parent(a)(data(a) - b)

-(a::fmpz, b::ResElem{fmpz}) = parent(b)(a - data(b))

function ==(a::ResElem{fmpz}, b::fmpz)
   z = base_ring(a)(b)
   return data(a) == mod(z, modulus(a))
end

==(a::fmpz, b::ResElem{fmpz}) = b == a

#

*(::fmpz, ::PolyElem{fmpz}) = nothing

*(::PolyElem{fmpz}, ::fmpz) = nothing

+(::fmpz, ::PolyElem{fmpz}) = nothing

+(::PolyElem{fmpz}, ::fmpz) = nothing

-(::fmpz, ::PolyElem{fmpz}) = nothing

-(::PolyElem{fmpz}, ::fmpz) = nothing

==(::fmpz, ::PolyElem{fmpz}) = nothing

==(::PolyElem{fmpz}, ::fmpz) = nothing

divexact(::PolyElem{fmpz}, ::fmpz) = nothing

evaluate(::PolyElem{fmpz}, ::fmpz) = nothing

#

*(::fmpz, ::SeriesElem{fmpz}) = nothing

*(::SeriesElem{fmpz}, ::fmpz) = nothing

+(::fmpz, ::SeriesElem{fmpz}) = nothing

+(::SeriesElem{fmpz}, ::fmpz) = nothing

-(::fmpz, ::SeriesElem{fmpz}) = nothing

-(::SeriesElem{fmpz}, ::fmpz) = nothing

==(::fmpz, ::SeriesElem{fmpz}) = nothing

==(::SeriesElem{fmpz}, ::fmpz) = nothing

*(::fmpz, ::RelSeriesElem{fmpz}) = nothing

*(::RelSeriesElem{fmpz}, ::fmpz) = nothing

+(::fmpz, ::RelSeriesElem{fmpz}) = nothing

+(::RelSeriesElem{fmpz}, ::fmpz) = nothing

-(::fmpz, ::RelSeriesElem{fmpz}) = nothing

-(::RelSeriesElem{fmpz}, ::fmpz) = nothing

==(::fmpz, ::RelSeriesElem{fmpz}) = nothing

==(::RelSeriesElem{fmpz}, ::fmpz) = nothing

*(::fmpz, ::AbsSeriesElem{fmpz}) = nothing

*(::AbsSeriesElem{fmpz}, ::fmpz) = nothing

+(::fmpz, ::AbsSeriesElem{fmpz}) = nothing

+(::AbsSeriesElem{fmpz}, ::fmpz) = nothing

-(::fmpz, ::AbsSeriesElem{fmpz}) = nothing

-(::AbsSeriesElem{fmpz}, ::fmpz) = nothing

==(::fmpz, ::AbsSeriesElem{fmpz}) = nothing

==(::AbsSeriesElem{fmpz}, ::fmpz) = nothing

*(::fmpz, ::MatElem{fmpz}) = nothing

*(::MatElem{fmpz}, ::fmpz) = nothing

+(::fmpz, ::MatElem{fmpz}) = nothing

+(::MatElem{fmpz}, ::fmpz) = nothing

-(::fmpz, ::MatElem{fmpz}) = nothing

-(::MatElem{fmpz}, ::fmpz) = nothing

==(::MatElem{fmpz}, ::fmpz) = nothing

divexact(::MatElem{fmpz}, ::fmpz) = nothing

#

setindex_t!(a::nmod_mat, b::GenRes{fmpz}, i::Int, j::Int) = setindex_!(a, data(b), i, j)

*(::FracElem{fmpz}, ::fmpz) = nothing

*(::fmpz, ::FracElem{fmpz}) = nothing

+(::FracElem{fmpz}, ::fmpz) = nothing

+(::fmpz, ::FracElem{fmpz}) = nothing

-(::FracElem{fmpz}, ::fmpz) = nothing

-(::fmpz, ::FracElem{fmpz}) = nothing

==(::FracElem{fmpz}, ::fmpz) = nothing

==(::fmpz, ::FracElem{fmpz}) = nothing

divexact(::FracElem{fmpz}, ::fmpz) = nothing

divexact(::fmpz, ::FracElem{fmpz}) = nothing


#####
##### Boundary conditions on pressure are derived from boundary conditions
##### on the north-south horizontal velocity, v.
#####

# Pressure boundary conditions are either zero flux (Neumann) or Periodic.
PressureBC(::BC)  = BoundaryCondition(Flux, nothing)
PressureBC(::PBC) = PeriodicBC()

function PressureBoundaryConditions(vbcs)
    x = CoordinateBoundaryConditions(PressureBC(vbcs.x.left), PressureBC(vbcs.x.right))
    y = CoordinateBoundaryConditions(PressureBC(vbcs.y.left), PressureBC(vbcs.y.right))
    z = CoordinateBoundaryConditions(PressureBC(vbcs.z.left), PressureBC(vbcs.z.right))
    return (x=x, y=y, z=z)
end


"""
    Repo{T}

one-dimensional array, containing classifiable elements of type T.

Each element of Repo can be assigned to a class, identified by an integer.
All elements are intially in class one. Importantly, the time needed for
accessing elements, reassigning an element to a different class and
counting the number of elements is independent of the number of elements in the
class. This can be used for example to write highly-performant event-driven
simulations.

# Example
```jldoctest
julia> using Repos

julia> r=Repo(["Apple","Carrot","Cherry","Pepper"],2)
Repository of 4 objects in 2 classes

julia> setclass!(r,1,2)

julia> setclass!(r,3,2)

julia> print_repo(r)
Repository of 2 classes
  Class 1
    1 - 4: Pepper
    2 - 2: Carrot
  Class 2
    1 - 3: Cherry
    2 - 1: Apple

julia> fruits=class(r,2)
Class of 2 objects

julia> print_repo(fruits)
Class of 2 objects
    1 - 3: Cherry
    2 - 1: Apple
```
"""
mutable struct Repo{T} <: AbstractVector{T}
    starts::Array{Int,1}
    counts::Array{Int,1}
    ids::Array{Int,1}
    pos::Array{Int,1}
    cargo::Array{T,1}
    N::Int
    C::Int
end

"""
    RepoClass{T}

A class of elements in a Repo structure.

The RepoClass is an interface for the Repo structure. This means that instances
of RepoClass automatically update if the corresponding Repo is changed.

# Example
```jldoctest
julia> using Repos

julia> r=Repo(["Peter","Paul","Mary"],2)
Repository of 3 objects in 2 classes

julia> infected=class(r,2)
Class of 0 objects

julia> setclass!(r,1,2)

julia> infected
Class of 1 objects

julia> print_repo(infected)
Class of 1 objects
    1 - 1: Peter
```
"""
struct RepoClass{T} <: AbstractVector{T}
    repo::Repo{T}
    c::Int
end


function tcm_estimateForest_RTS_kcv_weighted(N, K_sample, d_list, nfolds, transaction_folds, transactions, time_limit_EM, perTol_EM, initial_left, initial_right, initial_product, initial_isLeaf)
	unique_fold_ids = unique(transaction_folds)

	KL_total_holdout_fold = zeros(length(d_list), nfolds)
	noinf_count_holdout_fold = zeros(length(d_list), nfolds)

	start_time = time()

	for fold_ind in 1:nfolds
		fold = unique_fold_ids[fold_ind]

		@show fold

		@show unique_fold_ids

		println("Pre-subsetting ")

		#@show transaction_folds
		#temp = find(transaction_folds .!= fold)
		#@show temp
		@show size(transaction_folds)
		@show size(transactions)
		@show fold

		temp = find(transaction_folds .!= fold)
		@show temp[1:5]
		trainfolds_transactions = transactions[ find(transaction_folds .!= fold), :]
		println("Post-subsetting")
		testfold_transactions = transactions[ find(transaction_folds .== fold), :]

		trainfolds_assortments, trainfolds_transaction_counts = tcm_transactionsToCounts(N, trainfolds_transactions)
		testfold_assortments, testfold_transaction_counts = tcm_transactionsToCounts(N, testfold_transactions)

		for i in 1:length(d_list)
			println("Depth ", d_list[i], ", fold ", fold)
			depth_limit = d_list[i]

			if (isempty(initial_left))
				lambda, forest_left, forest_right, forest_product, forest_isLeaf, loglik, elapsed_time = tcm_estimateForest_RTS(N, trainfolds_assortments, trainfolds_transaction_counts, K_sample, depth_limit, perTol_EM, time_limit_EM)

			else
				lambda, forest_left, forest_right, forest_product, forest_isLeaf, loglik, elapsed_time = tcm_estimateForest_RTS_withInitial(N, trainfolds_assortments, trainfolds_transaction_counts, K_sample, depth_limit, perTol_EM, time_limit_EM,
																																initial_left, initial_right, initial_product, initial_isLeaf)
			end

			K = length(lambda)
			testfold_forest_predict, testfold_forest_predict_by_class = tcm_predictForest(N, K, lambda, forest_left, forest_right, forest_product, forest_isLeaf, testfold_assortments)

			testfold_KL, testfold_noinf_count, testfold_KL_avg = tcm_evaluateKL(N, testfold_forest_predict, testfold_transaction_counts)

			KL_total_holdout_fold[i, fold_ind] = testfold_KL #testfold_KL / sum(testfold_transaction_counts)
			noinf_count_holdout_fold[i, fold_ind] = testfold_noinf_count
		end
	end

	end_time = time()
	elapsed_time = end_time - start_time

	KL_avg_by_d = sum(KL_total_holdout_fold, 2) ./ sum(noinf_count_holdout_fold, 2)  * 100

	@show KL_total_holdout_fold
	@show noinf_count_holdout_fold
	@show KL_avg_by_d

	best_i = indmin(KL_avg_by_d)
	best_d = d_list[best_i]
	best_KL_avg_cv = KL_avg_by_d[best_i]

	return best_d, best_KL_avg_cv, elapsed_time

end

function tcm_estimateForest_RTS_kcv_weighted_ass(N, K_sample, d_list, nfolds, assortment_folds, assortments, transaction_counts, time_limit_EM, perTol_EM, initial_left, initial_right, initial_product, initial_isLeaf)
	unique_fold_ids = unique(assortment_folds)

	KL_total_holdout_fold = zeros(length(d_list), nfolds)
	noinf_count_holdout_fold = zeros(length(d_list), nfolds)

	start_time = time()

	for fold_ind in 1:nfolds
		fold = unique_fold_ids[fold_ind]

		# trainfolds_assortments, trainfolds_transaction_counts = tcm_transactionsToCounts(N, trainfolds_transactions)
		# testfold_assortments, testfold_transaction_counts = tcm_transactionsToCounts(N, testfold_transactions)

		trainfolds_assortments = assortments[ find(assortment_folds .!= fold), :]
		testfold_assortments = assortments[ find(assortment_folds .== fold), :]

		trainfolds_transaction_counts = transaction_counts[ find(assortment_folds .!= fold), :]
		testfold_transaction_counts = transaction_counts[ find(assortment_folds .== fold), :]

		for i in 1:length(d_list)
			println("Depth ", d_list[i], ", fold ", fold)
			depth_limit = d_list[i]

			if (isempty(initial_left))
				lambda, forest_left, forest_right, forest_product, forest_isLeaf, loglik, elapsed_time = tcm_estimateForest_RTS(N, trainfolds_assortments, trainfolds_transaction_counts, K_sample, depth_limit, perTol_EM, time_limit_EM)

			else
				lambda, forest_left, forest_right, forest_product, forest_isLeaf, loglik, elapsed_time = tcm_estimateForest_RTS_withInitial(N, trainfolds_assortments, trainfolds_transaction_counts, K_sample, depth_limit, perTol_EM, time_limit_EM,
																																initial_left, initial_right, initial_product, initial_isLeaf)
			end

			K = length(lambda)
			testfold_forest_predict, testfold_forest_predict_by_class = tcm_predictForest(N, K, lambda, forest_left, forest_right, forest_product, forest_isLeaf, testfold_assortments)

			testfold_KL, testfold_noinf_count, testfold_KL_avg = tcm_evaluateKL(N, testfold_forest_predict, testfold_transaction_counts)

			KL_total_holdout_fold[i, fold_ind] = testfold_KL #testfold_KL / sum(testfold_transaction_counts)
			noinf_count_holdout_fold[i, fold_ind] = testfold_noinf_count
		end
	end

	end_time = time()
	elapsed_time = end_time - start_time

	KL_avg_by_d = sum(KL_total_holdout_fold, 2) ./ sum(noinf_count_holdout_fold, 2)  * 100

	best_i = indmin(KL_avg_by_d)
	best_d = d_list[best_i]
	best_KL_avg_cv = KL_avg_by_d[best_i]

	return best_d, best_KL_avg_cv, elapsed_time

end

function tcm_estimateForest_RTS_RMredo_kcv_weighted_ass(N, K_sample, d_list, nfolds, assortment_folds, assortments, transaction_counts, time_limit, perTol, perTol_EM)
	unique_fold_ids = unique(assortment_folds)

	KL_total_holdout_fold = zeros(length(d_list), nfolds)
	noinf_count_holdout_fold = zeros(length(d_list), nfolds)

	start_time = time()

	for fold_ind in 1:nfolds
		fold = unique_fold_ids[fold_ind]

		# trainfolds_assortments, trainfolds_transaction_counts = tcm_transactionsToCounts(N, trainfolds_transactions)
		# testfold_assortments, testfold_transaction_counts = tcm_transactionsToCounts(N, testfold_transactions)

		trainfolds_assortments = assortments[ find(assortment_folds .!= fold), :]
		testfold_assortments = assortments[ find(assortment_folds .== fold), :]

		trainfolds_transaction_counts = transaction_counts[ find(assortment_folds .!= fold), :]
		testfold_transaction_counts = transaction_counts[ find(assortment_folds .== fold), :]

		for i in 1:length(d_list)
			println("Depth ", d_list[i], ", fold ", fold)
			depth_limit = d_list[i]

			ranking_lambda, orderings, ranking_loglik, ranking_elapsed_time = tcm_estimateRanking(N, trainfolds_assortments, trainfolds_transaction_counts, time_limit, perTol)
			initial_left, initial_right, initial_product, initial_isLeaf = tcm_convertRankingToForest(N, orderings)

			lambda, forest_left, forest_right, forest_product, forest_isLeaf, loglik, elapsed_time = tcm_estimateForest_RTS_withInitial(N, trainfolds_assortments, trainfolds_transaction_counts, K_sample, depth_limit, perTol_EM, time_limit,
																															initial_left, initial_right, initial_product, initial_isLeaf)

			K = length(lambda)
			testfold_forest_predict, testfold_forest_predict_by_class = tcm_predictForest(N, K, lambda, forest_left, forest_right, forest_product, forest_isLeaf, testfold_assortments)

			testfold_KL, testfold_noinf_count, testfold_KL_avg = tcm_evaluateKL(N, testfold_forest_predict, testfold_transaction_counts)

			KL_total_holdout_fold[i, fold_ind] = testfold_KL #testfold_KL / sum(testfold_transaction_counts)
			noinf_count_holdout_fold[i, fold_ind] = testfold_noinf_count
		end
	end

	end_time = time()
	elapsed_time = end_time - start_time

	KL_avg_by_d = sum(KL_total_holdout_fold, 2) ./ sum(noinf_count_holdout_fold, 2)  * 100

	best_i = indmin(KL_avg_by_d)
	best_d = d_list[best_i]
	best_KL_avg_cv = KL_avg_by_d[best_i]

	return best_d, best_KL_avg_cv, elapsed_time

end


using ITensors
using ITensorInfiniteMPS
using ITensorInfiniteMPS.HDF5
using Test

@testset "HDF5 Read and Write" begin
  function space_shifted(::Model"ising", q̃sz)
    return [QN("SzParity", 1 - q̃sz, 2) => 1, QN("SzParity", 0 - q̃sz, 2) => 1]
  end

  N = 2
  model = Model"ising"()
  space_ = fill(space_shifted(model, 0), N)
  s = infsiteinds("S=1/2", N; space=space_)
  initstate(n) = "↑"
  ψ = InfMPS(s, initstate)

  @testset "InfiniteCanonicalMPS" begin
    fo = h5open("data.h5", "w")
    write(fo, "ψ", ψ)
    close(fo)

    fi = h5open("data.h5", "r")
    ψr = read(fi, "ψ", InfiniteCanonicalMPS)
    close(fi)

    @test ψ.AL.data == ψr.AL.data
    @test ψ.C.data == ψr.C.data
    @test ψ.AR.data == ψr.AR.data
  end

  # Clean up the test hdf5 file
  rm("data.h5"; force=true)
end


@testset "Receiver" begin

    receiver = Receiver(
        4e6Hz,
        intermediate_frequency = 0.0Hz,
        gain_phase_mism_crosstalk = SMatrix{4,4,ComplexF64}(I),
        attitude = RotXYZ(0.0, 0.0, 0.0),
        n0 = 1/Hz
    )
    next_receiver = GNSSSimulator.propagate(receiver, 1000, Random.GLOBAL_RNG)
    @test get_gain_phase_mism_crosstalk(next_receiver) == SMatrix{4,4}(I)
    @test get_attitude(next_receiver) == RotXYZ(0.0, 0.0, 0.0)
    @test get_noise_density(next_receiver) == 1/Hz
    @test get_sampling_frequency(next_receiver) == 4e6Hz
    @test get_intermediate_frequency(next_receiver) == 0.0Hz
    @test get_gain_phase_mism_crosstalk(next_receiver) == SMatrix{4,4}(I)
    @test @inferred(get_noise_std(next_receiver)) == sqrt(1/Hz * 4e6Hz)

    receiver = Receiver(
        4e6Hz,
        intermediate_frequency = 0.0Hz,
        gain_phase_mism_crosstalk = SMatrix{4,4,ComplexF64}(I),
        attitude = RotXYZ(0.0, 0.0, 0.0),
        n0 = 1/Hz
    )
    @test get_gain_phase_mism_crosstalk(receiver) == SMatrix{4,4}(I)
    @test get_attitude(receiver) == RotXYZ(0.0, 0.0, 0.0)
    @test get_noise_density(receiver) == 1/Hz
    @test get_sampling_frequency(receiver) == 4e6Hz
    @test get_intermediate_frequency(receiver) == 0.0Hz
    @test get_gain_phase_mism_crosstalk(receiver) == SMatrix{4,4}(I)
end


#
# FIXME this is obsolete and superseded by
# `vs_[xx]_tests.jl` direct tests against online calculators.
#
using Test
using ScottishTaxBenefitModel
using .ModelHousehold: Household, Person, People_Dict, is_single,
    default_bu_allocation, get_benefit_units, get_head, get_spouse, search,
    pers_is_disabled, pers_is_carer
using .ExampleHouseholdGetter
using .Definitions
using .LegacyMeansTestedBenefits:
    calc_legacy_means_tested_benefits!, tariff_income,
    LMTResults, is_working_hours, make_lmt_benefit_applicability, calc_premia,
    working_disabled, calc_allowances,
    apply_2_child_policy, calc_incomes, calc_NDDs

using .Intermediate: MTIntermediate, make_intermediate

using .STBParameters: LegacyMeansTestedBenefitSystem, IncomeRules, HoursLimits
using .Results: LMTResults, LMTCanApplyFor, init_household_result

using .ExampleHelpers

using Dates
using DataFrames
using CSV

## FIXME don't need both
lmt = LegacyMeansTestedBenefitSystem{Float64}()
sys = get_system( scotland=true )

"""
Extract a household from
"""
function spreadsheet_ss_example( key :: AbstractString ) :: NamedTuple
    re = r"([A-Z0-9a-z\-]+)_([0-9]+)k"
    m = match( re, key )
    fam = m[1]
    n = m[2]
    examples = get_all_examples()
    name = ""
    subtype = ""
    m2 = match( r"(.*)-(.*)", fam )
    if m2 !== nothing
        fam = m2[1]
        subtype = m2[2]
    end
    hh = nothing
    rent = 600/4
    ct = 123.25/4
    earn = (1000*parse( Float64, n ))/52.0
    earn = round(earn, digits=2)

    if fam == "B"
        name = "Basic Case"
        hh = examples[cpl_w_2_children_hh]
        bu = get_benefit_units(hh)[1]
        spouse = get_spouse( bu )
        head = get_head( bu )
        head.age = 40
        head.usual_hours_worked = 40
        head.income[wages]=earn

    elseif fam == "K3"
        name = "3 Kids"
        hh = examples[cpl_w_2_children_hh]
    elseif fam == "SP"
        name = "Single Parent"
        hh = examples[single_parent_hh]
    elseif fam == "SE2"
        name = "Single Earner"
        hh = examples[cpl_w_2_children_hh]
    elseif fam == "2C"
        name = "Basic; 2 child limit test"
        hh = examples[cpl_w_2_children_hh]
    elseif fam == "2E"
        name = "2 Earner"
        hh = examples[cpl_w_2_children_hh]
    elseif fam == "CC"
        name = "Child Care"
        hh = examples[cpl_w_2_children_hh]
    elseif fam == "DC"
        name = "Disabled Child"
        hh = examples[cpl_w_2_children_hh]
    elseif fam == "DA"
        name = "Disabled Adult"
        hh = examples[cpl_w_2_children_hh]
    elseif fam == "CL"
        name = "Childless"
        hh = examples[childless_couple_hh]
    else
        error( "unknown key $fam " )
    end
    name = "$name :: $(earn)p.w."
    hh.gross_rent = rent
    hh.council_tax = ct
    return ( name=name, hh=hh )
end

@testset "Complete SS Tests - Legacy System" begin
    sys = get_system( scotland = true )
    df=CSV.File( "$(@__DIR__)/../docs/uc_test_cases_main_results_transposed.csv")|>DataFrame
    keys = split("B_7k,B_4k,B_12k,B_17k,B_22k,B_30k,B_50k,K3_7k,K3_12k,K3_12k,SP_7k,SP_12k,SP_30k,2C-a_7k,2C-b_7k,2C-c_7k,2C-c_22k,SE2_17k,2E_17k,2E_2k,SP-CC_7k,SP-CC_17k,SP-CC_22k,DC-a_7k,DC-b_7k,DC-b_22k,DA-a_7k,DA-b_7k,DA-b_22k,CL_7k,CL_12k,CL_22k", "," )
    for k in keys
        println( "on key $k ")
        name,hh = spreadsheet_ss_example( k )
        println( "name=$name" )
        res = df[df.Key.==k,:][1,:]
        println( res.CTB_NEW )
        hhres = init_household_result( hh )
        intermed = make_intermediate(
            hh,
            sys.hours_limits,
            sys.age_limits,
            sys.child_limits )
        calc_legacy_means_tested_benefits!(
            hhres,
            hh,
            intermed,
            sys.lmt,
            sys.age_limits,
            sys.hours_limits,
            sys.nmt_bens,
            sys.hr )
    end
end


using RecipesBase
#using Plots
#gr()

using Mapper2.MapperGraphs
#pyplot()

################################################################################
# Post routing plotting.
################################################################################
struct Polygon
    x::Vector{Float64}
    y::Vector{Float64}
    fill::Symbol
end

# ------- #
# Methods #
# ------- #

# Given a box, give X and Y vectors to trace the outline of the box.
getx(d::Polygon) = d.x
gety(d::Polygon) = d.y

# Encoding for a route through the architecture.
struct DrawRoute
    x       ::Vector{Float64}
    y       ::Vector{Float64}
    color   ::Symbol
end

@recipe function f(map::Map; plot_route = true)
    # Unpack map
    #map = r.map
    #plot_route = r.plot_route

    # Set up parameters
    spacing = 10
    tilesize = 20

    # Build boxes and routes
    boxes = getboxes(map)
    routes = plot_route ? getroutes(map) : getlines(map)

    # Set up plot attributes
    legend := false
    ticks  := nothing
    grid   := false
    yflip  := true

    # Plot boxes
    seriestype := :shape

    for box in boxes
        @series begin
            #linecolor := :black
            c := box.fill

            # Get x,y coordinates from box
            x = getx(box)
            y = gety(box)
            x, y
        end
    end

    seriestype := :path
    linewidth  := 2

    # Plot routes
    for route in routes
        @series begin
            linecolor := route.color
            x = route.x
            y = route.y
            x,y
        end
    end
end

function getboxes(m::Map{2})
    a = m.toplevel
    style = a.metadata["style"]

    # Create draw boxes for each tile in the array.
    boxes = Polygon[]
    for (name, child) in a.children
        addr = getaddress(a, name)
        # scale x,y
        if haskey(child.metadata, "shadow_offset")
            addrs = [addr + o for o in child.metadata["shadow_offset"]]
            push!(addrs, addr)
        else
            addrs = [addr]
        end

        # Unpack tuple after manipulation
        y, x = cartesian(style, dim_min(addrs))

        # fill with cyan if box address is used.
        if MapperCore.isused(m, addr)
            fill = :cyan
        else
            fill = :white
        end
        push!(boxes, Polygon(polygon(style, x, y)..., fill))
    end
    return boxes
end

function iscongested(edge::Mapper2.TaskgraphEdge)
    metadata = edge.metadata
    return get(metadata, "iscongested", false)
end

function getroutes(m::Map{2})
    a = m.toplevel
    style = a.metadata["style"]

    routes = DrawRoute[]
    for (edge_index, graph) in enumerate(m.mapping.edges)
        x = Float64[]
        y = Float64[]
        for path in Mapper2.MapperGraphs.linearize(graph)
            # Only look at global port paths.
            isglobalport(path) || continue
            # Get the address from the path.
            address = getaddress(a, path)

            # Create offsets for smooth paths
            metadata = a[path].metadata
            y0, x0 = cartesian(style, address)

            push!(x, x0 + get(metadata, "x", 0.5))
            push!(y, y0 + get(metadata, "y", 0.5))
        end
        # Choose color based on length of path
        edge = getedge(m.taskgraph, edge_index)

        if iscongested(edge)
            color = :red
        elseif length(x) <= 2
            color = :black
        elseif length(x) <= 5
            color = :blue
        else
            color = :green
        end
        # Add this route to the routes vector
        push!(routes, DrawRoute(x, y, color))
    end
    return routes
end

function getlines(m::Map{2})
    a = m.toplevel
    lines = DrawRoute[]
    for edge in getedges(m.taskgraph)

        x = Float64[]
        y = Float64[]
        source = first(getsources(edge))
        dest   = first(getsinks(edge))

        for node in (source, dest)
            # Get the address from the path.
            path = Mapper2.MapperCore.getpath(m.mapping, node)
            address = getaddress(a, path)
            # Create offsets for smooth paths

            # Big offset for macro location in the whole array
            x_offset_big = address[2]*(spacing + tilesize)
            y_offset_big = address[1]*(spacing + tilesize)
            # Small offset for offset within a tile
            x_offset_small = get(a[path].metadata, "x", 0.5) * tilesize
            y_offset_small = get(a[path].metadata, "y", 0.5) * tilesize

            push!(x, x_offset_big + x_offset_small)
            push!(y, y_offset_big + y_offset_small)
        end
        # Default to black color
        color = :black
        # Add this route to the routes vector
        push!(lines, DrawRoute(x,y,color))
    end
    return lines
end


#### Variable Mapper

"""
    get_var_mapper(vars)

Get a `NamedTuple` that maps a variable name
and subdomain ID to a unique index. Example:
```
julia> vars = ((:ρ_0, 1), (:w, 3), (:a, 3), (:α_0, 1));

julia> var_mapper = get_var_mapper(vars)
(ρ_0 = (1,), w = (2, 3, 4), a = (5, 6, 7), α_0 = (8,))
```
"""
function get_var_mapper(vars, domain_set::DomainSet)
    var_names = tuple([v for (v, dss) in vars]...)
    n_sd_per_var = [sum(domain_set, dss) for (v, dss) in vars]
    end_index, n_sd_max = cumsum(n_sd_per_var), max(n_sd_per_var...)
    start_index = [1, [1 + x for x in end_index][1:(end - 1)]...]
    vals = Tuple([Tuple(a:b) for (a, b) in zip(start_index, end_index)])
    var_mapper = NamedTuple{var_names}(vals)
    D = Dict([v => dss for (v, dss) in vars]...)
    dss_per_var = NamedTuple{Tuple(keys(D))}(Tuple(values(D)))
    return var_mapper, dss_per_var, var_names
end

function get_sd_mapped(vars, idx::DomainIdx, domain_set::DomainSet)
    sd_mapped = Dict()
    al = alldomains(idx)
    for (v, dss) in vars
        idx_ss = DomainIdx(domain_set, dss)
        if !(haskey(sd_mapped, v))
            sd_mapped[v] = Int[]
        end
        for i in al
            i == gridmean(idx) && push!(sd_mapped[v], gridmean(idx_ss))
            i == environment(idx) && push!(sd_mapped[v], environment(idx_ss))
            if i in updraft(idx) && !(i in sd_mapped[v])
                push!(sd_mapped[v], updraft(idx_ss)...)
            end
        end
    end
    return sd_mapped
end

function get_sv_a_map(idx::DomainIdx, idx_ss::DomainIdx)
    a = zeros(Int, length(alldomains(idx)))
    for i in alldomains(idx)
        if i ≠ 0 # protect against null map
            if i == gridmean(idx) && has_gridmean(idx_ss)
                a[i] = gridmean(idx_ss)
            elseif i == environment(idx) && has_environment(idx_ss)
                a[i] = environment(idx_ss)
            elseif i in updraft(idx) && has_updraft(idx_ss)
                a[i] = updraft(idx_ss)[i]
            else
                a[i] = 0
            end
        end
    end
    return a
end

function get_sd_unmapped(vars, idx::DomainIdx, domain_set::DomainSet)
    sd_unmapped = Dict{Symbol, Vector{Int}}()
    al = alldomains(idx)
    for (v, dss) in vars
        idx_ss = DomainIdx(domain_set, dss)
        if !(haskey(sd_unmapped, v))
            sd_unmapped[v] = Int[]
        end
        for i in al
            i == gridmean(idx_ss) && push!(sd_unmapped[v], gridmean(idx))
            i == environment(idx_ss) && push!(sd_unmapped[v], environment(idx))
            if i in updraft(idx_ss) && !(i in sd_unmapped[v])
                push!(sd_unmapped[v], updraft(idx)...)
            end
        end
    end
    return sd_unmapped
end


module ProfileAligner

  export AlignmentMatrix, Profile,
         score, align, getstrings, scoreprofiles,
         measurequality, setScoringMatrix

  import DataReader.FastaRecord, DataReader.ScoreMatrix

  scoringMatrix = ScoreMatrix()

  setScoringMatrix(sm :: ScoreMatrix) = scoringMatrix = sm

  getScoringMatrixValue(matrix :: ScoreMatrix, i1 ::Int64, i2::Int64) = i1 > i2 ? matrix.hsh[i1][i2] : matrix.hsh[i2][i1]

  function getScoringMatrixValue(aa1 :: Char, aa2 :: Char)
    haskey(scoringMatrix.keys, aa1) && haskey(scoringMatrix.keys, aa2) &&
      return getScoringMatrixValue(scoringMatrix, scoringMatrix.keys[aa1], scoringMatrix.keys[aa2])
    aa1 == aa2 ? 1 : -1
  end

  debugprint(str :: Any) = 0
  #debugprintln(str :: Any) = println(str)

  immutable Profile{T}
    rawdata :: Array{Char, 2}
    data :: Array{Dict{Char, T}, 1}
    stringsize :: Int
    numberofstrings :: Int
    descriptions :: Array{ASCIIString, 1}

    #Profile(raw :: Array{Char, 2}, desc :: Array{ASCIIString, 1}) = getprofile{T}(raw, desc)
    #Profile(raw :: Array{Char, 2}, desc :: Array{String, 1}) = getprofile(raw, desc)
    Profile(str :: ASCIIString, desc :: ASCIIString = "") = Profile{T}( reshape([ letter for letter in str ], length(str), 1), [desc] )
    Profile(record :: FastaRecord) = Profile{T}(record.sequence, record.description)

    Profile(raw :: Array{Char, 2},
            d :: Array{Dict{Char, T}, 1},
            s1 :: Int,
            s2 :: Int,
            desc :: Array{ASCIIString, 1}) = new{T}(raw, d, s1, s2, desc)
    #function getprofile(raw :: Array{Char, 2}, descriptions :: Array{ASCIIString, 1})

    #function call{T}(::Type{SummedArray}, a::Vector{T})
    function  call{T}(::Type{Profile{T}}, raw :: Array{Char, 2}, descriptions :: Array{ASCIIString, 1} = [])
      rawdata = copy(raw)
      size_1 = size(raw, 1)
      size_2 = size(raw, 2)
      data = Array(Dict{Char, T}, size_1)
      for row in 1 : size_1
        data[row] = Dict{Char, T}()
        s = zero(T)
        for col in 1 : size_2
          letter = raw[row, col]
          data[row][letter] = get!(data[row], letter, zero(T)) + one(T)
          s += one(T)
        end
        if s > 0
          for (key, value) in data[row]
            data[row][key] /= s
          end
        end
      end
      Profile{T}(rawdata, data, size_1, size_2, descriptions)
    end
  end

  type AlignmentMatrix{T}
    matrix :: Array{T, 2}
    path :: Array{Char, 2}

    AlignmentMatrix(m :: Array{T, 2}, p :: Array{Char, 2}) = new(m, p)

    function call{T}(::Type{AlignmentMatrix{T}}, n :: Int64, m :: Int64)
      matrix = zero(Array(T, n, m))
      path = Array(Char, n, m)
      AlignmentMatrix{T}(matrix, path)
    end
  end

  #method returns pair of previous coordinates
  function getprev(direction :: Char, i :: Int64, j :: Int64)
    i < 2 && direction != 'R' && error("1st index is less than 2")
    j < 2 && direction != 'D' && error("2nd index is less than 2")
    direction == 'D' && return (i - 1, j)
    direction == 'R' && return (i, j - 1)
    direction == 'M' && return (i - 1, j - 1)
    error("unknown direction")
  end

  get{T}(profile::Profile{T}, i::Int64, aa :: Char) = Base.get(profile.data[i], aa, zero(T))

  aminoacids = "ARNDCQEGHILKMFPSTWYVBZX"

  get_ri{T}(profile::Profile{T}, i::Int64) = length(keys(profile.data[i]))

  get_nij{T}(profile::Profile{T}, i::Int64, aa::Char) = Base.get(profile.data[i], aa, 0.0)

  # 1. define weighting schemes
  function sequenceWeight1{T}(profile::Profile{T}, sequence::ASCIIString)
    sum([
      1.0/(get_ri(profile, i) * get_nij(profile, i, sequence[i]))
      for i in 1:length(sequence)
    ])
  end

  # 2. define scores
  function score2{T}(P::Profile{T}, Q::Profile{T}, i, j)
    sum([ get(P, i, aa) * get(Q, j, aa) for aa in aminoacids ] )
  end

  function score{T}(P::Profile{T}, Q::Profile{T}, i, j)
    sum([
          get(P, i, aa1) * get(Q, j, aa2) * getScoringMatrixValue(aa1, aa2)
            for aa1 in aminoacids, aa2 in aminoacids
        ] )
  end

  GAP_COST = -1.0

  function prepareAlignmentMatrix{T}(
                P :: Profile{T},
                Q :: Profile{T},
                matrixdata :: AlignmentMatrix{T}
                )
    n = P.stringsize
    m = Q.stringsize
    matrixdata.matrix[1, 1] = zero(T)
    matrixdata.path[1, 1] = 'U'
    for i = 1 : n
      matrixdata.matrix[i + 1, 1] = matrixdata.matrix[i, 1] + GAP_COST
      matrixdata.path[i + 1, 1] = 'D'
    end
    for j = 1 : m
      matrixdata.matrix[1, j + 1] = matrixdata.matrix[1, j] + GAP_COST
      matrixdata.path[1, j + 1] = 'R'
    end
  end

  function buildAlignmentMatrix{T}(
                  P::Profile{T},
                  Q :: Profile{T},
                  matrixdata :: AlignmentMatrix{T},
                  scoreFunc
                  )
    n = P.stringsize
    m = Q.stringsize
    for j = 1 : m
      for i = 1 : n
        # 1. Match
        matrixdata.matrix[i + 1, j + 1] = matrixdata.matrix[i, j] + scoreFunc(P, Q, i, j)
        matrixdata.path[i + 1, j + 1] = 'M'
        # 2. Down
        temp_value = matrixdata.matrix[i, j + 1] + GAP_COST
        if (matrixdata.matrix[i + 1, j + 1] < temp_value)
          matrixdata.matrix[i + 1, j + 1] = temp_value
          matrixdata.path[i + 1, j + 1] = 'D'
        end
        # 3. Right
        temp_value = matrixdata.matrix[i + 1, j] + GAP_COST
        if (matrixdata.matrix[i + 1, j + 1] < temp_value)
          matrixdata.matrix[i + 1, j + 1] = temp_value
          matrixdata.path[i + 1, j + 1] = 'R'
        end

      end
    end
  end

  #returns score from precomputed matrix and last column/row coordinates
  function getscore{T}(P::Profile{T}, Q :: Profile{T}, matrixdata :: AlignmentMatrix{T})
    n = P.stringsize
    m = Q.stringsize
    return (n + 1, m + 1, matrixdata.matrix[n + 1, m + 1])
  end

  function backtracing{T}(P :: Profile{T}, Q :: Profile{T},
                          matrixdata :: AlignmentMatrix{T},
                          lastrow :: Int64, lastcol :: Int64)
    debugprint(matrixdata.path)
    path = Tuple{Char, Int64, Int64}[]
    (direction, row, col) = (matrixdata.path[lastrow, lastcol], lastrow, lastcol)
    while (row > 1 || col > 1)
      push!(path, (direction, row - 1, col - 1))
      debugprint(matrixdata.path[row, col])
      (row, col) = getprev(direction, row, col)
      direction = matrixdata.path[row, col]
    end
    path
  end

  construct{T}(a1 :: Array{T}, a2 :: Array{T}) = hcat(a1, a2)

  function mixprofilecolumn{T}(P :: Profile{T}, Q :: Profile{T},
                                 direction :: Char, i :: Int64, j :: Int64)
    direction == 'M' && return construct(P.rawdata[i, 1:end], Q.rawdata[j, 1:end])
    direction == 'R' && return construct(reshape(['-' for k in 1 : P.numberofstrings], 1, P.numberofstrings),
              Q.rawdata[j, 1 : end])
    direction == 'D' && return construct(
              P.rawdata[i, 1 : end],
              reshape(['-' for k in 1 : Q.numberofstrings], 1, Q.numberofstrings)
              )
    error("unknown direction in mix profile column")
  end

  function mixprofiles{T}(P :: Profile{T}, Q :: Profile{T}, indices :: Vector{Tuple{Char, Int64, Int64}})

    newProfileSize = length(indices)
    tempMatrix = Array(Char, newProfileSize, P.numberofstrings + Q.numberofstrings)
    for index in  1 : newProfileSize
      newProfileColumn = mixprofilecolumn(P, Q,
          indices[index][1], indices[index][2], indices[index][3])
      for j in 1 : length(newProfileColumn)
        tempMatrix[newProfileSize - index + 1, j] = newProfileColumn[j]
      end
    end

    Profile{T}(tempMatrix, append!(P.descriptions, Q.descriptions))
  end

  #
  # main alignment function for profiles.
  # returns consensus profile + alignment score
  #
  function align{T}(P :: Profile{T}, Q :: Profile{T})
    debugprint(P)
    debugprint(Q)
    n = P.stringsize
    m = Q.stringsize
    matrix = AlignmentMatrix{T}(n + 1, m + 1)
    prepareAlignmentMatrix(P, Q, matrix)
    buildAlignmentMatrix(P, Q, matrix, score)
    (lastrow, lastcol, sc) = getscore(P, Q, matrix)
    debugprint(sc)

    pairsofindices = backtracing(P, Q, matrix, lastrow, lastcol) # this should return new Profile{T} with resulting alignment and score
    debugprint(pairsofindices)
    res = mixprofiles(P, Q, pairsofindices)
    return res
  end

  function scoreprofiles{T}(P :: Profile{T}, Q :: Profile{T})
    debugprint(P)
    debugprint(Q)
    n = P.stringsize
    m = Q.stringsize
    matrix = AlignmentMatrix{T}(n + 1, m + 1)
    prepareAlignmentMatrix(P, Q, matrix)
    buildAlignmentMatrix(P, Q, matrix, score)
    (lastrow, lastcol, sc) = getscore(P, Q, matrix)
    debugprint(sc)
    return sc
  end

  #
  function measurequality{T}(P :: Profile{T})
    sum([
      sum([- Base.get(P.data[i], aa, zero(T))*log2(Base.get(P.data[i], aa, one(T)))
        for aa in setdiff(keys(P.data[i]), '-')])
      for i in 1 : P.stringsize])
  end

  #
  # method should return FastaRecord array for given profile
  #
  getstrings{T}(P :: Profile{T}) =  [ FastaRecord(P.descriptions[i], ascii(P.rawdata[1:end, i])) for i in 1 : P.numberofstrings]

end


export weightedSample

function weightedSample(m::ConditionalModel, _data)
    return _weightedSample(getmoduletypencoding(m.model), m.model, m.args, _data)
end

export sourceWeightedSample

sourceWeightedSample(m::Model, data=NamedTuple()) = sourceWeightedSample(data)(m)

function sourceWeightedSample(_data)
    function(_m::Model)

        _datakeys = getntkeys(_data)
        proc(_m, st :: Assign)     = :($(st.x) = $(st.rhs))
        proc(_m, st :: Return)     = nothing
        proc(_m, st :: LineNumber) = nothing

        function proc(_m, st :: Sample)
            st.x ∈ _datakeys && return :(_ℓ += logdensity($(st.rhs), $(st.x)))
            return :($(st.x) = rand($(st.rhs)))
        end

        vals = map(x -> Expr(:(=), x,x),variables(_m))

        wrap(kernel) = @q begin
            _ℓ = 0.0
            $kernel

            return (_ℓ, $(Expr(:tuple, vals...)))
        end

        buildSource(_m, proc, wrap) |> MacroTools.flatten
    end
end

@gg function _weightedSample(M::Type{<:TypeLevel}, _m::Model, _args, _data)
    body = type2model(_m) |> sourceWeightedSample(_data) |> loadvals(_args, _data)
    @under_global from_type(_unwrap_type(M)) @q let M
        $body
    end
end