module System_parameters
    using Random
    export Params

    # Physical setting
    printlist_physical = ["L","β","NC","Nthermalization","Nsteps","initial","initialtrj","update_method","useOR","numOR","Nwing"]
    # Physical setting(fermions)
    printlist_fermions  = ["quench","Dirac_operator","Clover_coefficient","r","hop","Nf","mass",
                            "Domainwall_M","Domainwall_m","Domainwall_N5","Domainwall_ωs","Domainwall_b","Domainwall_c","Domainwall_r",
                            "BoundaryCondition",
                                "smearing_for_fermion","stout_numlayers","stout_ρ","stout_loops"]
    # System Control
    printlist_systemcontrol = ["log_dir","logfile","loadU_format","loadU_dir","loadU_fromfile","loadU_filename","saveU_dir","saveU_format",
                                "saveU_every","verboselevel","randomseed","measurement_basedir","measurement_dir","julian_random_number"]
    # HMC related
    printlist_HMCrelated = ["Δτ","SextonWeingargten","N_SextonWeingargten","MDsteps","eps","MaxCGstep"]
    # Action parameter for SLMC
    printlist_parameters_SLMC = ["βeff","firstlearn","use_autogeneratedstaples","couplingcoeff","couplinglist","coupling_loops"]
    # Measurement set
    printlist_measurement = ["measurement_methods"]

    const printlists = [printlist_physical,printlist_fermions,printlist_systemcontrol,printlist_HMCrelated,printlist_parameters_SLMC,printlist_measurement]
    const printlists_header = ["# Physical setting ","# Physical setting(fermions)","# System Control","# HMC related","# Action parameter for SLMC","# Measurement set"]

    defaultmeasures = Array{Dict,1}(undef,2)
    for i=1:length(defaultmeasures)
        defaultmeasures[i] = Dict()
    end
    defaultmeasures[1]["methodname"] = "Plaquette"
    defaultmeasures[1]["measure_every"] = 1
    defaultmeasures[1]["fermiontype"] = nothing
    defaultmeasures[2]["methodname"] = "Polyakov_loop"
    defaultmeasures[2]["measure_every"] = 1
    defaultmeasures[2]["fermiontype"] = nothing

    system = Dict()
    actions = Dict()
    md = Dict()
    cg = Dict()
    wilson = Dict()
    staggered = Dict()
    measurement = Dict()

    system["Dirac_operator"]  = "Wilson"
    system["quench"] = false
    system["initial"] = "cold"
    system["BoundaryCondition"]=[1,1,1,-1]
    system["Nwing"]= 1
    system["randomseed"] = 111
    #system["IntegratedHMC"] = false
    #system["Heatbath"] = false
    system["verboselevel"] = 2

    system["saveU_format"] = "JLD"
    system["saveU_every"] = 1
    system["saveU_dir"] = "./confs"

    system["initialtrj"] = 1

    #system["loadU_format"] = "JLD"
    #system["loadU_dir"] = "./confs"

    system["update_method"] = "HMC"

    actions["use_autogeneratedstaples"] = false
    #actions["couplinglist"] = []
    actions["couplingcoeff"] = []
    actions["coupling_loops"] = nothing

    md["Δτ"] = 0.05
    md["MDsteps"] = 20
    md["SextonWeingargten"] = false
    system["Nsteps"] = 100
    system["Nthermalization"] = 0

    #md["N_SextonWeingargten"]

    cg["eps"] = 1e-19
    cg["MaxCGstep"] = 3000

    #wilson["hop"]
    wilson["r"] = 1
    wilson["Clover_coefficient"] =  1.5612

    #staggered["mass"]
    staggered["Nf"] = 4

    measurement["measurement_methods"] = defaultmeasures

    system["julian_random_number"] = false

    mutable struct Params_set
        system::Dict
        actions::Dict
        md::Dict
        cg::Dict
        wilson::Dict
        staggered::Dict
        measurement::Dict

        function Params_set()
            return new(system,actions,md,cg,wilson,staggered,measurement)
        end

        function Params_set(system,actions,md,cg,wilson,staggered,measurement)
            return new(system,actions,md,cg,wilson,staggered,measurement)
        end

        function Params_set(system,actions,md,cg,measurement;wilson=Dict(),staggered=Dict())
            return new(system,actions,md,cg,wilson,staggered,measurement)
        end
    end

    function make_parameters(system,actions,md,cg,measurement;wilson=Dict(),staggered=Dict())
        return Params_set(system,actions,md,cg,measurement,wilson=wilson,staggered=staggered)
    end

    struct Params
        L::Tuple  # = (2,2,2,2) # Mandatory
        β::Float64 # = 6 # Mandatory
        NC::Int64 # = 3
        Dirac_operator::Union{Nothing,String} # = "Wilson"
        quench::Bool # = false
        initial::String # = "cold"
        BoundaryCondition::Array{Int8,1} # =[1,1,1,-1]
        Nwing::Int64  # = 1
        randomseed::Int64 # = 111

        #For actions
        use_autogeneratedstaples::Bool # = false
        couplinglist::Array{String,1} # = []
        couplingcoeff::Array{Float64,1} # = []
        coupling_loops::Union{Nothing,Array{Array{Array{Tuple{Int,Int},1},1},1}}

        #For MD
        Δτ::Float64 # = 0.05
        MDsteps::Int64 # = 20
        SextonWeingargten::Bool # = false
        Nsteps::Int64 # = 100
        Nthermalization::Int64 # = 10

        #For Sexton-Weingargten
        N_SextonWeingargten::Int64 # = 10 # Mandatory

        #For IntegratedHMC
        #IntegratedHMC::Bool # = false

        #For Heatbath
        #Heatbath::Bool # = false

        #For CG
        eps::Float64 # = 1e-19
        MaxCGstep::Int64 # = 3000

        #For Wilson

        hop::Float64 # = 0.141139#Hopping parameter # Mandatory
        r::Float64 # = 1 #Wilson term
        #For WilsonClover
        Clover_coefficient::Float64  # = 1.5612

        #For Staggered
        mass::Float64 # = 0.5 # Mandatory
        Nf::Int8 # = 4

        #verbose
        verboselevel::Int8 # = 2

        #For measurement
        measurement_methods::Array{Dict,1} # = defaultmeasures

        saveU_format::Union{String,Nothing}
        saveU_every::Int64
        saveU_dir::String

        loadU_format::Union{String,Nothing}
        loadU_dir::String

        update_method::String
        βeff::Union{Float64,Array{Float64,1}}
        firstlearn::Int64
        log_dir::String
        logfile::String
        load_fp::IOStream
        measuredir::String
        integratedFermionAction::Bool
        training_data_name::String
        useOR::Bool
        numOR::Int64
        initialtrj::Int64
        loadU_fromfile::Bool
        loadU_filename::String

        smearing_for_fermion::String
        stout_numlayers::Union{Nothing,Int64}
        stout_ρ::Union{Nothing,Array{Float64,1}}
        stout_loops::Union{Nothing,Array{String,1}}

        Domainwall_M::Union{Nothing,Float64}
        Domainwall_m::Union{Nothing,Float64}
        Domainwall_N5::Union{Nothing,Int64}
        Domainwall_b::Union{Nothing,Float64}
        Domainwall_c::Union{Nothing,Float64}
        Domainwall_ωs::Union{Nothing,Array{Float64,1}}
        Domainwall_r::Union{Nothing,Float64}

        julian_random_number::Bool

        function Params(system,actions,md,cg,wilson,staggered,measurement)
            #pnames = fieldnames(Params)
            #display(pnames)

            if haskey(system,"L")
                L = system["L"] #::Tuple  = (2,2,2,2) # Mandatory
            else
                error("system[\"L\"] should be set. eg. L = (4,4,4,4)")
            end

            if haskey(system,"β")
                β = system["β"] #::Float64 # = 6 # Mandatory
            else
                error("system[\"β\"] should be set. eg. β = 2.1")
            end

            NC = system["NC"] #::Int64 = 3
            Dirac_operator = system["Dirac_operator"] #::Union{Nothing,String} = "Wilson"
            quench = system["quench"] #::Bool = false
            initial = system["initial"] #::String = "cold"
            BoundaryCondition = system["BoundaryCondition"] #::Array{Int8,1}=[1,1,1,-1]
            Nwing = system["Nwing"] #::Int64 = 1
            randomseed = system["randomseed"] #::Int64 = 111

            use_autogeneratedstaples = actions["use_autogeneratedstaples"] #::Bool = false
            if use_autogeneratedstaples

                couplingcoeff = actions["couplingcoeff"] #::Array{Float64,1} = []
                #println(couplingcoeff)

                if haskey(actions,"coupling_loops")
                    coupling_loops = actions["coupling_loops"]
                else
                    error("actions[\"coupuling_loops\"] should be set if actions[\"use_autogeneratedstaples\"] is true")
                end
                if haskey(actions,"couplinglist")
                    @assert length(actions["couplinglist"]) == length(couplingcoeff) "the length of actions[\"couplinglist\"] should be equivalent to the length of actions[\"couplingcoeff\"]"
                    #println("1" )
                    couplinglist = actions["couplinglist"] #::Array{String,1} = []
                else
                    #println("2" )
                    couplinglist = String["$n" for n=1:length(couplingcoeff)]
                end
                #println(couplinglist )
            else
                couplinglist = []
                couplingcoeff = []
                coupling_loops = nothing
            end

            Δτ = md["Δτ"] #::Float64 = 0.05
            MDsteps = md["MDsteps"]#::Int64 = 20
            SextonWeingargten = md["SextonWeingargten"] #::Bool = false
            Nsteps = system["Nsteps"] #::Int64 = 100
            Nthermalization = system["Nthermalization"] #::Int64 = 10
            if SextonWeingargten
                if haskey(md,"N_SextonWeingargten")
                    N_SextonWeingargten = md["N_SextonWeingargten"]
                else
                    error("md[\"N_SextonWeingargten\"] should be set.")
                end
            else
                N_SextonWeingargten = 0
            end

            #For Heatbath
            #Heatbath = system["Heatbath"] #::Bool = false

            #For CG
            if quench == false
                eps = cg["eps"] #::Float64= 1e-19
                MaxCGstep = cg["MaxCGstep"] #::Int64 = 3000
            else
                eps = 1e-19
                MaxCGstep = 3000
            end

            #For Wilson
            if Dirac_operator == "Wilson" || Dirac_operator == "WilsonClover"
                println("$Dirac_operator fermion is used")
                if haskey(wilson,"hop")
                    hop = wilson["hop"] #::Float64 # = 0.141139#Hopping parameter # Mandatory
                else
                    error("wilson[\"hop\"] should be set.")
                end
                r = wilson["r"] #::Float64 = 1 #Wilson term

            else
                wilson["hop"]  = 0
                hop = 0
                r = 0
            end

            if Dirac_operator == "Domainwall"
                println("$Dirac_operator fermion is used")
                Domainwall_M = wilson["Domainwall_M"]
                Domainwall_m = wilson["Domainwall_m"]
                Domainwall_N5 = wilson["Domainwall_N5"]
                Domainwall_b = wilson["Domainwall_b"]
                Domainwall_c = wilson["Domainwall_c"]
                Domainwall_ωs =wilson["Domainwall_ωs"]
                Domainwall_r = wilson["Domainwall_r"]
                #r = wilson["r"]

            else
                Domainwall_N5 = nothing
                Domainwall_b = nothing
                Domainwall_c = nothing
                Domainwall_ωs = nothing
                Domainwall_r = nothing
                Domainwall_M = nothing
                Domainwall_m = nothing
            end

            #For WilsonClover
            if Dirac_operator == "WilsonClover"
                Clover_coefficient = wilson["Clover_coefficient"] #::Float64  = 1.5612
            else
                wilson["Clover_coefficient"] = 0
                Clover_coefficient = 0
            end

            #For Staggered
            if Dirac_operator == "Staggered"
                println("Staggered fermion is used")
                if haskey(staggered,"mass")
                    mass = staggered["mass"] #::Float64 # = 0.5 # Mandatory
                else
                    error("staggered[\"mass\"] should be set.")
                end
                Nf = staggered["Nf"] #::Int8 = 4
            else
                staggered["mass"] = 0
                mass = 0
                staggered["Nf"] = 0
                Nf = staggered["Nf"] #::Int8 = 4
            end

            verboselevel = system["verboselevel"]

            measurement_methods = measurement["measurement_methods"]

            saveU_format = system["saveU_format"]
            seveU_every = system["saveU_every"]
            saveU_dir = system["saveU_dir"]

            if saveU_format ≠ nothing
                if isdir(saveU_dir) == false
                    mkdir(saveU_dir)
                end
                println("$saveU_dir is used for saving configurations")
            end

            βeff = 0.0
            firstlearn = 0

            update_method = system["update_method"]
            useOR = false
            numOR = 0

            if update_method == "HMC"
                println("HMC will be used")
            elseif update_method == "Heatbath"
                println("Heatbath will be used")

                if system["quench"] == false
                    error("system[\"quench\"] = false. The Heatbath method needs the quench update. Put the other system[\"update_method\"] != \"Heatbath\" or system[\"quench\"] = true")
                end

                useOR = set_params(system,"useOR",false)
                numOR = set_params(system,"numOR",0)

            elseif update_method == "IntegratedHMC"
                println("IntegratedHMC will be used")

                #IntegratedHMC = system["IntegratedHMC"]
                if system["quench"] == false
                    println("system[\"quench\"] = true is set")
                    system["quench"] = true
                    #error("system[\"quench\"] = false. The IntegratedHMC needs the quench update. Put the other system[\"update_method\"] != \"IntegratedHMC\" or system[\"quench\"] = true")
                end

            elseif update_method == "Fileloading"
                println("No update will be used (read-measure mode)")
                system["quench"] = true
            elseif update_method == "SLHMC"
                println("SLHMC will be used")
                if system["quench"] == false
                    println("system[\"quench\"] = true is set")
                    system["quench"] = true
                    #error("system[\"quench\"] = false. The SLHMC needs the quench update. Put the other system[\"update_method\"] != \"SLHMC\" or system[\"quench\"] = true")
                end

                if haskey(system,"βeff")
                    βeff = system["βeff"]
                else
                    error("system[\"βeff\"] should be set when you want to do SLHMC.")
                end

                if haskey(system,"firstlearn")
                    firstlearn = system["firstlearn"]
                else
                    error("system[\"firstlearn\"] should be set when you want to do SLHMC.")
                end
            elseif update_method == "SLMC"
                println("SLMC will be used")
                if system["quench"] == false
                    println("system[\"quench\"] = true is set")
                    system["quench"] = true
                    #error("system[\"quench\"] = false. The SLMC needs the quench update. Put the other system[\"update_method\"] != \"SLMC\" or system[\"quench\"] = true")
                end

                if haskey(system,"βeff")
                    βeff = system["βeff"]
                else
                    error("system[\"βeff\"] should be set when you want to do SLMC.")
                end

                if haskey(system,"firstlearn")
                    firstlearn = system["firstlearn"]
                else
                    error("system[\"firstlearn\"] should be set when you want to do SLMC.")
                end
            elseif update_method == "IntegratedHB"
                println("IntegratedHB will be used")

                #IntegratedHMC = system["IntegratedHMC"]
                if system["quench"] == false
                    println("system[\"quench\"] = true is set")
                    system["quench"] = true
                    #error("system[\"quench\"] = false. The IntegratedHB needs the quench update. Put the other system[\"update_method\"] != \"IntegratedHB\" or system[\"quench\"] = true")
                end
            else
                error("""
                system[\"update_method\"] = $update_method is not supported.
                Supported methods are
                HMC
                Heatbath
                IntegratedHMC
                Fileloading
                """)
            end

            if update_method == "Fileloading"
                if haskey(system,"loadU_format")
                    loadU_format = system["loadU_format"]
                else
                    error("""system[\"loadU_format\"] should be set.
                    Supported formats are
                    JLD
                    """)
                end

                if haskey(system,"loadU_dir")
                    loadU_dir = system["loadU_dir"]
                else
                    error("system[\"loadU_dir\"] should be set")
                end

            else
                if haskey(system,"loadU_format")
                    loadU_format = system["loadU_format"]
                else
                    loadU_format = nothing
                end

                loadU_dir = "."
            end

            if haskey(system,"log_dir")
                log_dir = system["log_dir"]
                if isdir(log_dir) == false
                    mkdir(log_dir)
                end
            else
                error("system[\"log_dir\"] should be set")
            end

            if haskey(system,"logfile")
                logfile = pwd()*"/"*log_dir*"/"*system["logfile"]
            else
                error("system[\"logfile\"] should be set")
            end
            load_fp = open(logfile,"a")

            if haskey(measurement,"measurement_basedir")
                measurement_basedir = measurement["measurement_basedir"]
                if isdir(measurement_basedir) == false
                    mkdir(measurement_basedir)
                end
            else
                measurement_basedir = "nothing"
            end

            if haskey(measurement,"measurement_dir")
                measurement_dir = measurement["measurement_dir"]
                if isdir(pwd()*"/"*measurement_basedir*"/"*measurement_dir) == false
                    mkdir(pwd()*"/"*measurement_basedir*"/"*measurement_dir)
                end
            else
                measurement_dir = "nothing"
            end
            measuredir = pwd()*"/"*measurement_basedir*"/"*measurement_dir

            if haskey(actions,"IntegratedFermionAction")
                integratedFermionAction = actions["IntegratedFermionAction"]
            else
                integratedFermionAction = false
            end

            if haskey(actions,"training_data_name")
                training_data_name = pwd()*"/"*actions["training_data_name"]
            else
                training_data_name = pwd()*"/"*"trainingdata.txt"
            end

            if haskey(system,"initialtrj")
                initialtrj = system["initialtrj"]
            else
                initialtrj  = 1
            end

            if haskey(system,"loadU_fromfile")
                loadU_fromfile = system["loadU_fromfile"]
            else
                loadU_fromfile = false
            end

            #=
            if haskey(system,"loadU_filename")
                loadU_filename = system["loadU_filename"]
            else
                loadU_filename = ""
            end
            =#

            loadU_filename =set_params(system,"loadU_filename","")
            smearing_for_fermion = set_params(system,"smearing_for_fermion","nothing")
            stout_numlayers = set_params(system,"stout_numlayers",nothing)
            stout_ρ = set_params(system,"stout_ρ",nothing)
            stout_loops = set_params(system,"stout_loops",nothing)

            julian_random_number = set_params(system,"julian_random_number",true)

            #error("r = $r desu")
            return new(
                L,# = system["L"] #::Tuple  = (2,2,2,2) # Mandatory
                β,# = system["β"] #::Float64 # = 6 # Mandatory
                NC,# = system["NC"] #::Int64 = 3
                Dirac_operator,# = system["Dirac_operator"] #::Union{Nothing,String} = "Wilson"
                quench,# = system["quench"] #::Bool = false
                initial,# = system["initial"] #::String = "cold"
                BoundaryCondition,# = system["BoundaryCondition"] #::Array{Int8,1}=[1,1,1,-1]
                Nwing,# = system["Nwing"] #::Int64 = 1
                randomseed,# = system["randomseed"] #::Int64 = 111
                use_autogeneratedstaples,# = actions["use_autogeneratedstaples"] #::Bool = false
                couplinglist,# = actions["couplinglist"] #::Array{String,1} = []
                couplingcoeff,# = actions["couplingcoeff"] #::Array{Float64,1} = []
                coupling_loops,
                Δτ,# = md["Δτ"] #::Float64 = 0.05
                MDsteps,# = md["MDsteps"]#::Int64 = 20
                SextonWeingargten,# = md["SextonWeingargten"] #::Bool = false
                Nsteps,# = system["Nsteps"] #::Int64 = 100
                Nthermalization,# = system["Nthermalization"] #::Int64 = 10
                N_SextonWeingargten,# = md["N_SextonWeingargten"]
                #IntegratedHMC,# = system["IntegratedHMC"]
                #Heatbath,# = system["Heatbath"] #::Bool = false
                eps,# = cg["cg"] #::Float64= 1e-19
                MaxCGstep,# = cg["MaxCGstep"] #::Int64 = 3000
                hop,# = wilson["hop"] #::Float64 # = 0.141139#Hopping parameter # Mandatory
                r,# = wilson["r"] #::Float64 = 1 #Wilson term
                Clover_coefficient,# = wilson["Clover_coefficient"] #::Float64  = 1.5612
                mass,# = staggered["mass"] #::Float64 # = 0.5 # Mandatory
                Nf,# = staggered["Nf"] #::Int8 = 4
                verboselevel,
                measurement_methods,
                saveU_format,
                seveU_every,
                saveU_dir,
                loadU_format,
                loadU_dir,
                update_method,
                βeff,
                firstlearn,
                log_dir,
                logfile,
                load_fp,
                measuredir,
                integratedFermionAction,
                training_data_name,
                useOR,
                numOR,
                initialtrj,
                loadU_fromfile,
                loadU_filename,
                smearing_for_fermion,
                stout_numlayers,
                stout_ρ,
                stout_loops,
                Domainwall_M,#::Float64
                Domainwall_m,#::Float64
                Domainwall_N5,#::Int64
                Domainwall_b,#::Float64
                Domainwall_c,#::Float64
                Domainwall_ωs,#::Union{Nothing,Array{Float64,1}}
                Domainwall_r,
                julian_random_number
            )

        end

        function Params(params_set::Params_set)
            return Params(params_set.system,params_set.actions,params_set.md,params_set.cg,params_set.wilson,params_set.staggered,params_set.measurement)
        end

    end

    function set_params(dict,string,default)
        if haskey(dict,string)
            return dict[string]
        else
            return default
        end
    end

    function make_parametersdict(p::T) where T <: Any
        pnames = fieldnames(T)
        pdict = Dict()
        for i=1:length(pnames)
            pdict[String(pnames[i])] = getfield(p,pnames[i])
        end
        return pdict,pnames
    end

    function print_parameters_file(p)
        filename = p.logfile*"_parameters.jl"# "parameters_used.jl"
        fp = open(filename,"w")
        println(fp,"# - - parameters - - - - - - - - - - - ")
        pdict,pnames = make_parametersdict(p)
        for param in pdict
            if typeof(param[2]) == String
                println(fp,"$(param[1]) = \"$(param[2])\"")
            else
                println(fp,"$(param[1]) = $(param[2])")
            end
        end
        println(fp,"# - - - - - - - - - - - - - - - - - - -")
        close(fp)
        return
    end

    #=
    function gather_allparameters(params_set::Params_set)
        allparams = Dict()
        for (name,key) in params_set.system
            allparams[name] = key
        end

        for (name,key) in params_set.actions
            allparams[name] = key
        end

        for (name,key) in params_set.md
            allparams[name] = key
        end

        for (name,key) in params_set.cg
            allparams[name] = key
        end

        for (name,key) in params_set.wilson
            allparams[name] = key
        end

        for (name,key) in params_set.staggered
            allparams[name] = key
        end

        for (name,key) in params_set.measurement
            allparams[name] = key
        end
        return allparams
    end
    =#

    function setprint(fp,fp2,string)
        println(fp,string)
        if fp2 != nothing
            println(fp2,string)
        end
        println(string)
    end

    function get_stringfromkey(key)
        if typeof(key) == String
            string = "\"$key\""
        else
            string = "$key"
        end
    end

    function get_header(params_set::Params_set,inputname)
        if haskey(params_set.system,inputname)
            return "system",params_set.system[inputname]
        elseif haskey(params_set.actions,inputname)
            return "actions",params_set.actions[inputname]
        elseif haskey(params_set.md,inputname)
            return "md",params_set.md[inputname]
        elseif haskey(params_set.cg,inputname)
            return "cg",params_set.cg[inputname]
        elseif haskey(params_set.wilson,inputname)
            return "wilson",params_set.wilson[inputname]
        elseif haskey(params_set.staggered,inputname)
            return "staggered",params_set.staggered[inputname]
        elseif haskey(params_set.measurement,inputname)
            return "measurement",params_set.measurement[inputname]
        else
            return nothing,nothing
        end
    end

    function print_measurementinfo(fp,fp2,key)
        string = "measurement[\"measurement_methods\"] = Dict[ "
        setprint(fp,fp2,string)
        for (i,data) in enumerate(key)
            string = "  Dict{Any,Any}(\"methodname\" => \"$(data["methodname"])\","
            setprint(fp,fp2,string)
            count = 0
            #println(data)
            for (name,key_i) in data
                #println(name)
                #println(key_i)
                if name != "methodname"
                    count += 1
                    paramstring = get_stringfromkey(key_i)
                    string = "    \"$(name)\" => "*paramstring
                    if count != length(data) -1
                        string *= ","
                    end
                    if name == "fermiontype" && key_i == nothing
                    else
                        setprint(fp,fp2,string)
                    end

                end
            end
            string = "  )"
            if i != length(key)
                string *= ","
            end
            setprint(fp,fp2,string)

        end
        string = "]"
        setprint(fp,fp2,string)
    end

    function print_parameters_list(params_set::Params_set,p=nothing;filename=nothing)
        if filename == nothing
            @assert p != nothing "wrong input!"

            filename = p.logfile*"_parameters.jl"
            fp2 = p.load_fp
        else
            fp2 = nothing
        end
        fp = open(filename,"w")

        setprint(fp,fp2,"# - - parameters - - - - - - - - - - - ")
        for (i,printlist_i) in enumerate(printlists)
            setprint(fp,fp2,printlists_header[i] )
            for name in printlist_i
                headstring,key = get_header(params_set,name)
                if headstring != nothing
                    if name == "measurement_methods"
                        print_measurementinfo(fp,fp2,key)
                    else
                        string = get_stringfromkey(key)
                        paramstring = headstring*"[\"$name\"] = "*string
                        setprint(fp,fp2,paramstring)
                    end
                end
            end
            setprint(fp,fp2,"\t" )
        end

        setprint(fp,fp2,"# - - - - - - - - - - - - - - - - - - -")

        close(fp)

        println("""
        # Your parameters were written in $filename
        # If you want to do the simulation with same parameters,
        # Just do
        # julia run.jl $filename
        """)
        flush(stdout)

    end

    function print_parameters(params_set::Params_set,p)
        #allparams = gather_allparameters(params_set)
        print_parameters_list(params_set,p)
        return
    end

    function print_parameters(filename,params_set::Params_set)
        #allparams = gather_allparameters(params_set)
        print_parameters_list(params_set,filename=filename)
    end

    function print_parameters(p)
        println("# - - parameters - - - - - - - - - - - ")

        pdict,pnames = make_parametersdict(p)
        for param in pdict
            if typeof(param[2]) == String
                println("$(param[1]) = \"$(param[2])\"")
            else
                println("$(param[1]) = $(param[2])")
            end
        end
        println("# - - - - - - - - - - - - - - - - - - -")

        print_parameters_file(p)
        println("""
        # Your parameters were written in parameters_used.jl
        # If you want to do the simulation with same parameters,
        # Just do
        # julia run.jl parameters_used.jl
        """)
        flush(stdout)
        return
    end

    #=
    system  = System_params()
    md = MDparams()
    fermimon = Wilson_parames()
    measurement = Measurement_parames()
    =#

    #=
    if length(ARGS) > 0
        include(pwd()*"/"*ARGS[1])
    end
    =#

    function defaultparameters()
        return Params_set(system,actions,md,cg,wilson,staggered,measurement)
    end

    #display(system)
    #exit()

    function parameterloading(system,actions,md,cg,wilson,staggered,measurement)
        param_set = Params_set(system,actions,md,cg,wilson,staggered,measurement)
        if param_set.system["Dirac_operator"] == nothing
            param_set.system["quench"] = true
        end
        p = Params(param_set)
        Random.seed!(p.randomseed)

        print_parameters(param_set,p)
        return p
    end

    function parameterloading(param_set::Params_set)
        if param_set.system["Dirac_operator"] == nothing
            param_set.system["quench"] = true
        end
        p = Params(param_set)
        Random.seed!(p.randomseed)

        print_parameters(param_set,p)
        return p
    end

    function parameterloading()
        param_set = defaultparameters()

        if param_set.system["Dirac_operator"] == nothing
            param_set.system["quench"] = true
        end
        p = Params(param_set)
        Random.seed!(p.randomseed)

        print_parameters(param_set)
        return p

    end

end

#using .System_parameters
#System_parameters.parameterloading(ARGS[1])


"""
    CurrentStep(num_steps, fmt, lc, header)

Trace column display the current step and total number of iteration,
e.g. `[ 4/10]`, with the figures colored according to progress made.
"""
struct CurrentStep <: TraceColumn
    num_steps::Integer
    fmt::FormatExpr
    lc::LinearColorant{Int}
    header::String
end

"""
    CurrentStep(num_step[; lc, header])

Create a current step trace column for `num_step` iterations. The
default color scale goes from red to green, and the default header is
blank.
"""
function CurrentStep(num_steps::Integer;
                     lc::LinearColorant=LinearColorant(1, num_steps, red_green_scale()),
                     header::String=repeat(" ", 2length(digits(num_steps))+3))
    nd = length(digits(num_steps))
    spacer = repeat(" ", max(0,length(header)-2nd-3))
    CurrentStep(num_steps,
                FormatExpr("[{1:s}{2:$(nd)d}/{3:$(nd)d}$(crayon"reset")]$spacer"),
                lc, rpad(header, 2nd+3))
end
(c::CurrentStep)(i::Integer) = (c.lc(i), i, c.num_steps)

export CurrentStep


# Load and save results, statistics and run parameters
function save_results_csv(results::Dict, file_name; save_order=DEFAULT_SAVE_ORDER, dir="")
	AC_inputs = results["inputs"]
	AC_outputs = results["outputs"]
	duals = results["duals"]
	return save_results_csv(AC_inputs, AC_outputs, duals, file_name, save_order=save_order, dir=dir)
end

"""
Saves the AC\\_output, AC\\_input, duals to a csv file
"""
function save_results_csv(AC_inputs, AC_outputs, duals, file_name; save_order=DEFAULT_SAVE_ORDER, dir="")

	data = results_to_array(AC_inputs, AC_outputs, duals, save_order=save_order, header=false, imag_j=true)

	save_path = joinpath(dir, file_name*".csv")

	#TASK: Create file directory automatically to save files?
	if isfile(save_path)  # File already exists append data to it
		open(save_path, "a") do io
		writedlm(io, data, ',')
		end
	else  # Build header for network and create new file
		header = build_header(AC_inputs, AC_outputs, duals, save_order)
		data = vcat(header, data)
		writedlm(save_path, data, ',')
	end
end

"""
Saves the given sample to the given csv file, appending it to the end of the file.
"""
function save_sample_csv(x, results, file_name, save_order, dual_vars; dir="")
	# Gen. & Ext. and Dual Results
	primal_vals, dual_vals = results

    # Load results
	new_AC_inputs = extract_load(x)

    # Duals results
	dual_vals["unique_active_sets"] = extract_active_set(dual_vals, dual_vars)

	save_results_csv(new_AC_inputs, primal_vals, dual_vals, file_name, save_order=save_order, dir=dir)
end

"""
Builds the header as a new row to add to a data array with input, output, then dual data.
The order of the header is dictated by the variables in the save_order argument with each
element of a variable becoming a single column. Uses element labels to determine the type
of element the result variable refers to.
"""
function build_header(AC_inputs, AC_outputs, duals, save_order)
	header = []

	all_data = merge(AC_inputs, AC_outputs, duals)

	# Remove any keys in save_order that are not in the given results
	all_data_keys = keys(all_data)
	matched_label_indxs = [key in all_data_keys for key in save_order]
	save_order = save_order[matched_label_indxs]

	for key in save_order
		element = ELEMENT_LABELS[key]
		append!(header, ["$(element)$(i):"*string(key) for i in 1:size(all_data[key],2)])
	end

	return reshape(header,(1,length(header)))
end

"""
Saves the A and b matrices defining the polytope Ax < b to two seperate comma
delimited csv files.
"""
function save_polytope(A, b, file_name; dir="")
    save_path_A = joinpath(dir, file_name * "_A" * ".csv")
    save_path_b = joinpath(dir, file_name * "_b" * ".csv")

    writedlm(save_path_A, A, ',')
    writedlm(save_path_b, b, ',')
end

"""
Saves an infeasibility certificate by appending new rows for the A and b matrices
to csv files. If these csv files do not already exist then calls save_polytope
to save the entire A and b matrices in new csv files.
"""
function save_cert(A, b, file_name; dir="")
	A_last = A[end, :]'
	b_last = b[end, :]

    save_path_A = joinpath(dir, file_name * "_A" * ".csv")
    save_path_b = joinpath(dir, file_name * "_b" * ".csv")

	if isfile(save_path_A) & isfile(save_path_b)  # File already exists append data to it
		for (x, save_path) in zip((A_last, b_last), (save_path_A, save_path_b))
			open(save_path, "a") do io
				writedlm(io, x, ',')
			end
		end
	else  # Build header for network and create new file
		save_polytope(A, b, file_name, dir=dir)
	end
end

"""
Saves the last iteration of level 1 stats to a csv file.
If the csv file does not already exist then calls save_stats
to save the stats data with a header to a new csv file.
"""
function save_stats(stats, file_name; dir="")
    save_path = joinpath(dir, file_name * ".csv")

	stat_labels = add_dim_h(sort(collect(keys(stats))))

	if stats[stat_labels[1]] isa Array && size(stats[stat_labels[1]], 1) > 1  # More than one iteration of data
		data = stats[stat_labels[1]]
	else
		data = [stats[stat_labels[1]]]
	end
	saved_stat_labels = [stat_labels[1]]
	for stat_label in stat_labels[2:end]
		if !(stats[stat_label] isa Dict)
			push!(saved_stat_labels, stat_label)
			data = hcat(data, stats[stat_label])
		end
	end
	saved_stat_labels = add_dim_h(saved_stat_labels)

	if isfile(save_path) # File already exists append data to it
		open(save_path, "a") do io
		writedlm(io, data, ',')
		end
	else  # Create new file with header
		stat_data = vcat(saved_stat_labels, data)
		writedlm(save_path, stat_data, ',')
	end
end

"Makes an array, x, have a second dimension and be vertically oriented"
add_dim_v(x::Array) = reshape(x, (size(x)...,1))

"Makes an array, x, have a second dimension and be horizontally oriented"
add_dim_h(x::Array) = reshape(x, (1, size(x)...))


#############################################################################
# ECOS.jl
# Wrapper around the ECOS solver https://github.com/ifa-ethz/ecos
# See http://github.com/JuliaOpt/ECOS.jl
#############################################################################
# ECOSSolverInterface.jl
# MathProgBase.jl interface for the ECOS.jl solver wrapper
#############################################################################

require(joinpath(Pkg.dir("MathProgBase"),"src","MathProgSolverInterface.jl"))
importall MathProgSolverInterface

#############################################################################
# Define the MPB Solver and Model objects
export ECOSSolver
immutable ECOSSolver <: AbstractMathProgSolver
end

type ECOSMathProgModel <: AbstractMathProgModel
    nvar::Int                           # Number of variables
    nineq::Int                          # Number of inequalities Gx <=_K h
    neq::Int                            # Number of equalities Ax = b
    npos::Int                           # Number of ???
    ncones::Int                         # Number of SO cones
    conedims::Vector{Int}               # ?
    G::SparseMatrixCSC{Float64,Int}     # The G matrix (inequalties)
    A::SparseMatrixCSC{Float64,Int}     # The A matrix (equalities)
    c::Vector{Float64}                  # The objective coeffs (always min)
    orig_sense::Symbol                  # Original objective sense
    h::Vector{Float64}                  # RHS for inequality
    b::Vector{Float64}                  # RHS for equality
    # Post-solve
    solve_stat::Symbol
    obj_val::Float64
    primal_sol::Vector{Float64}
    fwd_map::Vector{Int}                # To reorder solution if we solved
end                                     # using the conic interface
ECOSMathProgModel() = ECOSMathProgModel(0,0,0,0,0,
                                        Int[],
                                        spzeros(0,0),
                                        spzeros(0,0),
                                        Float64[], :Min,
                                        Float64[], Float64[],
                                        :NotSolved, 0.0, Float64[], Int[])

#############################################################################
# Begin implementation of the MPB low-level interface
# Implements
# - model
# - loadproblem!
# - optimize!
# - status
# http://mathprogbasejl.readthedocs.org/en/latest/lowlevel.html

model(s::ECOSSolver) = ECOSMathProgModel()

# Loads the provided problem data to set up the linear programming problem:
# min c'x
# st  lb <= Ax <= ub
#      l <=  x <= u
# where sense = :Min or :Max
function loadproblem!(m::ECOSMathProgModel, A, collb, colub, obj, rowlb, rowub, sense)
    (nvar = length(collb)) == length(colub) || error("Unequal lengths for column bounds")
    (nrow = length(rowlb)) == length(rowub) || error("Unequal lengths for row bounds")

    # Turn variable bounds into constraints
    # Inefficient, because keeps allocating memory!
    # Would need to batch, get tricky...
    for j = 1:nvar
        if collb[j] != -Inf
            # Variable has lower bound
            newrow = zeros(1, nvar)
            newrow[j] = -1.0
            A = vcat(A, newrow)
            rowlb = vcat(rowlb, -Inf)
            rowub = vcat(rowub, -collb[j])
            nrow += 1
        end
        if colub[j] != +Inf
            # Variable has upper bound
            newrow = zeros(1, nvar)
            newrow[j] = 1.0
            A = vcat(A, newrow)
            rowlb = vcat(rowlb, -Inf)
            rowub = vcat(rowub, colub[j])
            nrow += 1
        end
    end

    eqidx   = Int[]      # Equality row indicies
    ineqidx = Int[]      # Inequality row indicies
    eqbnd   = Float64[]  # Bounds for equality rows
    ineqbnd = Float64[]  # Bounds for inequality row
    for it in 1:nrow
        # Equality constraint
        if rowlb[it] == rowub[it]
            push!(eqidx, it)
            push!(eqbnd, rowlb[it])
        # Range constraint - not supported
        elseif rowlb[it] != -Inf && rowub[it] != Inf
            error("Ranged constraints unsupported!")
        # Less-than constraint
        elseif rowlb[it] == -Inf
            push!(ineqidx, it)
            push!(ineqbnd, rowub[it])
        # Greater-than constraint - flip sign so only have <= constraints
        else
            push!(ineqidx, it)
            push!(ineqbnd, -rowlb[it])
            A[it,:] *= -1 # flip signs so we have Ax<=b
        end
    end

    m.nvar      = nvar                  # Number of variables
    m.nineq     = length(ineqidx)       # Number of inequalities Gx <=_K h
    m.neq       = length(eqidx)         # Number of equalities Ax = b
    m.npos      = length(ineqidx)       # Number of ???
    m.ncones    = 0                     # Number of SO cones
    m.conedims  = Int[]                 # ???
    m.G         = sparse(A[ineqidx,:])  # The G matrix (inequalties)
    m.A         = sparse(A[eqidx,:])    # The A matrix (equalities)
    m.c         = (sense == :Max) ? obj * -1 : obj[:]
                                        # The objective coeffs (always min)
    m.orig_sense = sense                # Original objective sense
    m.h         = ineqbnd               # RHS for inequality
    m.b         = eqbnd                 # RHS for equality
    m.fwd_map   = [1:nvar]              # Identity mapping
end

function optimize!(m::ECOSMathProgModel)
    ecos_prob_ptr = setup(
        m.nvar, m.nineq, m.neq,
        m.npos, m.ncones, m.conedims,
        m.G, m.A,
        m.c[:],  # Seems to modify this
        m.h, m.b)

    flag = solve(ecos_prob_ptr)
    if flag == ECOS_OPTIMAL
        m.solve_stat = :Optimal
    elseif flag == ECOS_PINF
        m.solve_stat = :Infeasible
    elseif flag == ECOS_DINF  # Dual infeasible = primal unbounded, probably
        m.solve_stat = :Unbounded
    elseif flag == ECOS_MAXIT
        m.solve_stat = :UserLimit
    else
        m.solve_stat = :Error
    end
    # Extract solution
    ecos_prob = pointer_to_array(ecos_prob_ptr, 1)[1]
    m.primal_sol = pointer_to_array(ecos_prob.x, m.nvar)[:]
    m.obj_val = dot(m.c, m.primal_sol) * (m.orig_sense == :Max ? -1 : +1)
    cleanup(ecos_prob_ptr, 0)
end

status(m::ECOSMathProgModel) = m.solve_stat
getobjval(m::ECOSMathProgModel) = m.obj_val
getsolution(m::ECOSMathProgModel) = m.primal_sol[m.fwd_map]

#############################################################################
# Begin implementation of the MPB conic interface
# Implements
# - loadconicproblem!
# http://mathprogbasejl.readthedocs.org/en/latest/conic.html

function loadconicproblem!(m::ECOSMathProgModel, c, A, b, cones)
    # TODO (if it matters): make this more efficient for sparse A

    # We don't support SOCRotated, SDP, or Exp*
    bad_cones = [:SOCRotated, :SDP, :ExpPrimal, :ExpDual]
    for cone_vars in cones
        cone_vars[1] in bad_cones && error("Cone type $(cone_vars[1]) not supported")
    end

    # MathProgBase form             ECOS form
    # min c'x                       min c'x
    # st  A x = b                   st  A x = b
    #       x in K                      h - Gx in K

    # Expand out the cones info
    # The cones can come in any order, so we need to build a mapping
    # from the variable indices in the input to the internal ordering
    # we will use.

    # In the first past we'll just count up the number of variables
    # of each type.
    num_vars = 0
    for (cone_type, idxs) in cones
        num_vars += length(idxs)
    end
    fwd_map = Array(Int,    num_vars)  # Will be used for SOCs
    rev_map = Array(Int,    num_vars)  # Need to restore sol. vec.
    idxcone = Array(Symbol, num_vars)  # We'll uses this for non-SOCs

    # Now build the mapping
    pos = 1
    for (cone, idxs) in cones
        for i in idxs
            fwd_map[i]   = pos   # fwd_map = orig idx -> internal idx
            rev_map[pos] = i     # rev_map = internal idx -> orig idx
            idxcone[pos] = cone
            pos += 1
        end
    end

    # Rearrange data into the internal ordering
    ecos_c = c[rev_map]
    ecos_A = A[:,rev_map]
    ecos_b = b[:]

    # For all variables in the :Zero cone, fix at 0 with an
    # equality constraint. TODO: Don't even include them

    for j = 1:num_vars
        idxcone[j] != :Zero && continue

        new_row    = zeros(1,num_vars)
        new_row[j] = 1.0
        ecos_A     = vcat(ecos_A, new_row)
        ecos_b     = vcat(ecos_b, 0.0)
    end

    # Build G matrix
    # There will be one row for every :NonNeg and :NonPos cone
    # and an additional row for every variable in a :SOC cone
    # Or in other words, everything that isn't a :Free or :Zero
    # gets a row in G and h
    num_G_row = 0
    for j = 1:num_vars
        idxcone[j] == :Free && continue
        idxcone[j] == :Zero && continue
        num_G_row += 1
    end
    ecos_G = zeros(num_G_row,num_vars)
    ecos_h = zeros(num_G_row)

    # First, handle the :NonNeg, :NonPos cases
    num_pos_orth = 0
    G_row = 1
    for j = 1:num_vars
        if idxcone[j] == :NonNeg
            ecos_G[G_row,j] = -1.0
            G_row += 1
            num_pos_orth += 1
        elseif idxcone[j] == :NonPos
            ecos_G[G_row,j] = +1.0
            G_row += 1
            num_pos_orth += 1
        end
    end
    @assert G_row == num_pos_orth + 1
    # Now handle the SOCs
    # The MPB unput form is basically just says a vector of
    # variables (y,x) lives in the SOC  || x || <= y
    # ECOS wants somethings in the form h - Gx in Q so we
    # will prove 0 - Ix \in Q
    num_SOC_cones = 0
    SOC_conedims  = Int[]
    for (cone, idxs) in cones
        cone != :SOC && continue
        # Found a new SOC
        num_SOC_cones += 1
        push!(SOC_conedims, length(idxs))
        # Add the entries (carrying on from pos. orthant)
        for j in idxs
            ecos_G[G_row,fwd_map[j]] = -1.0
            G_row += 1
        end
    end
    @assert G_row == num_G_row + 1

    # Store in the ECOS structure
    m.nvar          = num_vars
    m.nineq         = num_G_row
    m.neq           = length(ecos_b)
    m.npos          = num_pos_orth
    m.ncones        = num_SOC_cones
    m.conedims      = SOC_conedims
    m.G             = ecos_G
    m.A             = ecos_A
    m.c             = ecos_c
    m.orig_sense    = :Min
    m.h             = ecos_h
    m.b             = ecos_b
    m.fwd_map       = fwd_map
end


#
println("Cj) Test of the GreenFunction module for a given list of bound-state configurations.")

@warn("\n\n !!! This example does not work properly at present !!! \n\n")
#
setDefaults("print summary: open", "zzz-GreenFunction.sum")
wa = Atomic.Computation("xx",  Nuclear.Model(26.); properties=[Green],
                        configs=[Configuration("[Ne] 3s^2 3p^6")],
                        greenSettings=GreenFunction.Settings(GreenFunction.SingleCSFwithoutCI(), Basics.DeExciteSingleElectron(),
                                                             5, [0, 1, 2], [LevelSymmetry(1//2,Basics.plus), LevelSymmetry(1//2,Basics.plus)],
                                                             true, false, Int64[]) )

wb = perform(wa)
setDefaults("print summary: close", "")


# overview over all variables used

meteo_UT	# unixtime meteodata
meteo_DT	# datetime meteodata
SWd		# shortwave radiation downwards
WS		# wind strength
WD		# wind direction
WDstdv		# standard deviation of wind direction
Temp 		# temperature at the peak
RH  		# relative humidity at the peak

TEMPstation
RHstation
PRESSUREstation

BC_UT 		# unixtime black carbon
BC_DT 		# datetime black carbon
BC_sum 		# total black carbon

ACSM_UT 	# unixtime ACSM
ACSM_DT		# datetime ACSM
ACSM_org	# total organics in particle
ACSM_sulf	# total sulfates in particle
ACSM_nitr	# total nitrate in particle
ACSM_ammon	# total ammonia in particle
ACSM_chlor	# total chlorine species in particle

SMPS_DT 	# datetime SMPS
SMPS_UT		# unixtime SMPS
SMPS_binsize 	# binsizes SMPS
SMPS_totconc	# total concentration of particles from SMPS
SMPS_binconc	# dNdlog(Dp) SMPS

no3cims_DT	# datetime Nitrate CIMS
no3cims_UT	# unixtime Nitrate CIMS
Nitrophenol 	# Nitrophenol from Nitrate CIMS
SO5trace	# SO5- trace from Nitrate CIMS
MSAtrace 	# MSA from Nitrate CIMS

SRRs		# Source Receptor Relationship Timetraces
SRRnames	# Names of SRRs

ptr3Traces	# all Traces from PTR3
ptr3Times	# Times from PTR3
ptr3Peaks	# all Peaks from PTR3


# include("Walk.jl")

abstract Simulation

type MetropolisSimulation <: Simulation
    walk::Walk

    accepted_steps::Int
    rejected_steps::Int
    # automatically_rejected_steps::Int # FIXME: add this
    trivial_steps::Int
    total_steps::Int

    # beeps::Int # FIXME: add this to keep track of numerical error rate

    measurements::Dict{ASCIIString, Measurement}

    function MetropolisSimulation(walk::Walk)
        return new(walk, 0, 0, 0, 0, Dict{ASCIIString, Measurement}())
    end
end

function add_accepted_step!(sim::MetropolisSimulation)
    sim.accepted_steps += 1
    sim.total_steps += 1
end

function add_rejected_step!(sim::MetropolisSimulation)
    sim.rejected_steps += 1
    sim.total_steps += 1
end

# FIXME: add this
# function add_automatically_rejected_step(sim::MetropolisSimulation)
#     sim.rejected_steps += 1
#     sim.automatically_rejected_steps +=1
#     sim.total_steps += 1
# end

function add_trivial_step!(sim::MetropolisSimulation)
    sim.trivial_steps += 1
    sim.total_steps += 1
end

function consistent_MCinfo(sim::MetropolisSimulation)
    return sim.accepted_steps + sim.rejected_steps + sim.trivial_steps == sim.total_steps
end

# FIXME: instead, just report one acceptance rate, with separate functions giving trival and auto reject %s
function acceptance_rate_1(sim::MetropolisSimulation)
    @assert consistent_MCinfo(sim)
    return (sim.accepted_steps + sim.trivial_steps) / sim.total_steps
end

function acceptance_rate_2(sim::MetropolisSimulation)
    @assert consistent_MCinfo(sim)
    return sim.accepted_steps / (sim.accepted_steps + sim.rejected_steps)
end

function acceptance_rate_3(sim::MetropolisSimulation)
    @assert consistent_MCinfo(sim)
    return sim.accepted_steps / sim.total_steps
end

# by default this will do nothing, but it can be overwritten by a given wavefunction implementation
function print_wf_info(wf)
end

function iterate!(sim::MetropolisSimulation, steps_to_iterate::Int, steps_per_check::Int = 100, print_rate::Float64 = 0.05)

    printlnf("Beginning to iterate a Metropolis simulation $(steps_to_iterate) steps!")
    for k in 1:steps_to_iterate

        if mod(k, print_rate*steps_to_iterate) == 0
        # if true
            printlnf("Starting step #$(k)/$(steps_to_iterate) --> $(100k/steps_to_iterate)%")
            print_wf_info(sim.walk.wf)
        end

        # check for error periodically
        if mod(sim.total_steps, steps_per_check) == 0
            # printlnf("Checking for numerical error!")
            check_for_numerical_error!(sim.walk)
        end

        propose_and_assess_random_transition!(sim.walk)
        if !is_nontrivial_transition(sim.walk)
            add_trivial_step!(sim)
            # printlnf("Step is trivial!")
            # printlnf()
            # FIXME: for trivial transitions, we shouldn't have to recompute the measurements!
            for k in keys(sim.measurements)
                possibly_measure(sim.walk.wf, sim.measurements[k].data, sim.measurements[k].measure_function)
            end
            continue
        else
            probrat = get_probability_ratio(sim.walk)
        end
        # printlnf("probrat = $(probrat)")

        # Metropolis!
        if probrat >= 1.0 || rand() < probrat # accept transition
            accept_transition!(sim.walk)
            add_accepted_step!(sim)
            # printlnf("Step accepted!")
        else # reject transition
            reject_transition!(sim.walk)
            add_rejected_step!(sim)
            # printlnf("Step rejected!")
        end
        # printlnf()

        for k in keys(sim.measurements)
            possibly_measure(sim.walk.wf, sim.measurements[k].data, sim.measurements[k].measure_function)
        end

    end
end


"""
    StatsBase.predict(m::LinearMixedModel, newdata;
                    new_re_levels=:missing)
    StatsBase.predict(m::GeneralizedLinearMixedModel, newdata;
                    new_re_levels=:missing, type=:response)

Predict response for new data.

!!! note
    Currently, no in-place methods are provided because these methods
    internally construct a new model and therefore allocate not just a
    response vector but also many other matrices.

!!! warning
    Models are assumed to be full rank.

!!! warning
    These methods construct an entire MixedModel behind the scenes and
    as such may use a large amount of memory when `newdata` is large.

The keyword argument `new_re_levels` specifies how previously unobserved
values of the grouping variable are handled. Possible values are
`:population` (return population values, i.e. fixed-effects only),
`:missing` (return `missing`), `:error` (error on this condition;
the error type is an implementation detail). If you want simulated values
for unobserved levels of the grouping variable, consider the
[`simulate!`](@ref) and `simulate` methods.

Predictions based purely on the fixed effects can be obtained by
specifying previously unobserved levels of the random effects and setting
`new_re_levels=:population`. In the future, it may be possible to specify
a subset of the grouping variables or overall random-effects structure to use,
but not at this time.

For `GeneralizedLinearMixedModel`, the `type` parameter specifies
whether the predictions should be returned on the scale of linear predictor
(`:linpred`) or on the response scale (`:response`). If you don't know the
difference between these terms, then you probably want `type=:response`.

Regression weights are not yet supported in prediction.
Similarly, offsets are also not supported for `GeneralizedLinearMixedModel`.
"""
function StatsBase.predict(
    m::LinearMixedModel, newdata::Tables.ColumnTable; new_re_levels=:population
)
    return _predict(m, newdata, m.β; new_re_levels)
end

function StatsBase.predict(
    m::GeneralizedLinearMixedModel,
    newdata::Tables.ColumnTable;
    new_re_levels=:population,
    type=:response,
)
    type in (:linpred, :response) || throw(ArgumentError("Invalid value for type: $(type)"))

    y = _predict(m.LMM, newdata, m.β; new_re_levels)

    return type == :linpred ? y : broadcast!(Base.Fix1(linkinv, Link(m)), y, y)
end

# β is separated out here because m.β != m.LMM.β depending on how β is estimated for GLMM
function _predict(m::MixedModel{T}, newdata, β; new_re_levels) where {T}
    new_re_levels in (:population, :missing, :error) ||
        throw(ArgumentError("Invalid value for new_re_levels: $(new_re_levels)"))

    # if we ever support simulation, here some old bits from the docstrings
    # `new_re_levels=:simulate` is also not yet available for `GeneralizedLinearMixedModel`.
    # , `:simulate` (simulate new values).
    # For `:simulate`, the values are determined by solving for their values by
    # using the existing model's estimates for the new data. (These are in general
    # *not* the same values as the estimates computed on the new data.)

    # the easiest thing here is to just assemble a new model and
    # pass that to the other predict methods....
    # this can probably be made much more efficient

    # note that the contrasts don't matter for prediction purposes
    # (at least for the response)

    # add a response column
    # we get type stability via constant propogation on `new_re_levels`
    y, mnew = let ytemp = ones(T, length(first(newdata)))
        f, contr = _abstractify_grouping(m.formula)
        lmm = LinearMixedModel(f, newdata; contrasts=contr)
        ytemp =
            new_re_levels == :missing ? convert(Vector{Union{T,Missing}}, ytemp) : ytemp

        ytemp, lmm
    end

    grps = fnames(m)
    mul!(y, mnew.X, β)
    # mnew.reterms for the correct Z matrices
    # ranef(m) for the BLUPs from the original fit

    # because the reterms are sorted during model construction by
    # number of levels and that number may not be the same for the
    # new data, we need to permute the reterms from both models to be
    # in the same order
    newreperm = sort(1:length(mnew.reterms); by=x -> string(mnew.reterms[x].trm))
    oldreperm = sort(1:length(m.reterms); by=x -> string(m.reterms[x].trm))
    newre = mnew.reterms[newreperm]
    oldre = m.reterms[oldreperm]

    if new_re_levels == :error
        for (grp, known_levels, data_levels) in
            zip(grps, levels.(m.reterms), levels.(mnew.reterms))
            if sort!(known_levels) != sort!(data_levels)
                throw(ArgumentError("New level enountered in $grp"))
            end
        end

        # we don't have to worry about the BLUP ordering within a given
        # grouping variable because we are in the :error branch
        blups = ranef(m)[oldreperm]
    elseif new_re_levels == :population
        blups = ranef(mnew)[newreperm]
        blupsold = ranef(m)[oldreperm]

        for (idx, B) in enumerate(blups)
            oldlevels = levels(oldre[idx])
            for (lidx, ll) in enumerate(levels(newre[idx]))
                oldloc = findfirst(isequal(ll), oldlevels)
                if oldloc === nothing
                    # setting a BLUP to zero gives you the population value
                    B[:, lidx] .= zero(T)
                else
                    B[:, lidx] .= @view blupsold[idx][:, oldloc]
                end
            end
        end
    elseif new_re_levels == :missing
        # we can't quite use ranef! because we need
        # Union{T, Missing} and not just T
        blups = Vector{Matrix{Union{T,Missing}}}(undef, length(m.reterms))
        copyto!(blups, ranef(mnew)[newreperm])
        blupsold = ranef(m)[oldreperm]
        for (idx, B) in enumerate(blups)
            oldlevels = levels(oldre[idx])
            for (lidx, ll) in enumerate(levels(newre[idx]))
                oldloc = findfirst(isequal(ll), oldlevels)
                if oldloc === nothing
                    # missing is poisonous so propogates
                    B[:, lidx] .= missing
                else
                    B[:, lidx] .= @view blupsold[idx][:, oldloc]
                end
            end
        end
        # elseif new_re_levels == :simulate
        #     @show m.θ
        #     updateL!(setθ!(mnew, m.θ))
        #     blups = ranef(mnew)[newreperm]
        #     blupsold = ranef(m)[oldreperm]
        #     for (idx, B) in enumerate(blups)
        #         oldlevels = levels(oldre[idx])
        #         for (lidx, ll) in enumerate(levels(newre[idx]))
        #             oldloc = findfirst(isequal(ll), oldlevels)
        #             if oldloc === nothing
        #                 # keep the new value
        #             else
        #                 B[:, lidx] = @view blupsold[idx][:, oldloc]
        #             end
        #         end
        #     end
    else
        throw(ErrorException("Impossible branch reached. Please report an issue on GitHub"))
    end

    for (rt, bb) in zip(newre, blups)
        mul!(y, rt, bb, one(T), one(T))
    end

    return y
end

# yup, I got lazy on this one -- let the dispatched method handle kwarg checking
function StatsBase.predict(m::MixedModel, newdata; kwargs...)
    return predict(m, columntable(newdata); kwargs...)
end

## should we add in code for prediction intervals?
# we don't directly implement (Wald) confidence intervals, so directly
# supporting (Wald) prediction intervals seems a step too far right now


## Test NarmaxRegressors constructor
@testset "Test NarmaxRegressors constructor" begin
    l = NarmaxLasso.generate_all(NarmaxLasso.Monomial, 4, 1:2)
    yterms = [1]
    uterms = [1]
    vterms = [1, 2]
    reg = NarmaxLasso.NarmaxRegressors(l, yterms, uterms, vterms)
    @test reg.n_vterms == 2
    @test reg.n_yterms == 1
    @test reg.n_uterms == 1
    for i = 1:(reg.noise_components_starts-1)
        @test all(reg.basis[i].exponents[3:4] .== 0)
    end
    for i = (reg.noise_components_starts):length(reg.basis)
        @test any(reg.basis[i].exponents[3:4] .> 0)
    end
end

## Test generate lag matrix
@testset "Test generate lag matrix" begin
    l = NarmaxLasso.generate_all(NarmaxLasso.Monomial, 4, 1:2)
    yterms = [1]
    uterms = [1]
    vterms = [1, 2]
    reg = NarmaxLasso.NarmaxRegressors(l, yterms, uterms, vterms)
    y = [1.0, 2.0, 3.0, 4.0]
    u = [10.0, 20.0, 30.0, 40.0]
    v = [100.0, 200.0, 300.0, 400.0]
    M = NarmaxLasso.lag_matrix(reg, y, u, v)
    @test M ≈ [1.0  10.0  100.0  100.0;
               1.0  10.0  100.0  100.0;
               2.0  20.0  200.0  100.0;
               3.0  30.0  300.0  200.0]
end

## Test generate regressor matrix
@testset "Test generate regressor matrix" begin
    l = NarmaxLasso.generate_all(NarmaxLasso.Monomial, 3, 1)
    yterms = [1]
    uterms = [1]
    vterms = [1]
    reg = NarmaxLasso.NarmaxRegressors(l, yterms, uterms, vterms)
    y = [1.0, 2.0, 3.0, 4.0]
    u = [10.0, 20.0, 30.0, 40.0]
    v = [100.0, 200.0, 300.0, 400.0]
    M = NarmaxLasso.lag_matrix(reg, y, u, v)
    @test M ≈ [1.0  10.0  100.0;
               1.0  10.0  100.0;
               2.0  20.0  200.0;
               3.0  30.0  300.0]
end

## Test generate all monomial terms
@testset "Test generate all monomial terms" begin
    reg = NarmaxLasso.generate_all(NarmaxLasso.NarmaxRegressors,
                                NarmaxLasso.Monomial, 2, 2, 2, 1:2)
    @test length(reg.basis) == 27
    reg = NarmaxLasso.generate_all(NarmaxLasso.NarmaxRegressors,
                                NarmaxLasso.Monomial, 2, 2, 2, 1:3)
    @test length(reg.basis) == 83
end


using Distributed
length(procs()) == 1 && addprocs(1)

@everywhere using YAActL, Printf

@everywhere begin
    struct PM <: Message        # define a message
        txt::String
    end

    # define two behaviors accepting a msg::Message as their last argument
    function pr(msg::PM)
        print(@sprintf("%s\n", msg.txt))
        become(pr, "Next") # change behavior
    end
    pr(info, msg::PM) = print(@sprintf("%s: %s\n", info, msg.txt))

    # a behavior for doing arithmetic
    function calc(op::F, v::U, msg::Request) where {F<:Function,U<:Number}
        send!(msg.from, Response(op(v,msg.x)))
    end
end

# t = Ref{Task}()                # this is for debugging
# lk = LinkParams(2, taskref=t)
sleep(5)
# start an actor with the first behavior and save the returned link
myactor = Actor(workers()[1], pr)

send!(myactor, PM("My first actor"))  # send a message to it

send!(myactor, PM("Something else"))  # send again a message

become!(myactor, pr, "New behavior")     # change the behavior to another one

send!(myactor, PM("bla bla bla"))     # and send again a message

become!(myactor, calc, +, 10)

send!(myactor, Request(5, USR))

take!(USR.chn)


@require TypedTables begin

immutable EnumerableTypedTable{T, TS} <: Enumerable{T}
    df::TypedTables.Table
    # This field hols a tuple with the columns of the DataFrame.
    # Having a tuple of the columns here allows the iterator
    # functions to access the columns in a type stable way.
    columns::TS
end

function query(df::TypedTables.Table)
    col_expressions = Array{Expr,1}()
    df_columns_tuple_type = Expr(:curly, :Tuple)
    for i in 1:length(df.data)
        push!(col_expressions, Expr(:(::), names(df)[i], eltype(df.data[i])))
        push!(df_columns_tuple_type.args, typeof(df.data[i]))
    end
    t_expr = NamedTuples.make_tuple(col_expressions)

    t2 = :(Query.EnumerableTypedTable{Float64,Float64})
    t2.args[2] = t_expr
    t2.args[3] = df_columns_tuple_type

    eval(NamedTuples, :(import Query))
    t = eval(NamedTuples, t2)

    e_df = t(df, df.data)

    return e_df
end

function length{T,TS}(iter::EnumerableTypedTable{T,TS})
    return size(iter.df,1)
end

function eltype{T,TS}(iter::EnumerableTypedTable{T,TS})
    return T
end

function start{T,TS}(iter::EnumerableTypedTable{T,TS})
    return 1
end

@generated function next{T,TS}(iter::EnumerableTypedTable{T,TS}, state)
    constructor_call = Expr(:call, :($T))
    for i in 1:length(iter.types[2].types)
        push!(constructor_call.args, :(columns[$i][i]))
    end

    quote
        i = state
        columns = iter.columns
        a = $constructor_call
        return a, state+1
    end
end

function done{T,TS}(iter::EnumerableTypedTable{T,TS}, state)
    return state>size(iter.df,1)
end

end


#=
  Package: FishEBM
  File: agents.jl
  Justin Angevaare, Devin Rose
  Functions for agent-level model components used by other functions
  Created: May 2015
=#

"""
  Description: A function to create an empty agent database for the specified
    simulation length.

  Returns: Vector(EnviroAgent)

  Last update: March 2016
"""
function AgentDB(enviro::EnvironmentAssumptions)
  #Initialize the database
  agent_db = [EnviroAgent(0)]; init = false;
  length = (size(enviro.habitat)[1])*(size(enviro.habitat)[2])

  #Push new agents into the database for each enviro location
  for i = 1:length
    if enviro.habitat[i] > 0
      if init == false
        agent_db[1].locationID = i
        init = true
      else
        push!(agent_db, EnviroAgent(i))
      end
    end
  end

  return agent_db
end

"""
  Description: Function used to find the current life stage of a cohort from
    the current age using the agent assumptions growth vector.

  Returns: Int64

  Last update: September 2016
"""
function findCurrentStage(current_week::Int64, spawn_week::Int64, growth_age::Vector{Int64})
  #Initialize the life stage number to 4
  currentStage = 4
  q = length(growth_age)-1
  current_age = current_week - spawn_week

  #Most cohorts are likely to be adults, thus check stages from old to young
  while q > 0 && current_age < growth_age[q]
    currentStage = q
    q-=1
  end

  return currentStage
end

"""
  Description: Returns age of cohort. If cohort is not adult, returns 0.

    Returns: Int64

  Last Update: July 2016
"""
function getAge(current_week::Int64, spawn_week::Int64)
  age = floor((current_week - spawn_week)/52)
  if age < 2
    return 0
  elseif age > 8
    return 8
  else
    return Int(age)
  end
end

"""
  Description: Returns population of a specific age in an environment agent.
    Used for functions that requires age-specific population to be taken into
    account (such as spawning or harvest).

    Returns: Int64

  Last Update: September 2016
"""
function getAgeSpecificPop(age::Int64, current_week::Int64, alive::Vector{Int64}, weekNum::Vector{Int64}, a_a::AgentAssumptions)
  @assert(2 <= age <= 8, "Age argument must be between 2 and 8, inclusive (age = $age was passed).")
  classLength = length(weekNum)
  pop = 0
  for i = 1:classLength
    if findCurrentStage(current_week, weekNum[i], a_a.growth) == 4
      if age == 8
        if floor((current_week - weekNum[i]) / 52) >= age
          pop += alive[i]
        end
      else
        if floor((current_week - weekNum[i]) / 52) == age
          pop += alive[i]
        end #if floor
      end #if age == 8 else
    end #findCurrentStage
  end #for i = 1:classLength

  return pop
end

"""
  Description: Returns index of cohort of specified age. Used in harvest! to
    determine which cohort to subtract the harvest size from. If no cohort exist
    for the specified age, function will return 0 and so calling function should
    check if this function returns 0.

  Returns: Int64

  Last Update: September 2016
"""
function getCohortNumber(age::Int64, current_week::Int64, weekNum::Vector{Int64})
  classLength = length(weekNum)
  for i = 1:classLength
    if floor((current_week - weekNum[i]) / 52) == age
      return i
    end #if floor
  end #for i

  return 0
end

"""
  Description:  Used to get population of any of the stages (egg, larva,
    juvenile, adult). Will only return population of one stage at a time. To
    get total population of fish, loop through for i = 1:4.

  Returns: Int64

  Last update: September 2016
"""
function getStagePopulation(stage::Int64, current_week::Int64, agent_db::Vector{EnviroAgent}, a_a::AgentAssumptions)

  classLength = length((agent_db[1]).weekNum)
  pop = 0
  for i = 1:length(agent_db)
    if (isEmpty(agent_db[i]) == false)
      for j = 1:classLength
        if findCurrentStage(current_week, agent_db[i].weekNum[j], a_a.growth) == stage
          pop += agent_db[i].alive[j]
        end #findCurrentStage
      end #for j=1:classLength
    end #if isEmpty
  end #for i=1:length

  return pop
end

"""
  Description: This function is used for efficiently finding the Agent number
    from a known environment id.

  Returns: Int64

  Last update: September 2016
"""
function IDToAgentNum(a_db::Vector{EnviroAgent}, id_num::Int64, max_val::Int64, min_val::Int64)
  testVal = rand(min_val:max_val)

  #uses recursion
  if id_num == a_db[testVal].locationID
    return testVal
  elseif a_db[testVal].locationID > id_num
    testVal = IDToAgentNum(a_db, id_num, testVal, min_val)
  elseif a_db[testVal].locationID < id_num
    testVal = IDToAgentNum(a_db, id_num, max_val, testVal)
  end

  return testVal
end

"""
  Description: This function injects agents into the environment, this is
    function is mainly used for adding agents to the environment to test new
    functions.

    Returns: Operates directly on agent_db

  Last update: September 2016
"""
function injectAgents!(agent_db::Vector{EnviroAgent}, spawn_agents::Vector{Int64}, new_stock::Int64, week_num::Int64)
  @assert(length(new_stock)<=4, "There can only by four independent life stages of fish.")

  addToEach = round(Int, floor(new_stock/length(spawn_agents)))
  leftOver = new_stock%length(spawn_agents)
  randomAgent = rand(1:length(spawn_agents))

  #add a new population class to every agent
  for agentRef = 1:length(agent_db)
    push!((agent_db[agentRef]).alive, 0)
    push!((agent_db[agentRef]).weekNum, week_num)
  end

  #Only adds agents to the last (newest) cohort in the agent db spawn locations
  classLength = length((agent_db[1]).weekNum)
  for agentNum = 1:length(spawn_agents)
    addToAgent = addToEach
    if agentNum == randomAgent
      addToAgent += leftOver
    end

    (agent_db[spawn_agents[agentNum]]).alive[classLength] = addToAgent
  end
end

"""
  Description: This function is used to remove an empty spawn class once all
    agents in the entire spawn class have been removed.

  Returns: Operates directly on age_db

  Last Update: September 2016
"""
function removeEmptyClass!(age_db::Vector{EnviroAgent})
  if length(age_db[1].alive) > 1
    removeClass = true
    for i = 1:length(age_db)
      if (age_db[i]).alive[1] != 0
        removeClass = false
        i = length(age_db)
      end
    end

    if removeClass
      for j = 1:length(age_db)
        shift!((age_db[j]).alive)
        shift!((age_db[j]).weekNum)
      end
      removeEmptyClass!(age_db)
    end
  end
end


@testset "134.gas-station.jl" begin
    @test can_complete_circuit([2, 3, 4], [3, 4, 3]) == -1
    @test can_complete_circuit([1, 2, 3, 4, 5], [3, 4, 5, 1, 2]) == 3
end


using CCDReduction: axes_min_length,
                    fits_indices,
                    convert_value

@testset "bias subtraction" begin
    # testing non-mutating version
    @test @inferred(subtract_bias(ones(500, 500), ones(500, 500))) == zeros(500, 500)

    # testing mutating version
    frame = ones(500, 500)
    bias = ones(500, 500)
    @inferred subtract_bias!(frame, bias)
    @test frame == zeros(500, 500)

    # testing error
    @test_throws DimensionMismatch subtract_bias(ones(500, 1), ones(500, 500))
end

@testset "overscan subtraction" begin
    # testing non-mutating version
    @test @inferred(subtract_overscan(ones(500, 600), (1:2, :))) == zeros(500, 600)
    @test @inferred(subtract_overscan(ones(500, 600), "[:, 1:2]")) == zeros(500, 600)

    # testing mutating version
    frame = ones(500, 600)
    frame1 = ones(500, 600)
    @inferred subtract_overscan!(frame, (:, 540:600))
    @inferred subtract_overscan!(frame1, "[540:600, :]")
    @test frame == zeros(500, 600)
    @test frame == frame1

    # testing error
    @test_throws BoundsError subtract_overscan(ones(500, 600), (500:600, :))
end

@testset "flat correction" begin
    #testing non-mutating version
    frame = ones(5, 5)
    flat = fill(2.0, (5, 5))
    @test @inferred(flat_correct(frame, flat)) == ones(5, 5)
    @test @inferred(flat_correct(frame, flat, norm_value = 4.0)) == fill(2.0, (5, 5))

    #testing mutating version
    frame = ones(5, 5)
    flat = rand(5, 5)
    reduced_flat = flat ./ mean(flat)
    reduced_frame = frame ./ reduced_flat

    @inferred flat_correct!(frame, flat)
    @test reduced_frame == frame

    #testing error
    @test_throws DimensionMismatch flat_correct(ones(5, 5), ones(5, 6))
    @test_throws ErrorException flat_correct(ones(5, 5), ones(5, 5), norm_value = -2)
end

@testset "trim" begin
    @test trim(reshape(1:25, 5, 5), (:, 4:5)) == trimview(reshape(1:25, 5, 5), (:, 4:5))
    @test trim(reshape(1:25, 5, 5), (4:5, :)) == trimview(reshape(1:25, 5, 5), (4:5, :))
    @test trim(reshape(1:25, 5, 5), (:, 4:5)) == [1:5 6:10 11:15]
    @test trim(reshape(1:25, 5, 5), (1:2, :)) == [3:5 8:10 13:15 18:20 23:25]
    @test trim(reshape(1:25, 5, 5), "[4:5, :]") == trimview(reshape(1:25, 5, 5), (:, 4:5))
    @test trim(reshape(1:25, 5, 5), "[:, 4:5]") == trimview(reshape(1:25, 5, 5), (4:5, :))
    @test trim(reshape(1:25, 5, 5), "[1:5, 4:5]") == trimview(reshape(1:25, 5, 5), (4:5, :))
    @test trim(reshape(1:25, 5, 5), "[4:5, 1:5]") == trimview(reshape(1:25, 5, 5), (:, 4:5))

    # testing output types
    @test trimview(ones(5, 5), (:, 3:5)) isa SubArray
    @test trim(ones(5, 5), (:, 3:5)) isa Array

    # testing errors
    @test_throws ErrorException trim(ones(5, 5), (4:5, 1:4))
    @test_throws ErrorException trim(ones(5, 5), (:, :))
    @test_throws ErrorException trim(ones(5, 5), (4:6, :))
end

@testset "cropping" begin
    # testing on odd dimension array
    x = reshape(1:25, (5, 5))

    @test crop(x, (3, 3)) == [7:9 12:14 17:19]
    @test_logs (:warn, "dimension 1 changed from 4 to 5") @test crop(x, (4, 3)) == [6:10 11:15 16:20]
    @test crop(x, (4, 3), force_equal = false) == [6:9 11:14 16:19]
    @test crop(x, (1, 1)) == fill(13, (1, 1))
    @test_logs (:warn, "dimension 2 changed from 4 to 5") @test crop(x, (3, 4)) == [2:4 7:9 12:14 17:19 22:24]
    @test crop(x, (3, 4), force_equal = false) == [2:4 7:9 12:14 17:19]

    # testing on even dimension array
    x = reshape(1:16, (4, 4))

    @test cropview(x, (2, 2))  == [6:7 10:11]
    @test_logs (:warn, "dimension 1 changed from 3 to 4") @test cropview(x, (3, 2)) == [5:8 9:12]
    @test cropview(x, (3, 2), force_equal = false) == [5:7 9:11]
    @test cropview(x, (1, 1), force_equal = false) == fill(6, (1, 1))
    @test_logs (:warn, "dimension 2 changed from 3 to 4") @test cropview(x, (2, 3)) == [2:3 6:7 10:11 14:15]
    @test cropview(x, (2, 3), force_equal = false) == [2:3 6:7 10:11]

    # testing with colon
    x = reshape(1:25, (5, 5))

    @test cropview(x, (:, :)) == x
    @test cropview(x, (:, 3)) == [6:10 11:15 16:20]
    @test cropview(x, (3, :)) == [2:4 7:9 12:14 17:19 22:24]
    @test_logs (:warn, "dimension 1 changed from 2 to 3") @test cropview(x, (2, :)) == [2:4 7:9 12:14 17:19 22:24]
    @test cropview(x, (2, :), force_equal = false) == [2:3 7:8 12:13 17:18 22:23]
    @test_logs (:warn, "dimension 2 changed from 2 to 3") @test cropview(x, (:, 2)) == [6:10 11:15 16:20]
    @test cropview(x, (:, 2), force_equal = false) == [6:10 11:15]

    # testing output types
    @test cropview(ones(5, 5), (3, 3)) isa SubArray
    @test crop(ones(5, 5), (3, 3)) isa Array

    # testing error
    @test_throws BoundsError cropview(ones(5, 5), (7, 3))
    @test_throws ErrorException cropview(ones(5, 5), (3, -5))
    @test_throws ErrorException cropview(ones(5, 5), (3, 4, 5))
end

@testset "combining" begin
    x = [reshape(1:4, (2,2)) for i = 1:10]

    @test combine(x) == [1:2 3:4]
    @test combine(x, method = sum) == [1:2 3:4] .* 10

    # testing error
    @test_throws DimensionMismatch combine(rand(5, 5), rand(6, 6))
    @test_throws MethodError combine()
end

@testset "dark subtraction" begin
    # testing non-mutating version
    @test @inferred(subtract_dark(ones(5, 5), ones(5, 5))) == zeros(5, 5)
    @test @inferred(subtract_dark(ones(5, 5), ones(5, 5), data_exposure = 1, dark_exposure = 4)) == fill(0.75, 5, 5)
    @test @inferred(subtract_dark(ones(Float32, 5, 5), ones(Float32, 5, 5), data_exposure = 13, dark_exposure = 17)) ≈ fill(Float32(4/17), 5, 5)

    # testing mutating version
    frame = ones(5, 5)
    dark_frame = ones(5, 5)
    @inferred(subtract_dark!(frame, dark_frame))
    @test frame == zeros(5, 5)

    frame = fill(5.0, 5, 5)
    dark_frame = ones(5, 5)
    @inferred(subtract_dark!(frame, dark_frame, data_exposure = 2, dark_exposure = 1))
    @test frame  == fill(3.0, 5, 5)

    frame = fill(Float32(3), 5, 5)
    dark_frame = fill(Float32(2), 5, 5)
    @inferred(subtract_dark!(frame, dark_frame, data_exposure = 11, dark_exposure = 19))
    @test frame ≈ fill(Float32(35/19), 5, 5)

    # testing error
    @test_throws DimensionMismatch subtract_dark!(ones(5,5), ones(6,6))
end

@testset "helper" begin
    # testing axes_min_length
    @test axes_min_length((:, :)) == 1
    @test axes_min_length((:, 5:10)) == 2
    @test axes_min_length((1:2, :)) == 1
    @test axes_min_length((1:2, 5:12)) == 1
    @test axes_min_length((10:100, 1:5)) == 2

    # testing fits_indices
    @test fits_indices("[1024:2048, 200:300]") == [200:300, 1024:2048]
    @test fits_indices("[:, 200:300]") == [200:300, :]
    @test fits_indices("[1024:2048, :]") == [:, 1024:2048]
    @test fits_indices("[200:300, 1024:2048]") == [1024:2048, 200:300]

    # testing convert_value
    @test convert_value(Int16, 5.4) == 5
    @test convert_value(Int32, 5.4) == 5
    @test convert_value(Int64, -5.4) == -5
    @test convert_value(Float64, -5.4) ≈ -5.4
    @test convert_value(Float32, -5.4) ≈ -5.4
    @test convert_value(Float32, -5.4) ≈ -5.4
end


function get_convergence_checker(obj::AbstractOptimizationProblem)
    return jcall(obj, "getConvergenceChecker", ConvergenceChecker, ())
end

function get_evaluation_counter(obj::AbstractOptimizationProblem)
    return jcall(obj, "getEvaluationCounter", Incrementor, ())
end

function get_iteration_counter(obj::AbstractOptimizationProblem)
    return jcall(obj, "getIterationCounter", Incrementor, ())
end


module Surprise
# package code goes here

using Persa
using PyCall

const surprise = PyNULL()

function __init__()
    copy!(surprise, pyimport("surprise"))
end

abstract type SurpriseModel{T} <: Persa.Model{T} end

function Persa.train!(model::SurpriseModel, ds::Persa.Dataset)
    ds_surprise = transform(ds)

    model.object.fit(ds_surprise)

    return nothing
end

include("utils.jl")
include("memorybased.jl")
include("modelbased.jl")

end # module


using Ripemd
using Nettle
using BenchmarkTools

reload("Ripemd")
d = b"";
@benchmark Ripemd.ripemd160($d)
@benchmark Nettle.digest("ripemd160", $d)

d = b"a";
@benchmark Ripemd.ripemd160($d)
@benchmark Nettle.digest("ripemd160", $d)

d = [b"a"[1] for j in 1:10000];
@benchmark Ripemd.ripemd160($d)
@benchmark Nettle.digest("ripemd160", $d)

d = *(["a" for j in 1:10000]...);
@benchmark Ripemd.ripemd160($d)
@benchmark Nettle.digest("ripemd160", $d)


using Test
using Yota
using Yota: Tape, Input, Call, Constant, trace, ctrace, irtrace, play!, transform, binarize_ops
using Yota: ∇mean, setfield_nested!, copy_with, simplegrad, remove_unused
using Yota: find_field_source_var, unwind_iterate, eliminate_common
using Yota: unvcat, unhcat, uncat

include("test_trace_cassette.jl")
include("test_trace_irtools.jl")
# include("test_trace_interp.jl")  # doesn't work for Julia 1.4, maybe will drop support at all
include("gradcheck.jl")
include("test_helpers.jl")
include("test_simple.jl")
include("test_grad.jl")
include("test_distributions.jl")
include("test_dynamic.jl")
include("test_simplegrad.jl")
include("test_update.jl")
include("test_transform.jl")
include("test_examples.jl")


module Hello

function f(x)
    return x^2 -2x + x
end

end


accessType = Union{Colon,<:Integer,<:AbstractArray{<:Union{Integer,Bool}},Nothing}
"""
    struct pathLocs{m}
        indices
    end
    pathLocs(varargs...; m::Int=2, d::Int=1, exs=Colon())
some words
    pathLocs(s::scattered)
make a pathLocs that accesses every location in the output
    pathLocs(ii, s::scattered)
make a pathLocs that accesses every location in layer `ii` of the output.
"""
struct pathLocs{m}
    indices
end
# specifying on
function pathLocs(varargs...; m::Int=2, d::Int=1, exs=Colon())
    if length(varargs) == 0
        # default constructor that includes everything
        paired = [(i, :) for i = 0:m]
    elseif length(varargs) % 2 == 0 && (varargs[1] isa Integer)
        # specifically enumerated, so alternates between a number and a reference shape
        # pair each layer with the tuple
        paired = [(varargs[i], varargs[i+1]) for i = 1:2:length(varargs)]
    elseif length(varargs) == m + 1
        # every layer specified
        paired = [(i, varargs[i+1]) for i = 0:m]
    end
    present = [x[1] for x in paired]
    notPresent = [ii for ii = 0:m if !(ii in present)] # unspecified layers
    paired = [paired..., [(x, nothing) for x in notPresent]...]
    paired = sort(paired, by=(x) -> x[1])
    indices = map(x -> parseOne(x, d, exs), paired)
    fullList = map(ii -> paired[ii+1], 0:m)
    return pathLocs{m + 1}((indices...,))
end

function parseOne(x, d, exs)
    lay, select = x
    if select == nothing
        return nothing
    end
    if lay == 0
        if select == Colon()
            return (map(x -> Colon(), 1:d)..., Colon(), exs)
        elseif length(select) == d + 2
            return select
        elseif typeof(select) <: AbstractArray && ndims(select) == d + 2
            return select
        elseif typeof(select) <: AbstractArray
            return (select, 1, exs)
        else
            return (select..., 1, exs)
        end
    elseif lay == 1
        if select == Colon()
            return (map(x -> Colon(), 1:d)..., Colon(), exs)
        elseif length(select) == d + 2
            return select
        elseif typeof(select) <: AbstractArray && ndims(select) == d + 2
            return select
        elseif typeof(select) <: AbstractArray || typeof(select) <: Number
            return (map(x -> Colon(), 1:d)..., select, exs)
        elseif length(select) == 1
            return (map(x -> Colon(), 1:d)..., select[1], exs)
        else
            return (select..., exs)
        end
    elseif lay >= 2
        if select == Colon()
            return (map(x -> Colon(), 1:d)..., Colon(), Colon(), exs)
        elseif (select isa Tuple) && length(select) >= d + 3
            return select
        elseif typeof(select) <: AbstractArray && ndims(select) == d + 3
            return select
        elseif typeof(select) <: AbstractArray # only specify the first layer
            return (map(x -> Colon(), 1:d+1)..., select, exs)
        elseif length(select) == 2 # specify both
            return (map(x -> Colon(), 1:d)..., select..., exs)
        else
            return (select..., 1, exs)
        end
    end
end

import Base: size
function size(p::pathLocs)
    ł(x::Nothing) = Tuple{Vararg{<:Integer,0}}()
    ł(x::AbstractArray) = size(x)
    # all that's left is a tuple of things
    ł(x) = map(a -> length(a), x)
    map(ł, p.indices)
end


using KNITRO
using Test

@testset "Test old API" begin
    include("oldexamples.jl")
end

if KNITRO.KNITRO_VERSION >= v"11.0"
    @testset "Test C API" begin
        include("knitroapi.jl")
    end

    @testset "Test examples" begin
        include("testexamples.jl")
    end

    @testset "Test MathOptInterface" begin
        include("MOI_wrapper.jl")
        include("MOI_additional.jl")
    end

    @testset "Test JuMP" begin
        include("jump_soc.jl")
    end
end


include("../imports.jl")

@testset ExtendedTestSet "projective" begin
    mask = MaskBinary(rand(Bool, 32, 32))
    tfm = Project(Translation(20, 10))

    @test_nowarn apply(tfm, mask)
    tmask = apply(tfm, mask)
    @test tmask.bounds.rs == (21:52, 11:42)

    @test_nowarn apply!(tmask, tfm, mask)
end


### A Pluto.jl notebook ###
# v0.14.8

using Markdown
using InteractiveUtils

# ╔═╡ c7b8a7d7-eda7-4cae-96c8-6c4e52d6e73d
using StructArrays, Query

# ╔═╡ 523d7e80-5c67-47a7-8ed7-69fa06b6b0f3
begin
	using XLSX, DataFrames
	using Statistics: mean
end

# ╔═╡ 9ae4f227-26cc-47c6-b95d-8f7811386171
#split a dataframe by case and dataset, then combine with summary stats
function case_dataset_pivot(summary, method)
	f = filter(row->row.method==method, summary)
	groups = groupby(f, [:case, :dataset])
	combine(groups, nrow => :n_problems,
		:total_elapsed_time => mean => :aver_run_time,
		:highest_rtol => mean => :aver_tol,
		:lowest_gap => mean => :aver_gap)
end

# ╔═╡ db530169-b2d3-48bf-a69a-49b4c070513b
begin
	# table helper functions
	tbl(s) = "
" * join(s) * "
"
	td(s) = "$(join(s))	"
	tr(s) = "
$(join(s))
"

	init_table(df) = fill("", (6*9, 6))
end

# ╔═╡ c9f0573c-ce4c-4114-8bee-5c52a3016246
# the combined dataset/case pivot table is a tall rectangle where every entry is a row
# we use this function to write each record to a specific location of a 2D summary
# table
""" write the passed dataframe to the record object, at the location specifed by df[:case] and df[:dataset]"""
function store!(record, df)
	str(i) = "$i"
	j = df[:case]
	i = ((df[:dataset] - 1) * 6) + 1

	record[i, j] = str(df[:case])
	record[i + 1, j] = str(df[:dataset])
	record[i + 2, j] = str(df[:n_problems])
	record[i + 3, j] = str(df[:aver_run_time])
	record[i + 4, j] = str(df[:aver_tol])
	record[i + 5, j] = str(df[:aver_gap])
end

# ╔═╡ e4f32a9c-34ff-4990-bd8b-2508eecd6e03
""" turn a combined GroupedDataframe into a flat table """
function flat_tbl(g)
	record = init_table(g)
	map(val -> store!(record, val), eachrow(g))
	record
end

# ╔═╡ 001d0435-b172-47c9-86e4-316a6a1104e0
show_tbl(g) = HTML(tbl(mapreduce(r -> tr(td.(r)), *, eachrow(flat_tbl(g)))))

# ╔═╡ 96a6702a-d181-45c7-b715-de9341eb31a6

# ╔═╡ 46b258b5-2e68-4739-a77b-a583afbe3dd6
"""turn an array into a head=>tail pair, used to add titles to the XLSX sheet when
making it a DataFrame"""
function col_to_pair(col)
	h, t = Iterators.peel(col)
	h => collect(t)
end

# ╔═╡ 8e2669a4-5382-45bc-b91e-2fe27b2a6910
#read the excel file we are summarizing
begin
	f = XLSX.readxlsx("../decision_tree_D.xlsx")
	sh = f["Sheet1"]
	dfm = DataFrame(map(col_to_pair, eachcol(DataFrame(sh[:]))))
	dfm
end

# ╔═╡ dde4a7e7-2298-440c-ae72-3546aaf8ef56
vasko_sum_table = case_dataset_pivot(dfm, "base");

# ╔═╡ 853f8eab-e829-423c-9a11-8153ff14090d
show_tbl(vasko_sum_table)

# ╔═╡ 8668715e-a505-4a59-9934-94d9ba824d95

# ╔═╡ 2ed608a9-b0f5-4c1d-85de-a0000f5009b2

# ╔═╡ 76515738-b71a-4d65-8583-3924d160813e

# ╔═╡ Cell order:
# ╠═853f8eab-e829-423c-9a11-8153ff14090d
# ╠═dde4a7e7-2298-440c-ae72-3546aaf8ef56
# ╠═9ae4f227-26cc-47c6-b95d-8f7811386171
# ╠═db530169-b2d3-48bf-a69a-49b4c070513b
# ╠═001d0435-b172-47c9-86e4-316a6a1104e0
# ╠═e4f32a9c-34ff-4990-bd8b-2508eecd6e03
# ╠═c9f0573c-ce4c-4114-8bee-5c52a3016246
# ╠═8e2669a4-5382-45bc-b91e-2fe27b2a6910
# ╠═96a6702a-d181-45c7-b715-de9341eb31a6
# ╠═46b258b5-2e68-4739-a77b-a583afbe3dd6
# ╠═c7b8a7d7-eda7-4cae-96c8-6c4e52d6e73d
# ╟─523d7e80-5c67-47a7-8ed7-69fa06b6b0f3
# ╠═8668715e-a505-4a59-9934-94d9ba824d95
# ╠═2ed608a9-b0f5-4c1d-85de-a0000f5009b2
# ╠═76515738-b71a-4d65-8583-3924d160813e