""" This file executes the CM with the previously calculated, new load profiles"""

using Pkg
if isfile("Project.toml") && isfile("Manifest.toml")
    Pkg.activate(".")
end

###################
# Terminal ########
###################

    usedayperiod = true
    startday = 1
    lastday = 365
    starthour = 1
    lasthour = 8760
	trm = 0	# reduces - or at negative value, increases - available capacity of transmission lines
	use221 = true	# whether the fictional generators with mc 221 are used

    datapath = "data/"
	newpath = "NewData/"
	inputpath = "output_2030/Output_LoadShift/Sens_quart/"
	outputpath = "output_2030/Output_DSOM/DSOM_Sens_quart_trm_0/"

   if usedayperiod == true
        timeperiod = string((startday-1)*24+1)*":"*string((lastday*24))
   else
        timeperiod = string(starthour)*":"*string(lasthour)
   end
    timeperiod_str = replace(string(timeperiod), ":" => "-")

    println()
    # println("Case >" * case * "< selected.")
    println("Calculating for time period >" * timeperiod * "<.")
    println()

###################
# Run CM ##########
###################

    println("Loading required packages: ProgressMeter, CSV, DataFrames, Dates, JuMP, Gurobi.")
    using ProgressMeter
    using CSV
    using DataFrames
    using Dates
    using JuMP
    using Gurobi
    println("Loading of packages complete.")

    include("model/src/Sesam_ptdf_Loadshift.jl") #WHEN USING PTDF
    # include("model/src/Sesam.jl") WHEN USING NO PTDF
    using .Sesam

###################
# Data ############
###################

	println("Reading and processing Data from CSV...")

    # nodes_df_30 =   # Dataframe mit ID, Lat, longitude
    nodes_df_30 = CSV.read(joinpath(newpath, "nodeCoordinates.csv"))
    nodes_df_30 = rename!(nodes_df_30, :node => :id)[:, 1:3]
#	nodes_df_30[:id] = nodes_df_30[:id] .+1

    # load_df_30 =    # Dataframe mit systemload und knotenweisen shares über alle stunden
    load_df_30 = CSV.read(joinpath(newpath, "demandHourly.csv"))
    rename!(load_df_30, 1 => :systemload)
    b = CSV.read(joinpath(newpath, "loadShare.csv"))
#   	b[:Column1] = b[:Column1] .+1
    for i in 1:nrow(b)
        load_df_30[Symbol(b[i, 1])] = repeat([b[i, 2]], 8760)
    end

    exchange_df_30 = CSV.read(joinpath(datapath, "exchange.csv"))

    # lines_df_30 =    # Dataframe mit id, from, to, voltage, reactance, resistance, circuits, pmax
    lines_df_30 = CSV.read(joinpath(newpath, "lines.csv"))
#	lines_df_30[:id] = lines_df_30[:id] .+1
#	lines_df_30[:from] = lines_df_30[:from] .+1
#	lines_df_30[:to] = lines_df_30[:to] .+1

    # powerplants_df_30 = # Dataframe mit unit, node, technology, fuel, capacity, efficiency, emission, varcost, rup, rdn, latitude, longitude
    temp = CSV.read(joinpath(newpath, "convGenData.csv"))
#	temp[:node] = temp[:node] .+ 1
    powerplants_df_30 = DataFrame(unit = String[], node = Int64[], technology = String[], fuel = String[], capacity = Float64[], efficiency = Float64[],
                                    emission = Float64[], varcost = Float64[], rup = Float64[], rdn = Float64[])
	use221 == true ? techs = names(temp)[2:end] : techs = names(temp)[2:end-1]
    for i in 1:nrow(temp)
        for class in names(temp)[2:end]
            if !ismissing(temp[class][i]) && temp[class][i] > 0
                pp_id = "NODE"*string(temp[i,1])*"C"*string(class)
                cap = temp[class][i]
                emission = 0
                efficiency = 1
                varcost = 0
                rup = 0
                rdn = 0
                push!(powerplants_df_30, [pp_id temp[i, 1] string(class) string(class) cap efficiency emission varcost rup rdn])
            end
        end
    end

    # avail_powerplants_df_30
    avail_powerplants_df_30 = DataFrame()
    for i in powerplants_df_30.unit
        avail_powerplants_df_30[Symbol(i)] = ones(8760)
    end

    # mustrun_powerplants_df_30 = # Dataframe mit überall nullen pro generator
    mustrun_powerplants_df_30 = DataFrame()
    for i in powerplants_df_30.unit
        mustrun_powerplants_df_30[Symbol(i)] = zeros(8760)
    end

    # fuelcost_df_30
    fuelcost_df_30 = DataFrame()
    for i in names(temp)[2:end]
        fuelcost_df_30[i] = [parse(Int, string(i))]
    end
    fuelcost_df_30[:CO2] = [0]

    # renewables_df_30 = # Dataframe mit Node, WindOnshore, WindOffshore, SolarPV, RoR, Geothermal
    solar_df_30 = CSV.read(joinpath(newpath, "solarHourly.csv"))
    solarshare_df_30 = CSV.read(joinpath(newpath, "solarShareNodes.csv"))
#	solarshare_df_30[:node] = solarshare_df_30[:node] .+1
    wind_df_30 = CSV.read(joinpath(newpath, "windHourly.csv"))
    windshare_df_30 = CSV.read(joinpath(newpath, "windShareNodes.csv"))
#	windshare_df_30[:node] = windshare_df_30[:node] .+ 1

    renewables_df_30 = DataFrame(node=nodes_df_30[:id])
    renewables_df_30[:SolarPV] = [maximum(solar_df_30[:solarhourly]*solarshare_df_30[i,2]) for i in 1:nrow(solarshare_df_30)]
    renewables_df_30[:Wind] = [maximum(wind_df_30[:windhourly]*windshare_df_30[i,2]) for i in 1:nrow(windshare_df_30)]
    renewables_df_30

    # avail_solar_pv_30 = # Dataframe mit nodalen, stündlichen relativen Verfügbarkeiten

    # avail_solar_pv_df
    avail_solar_pv_df_30 = DataFrame()
    for n in 1:nrow(solarshare_df_30)
        temparray = []
        for h in 1:8760
            renewables_df_30[:SolarPV][n] > 0 ? x = (solarshare_df_30[n,2]*solar_df_30[h,1])/renewables_df_30[n,2] : x = 0
            push!(temparray, x)
        end
        avail_solar_pv_df_30[Symbol(n)] = temparray
    end
	avail_solar_pv_df_30

    avail_wind_df_30 = DataFrame()
    for n in 1:nrow(windshare_df_30)
        temparray = []
        for h in 1:8760
            renewables_df_30[:Wind][n] > 0 ? x = (windshare_df_30[n,2]*wind_df_30[h,1])/renewables_df_30[n,3] : x = 0
            push!(temparray, x)
        end
        avail_wind_df_30[Symbol(n)] = temparray
    end
    avail_wind_df_30

    renewables_avail_dict_30 = Dict(:SolarPV => avail_solar_pv_df_30,
        :Wind => avail_wind_df_30)

		println("Reading of data complete.")
		println()

###################
# Model setup #####
###################

		println("Preparing model technologies, nodes, and grid...")
		prog = Progress(6)

	# Create sets
		# Create set of nodes and lines
		nodes = Nodes(nodes_df_30, load_df_30, exchange_df_30, 1000, 0)
			next!(prog)
		lines = Lines(lines_df_30, 1000, trm)
			next!(prog)
		# Create set of power plants	ED_L_S_opt, ED_p
		pp = PowerPlants(powerplants_df_30, avail_powerplants_df_30, mustrun_powerplants_df_30, fuelcost_df_30)
			next!(prog)
		# Create set of renewables
		res = Renewables(renewables_df_30, renewables_avail_dict_30)
			next!(prog)
		# Create a set of storages
		storages_df = CSV.read(joinpath(datapath, "storages.csv"))
			next!(prog)
		storages_df[:storage] = zeros(32)
		storages_df[:power] = zeros(32)
		storages = Storages(storages_df)
			next!(prog)
		sum(sum((res.infeed[i]) for i in keys(res.infeed)))
	# Create Dict Zone to Gen
		N = nodes.id
		P = pp.unit
		R = res.unit
		S = storages.unit
		Node2PP = Dict((i, []) for i in nodes.id)
		for i in keys(pp.node)
			push!(Node2PP[pp.node[i]], i)
		end
		Node2Res = Dict((i, []) for i in nodes.id)
		for i in keys(res.node)
			push!(Node2Res[res.node[i]], i)
		end
		Node2Stor = Dict((i, []) for i in nodes.id)
		for i in keys(storages.node)
			push!(Node2Stor[storages.node[i]], i)
		end

	println("Preparation complete.")
	println()

	println("Loading new load profiles.")
	Newloads_df = CSV.read(joinpath(inputpath, "Newload.csv"))

	newloads_dict = Dict((Newloads_df[r,1] => Newloads_df[r, 2:end]) for r in 1:nrow(Newloads_df))
	nodes.load = newloads_dict

		session_time = Dates.now()
		session = Dates.format(session_time, "yyyy-mm-dd__HH-MM-SS")*"_trm_" * string(trm)

	# Create session folder for output
		mkdir(joinpath(outputpath, session*"__"*timeperiod_str))

	prog = Progress(length(dayslicer(timeperiod)))

	### Parameter outputs
		println("Exporting parameters.")
		println("Exporting merit order...")
		# Merit order
		output_ED_merit_order_pp = DataFrame(unit=Symbol[],fuel=String[],capacity=Float64[],mc=Float64[])

		try
			for g in pp.unit
					mc = pp.mc[g]
					fuel = pp.fuel[g]
					capacity = pp.capacity[g]
					push!(output_ED_merit_order_pp, [g fuel capacity mc])
			end

			CSV.write(joinpath(outputpath, session*"__"*timeperiod_str, "ED_merit_order_pp.csv"), output_ED_merit_order_pp)
		catch e
			println("The marginal costs for power plants are not available.")
		end

		output_ED_merit_order_res = DataFrame(unit=Symbol[],fuel=Symbol[],capacity=Float64[],mc=Float64[])

		try
			for r in res.unit
					mc = 0
					fuel = res.fuel[r]
					capacity = res.capacity[r]
					push!(output_ED_merit_order_res, [r fuel capacity mc])
			end

			CSV.write(joinpath(outputpath, session*"__"*timeperiod_str, "ED_merit_order_res.csv"), output_ED_merit_order_res)
		catch e
			println("The marginal costs for renewables are not available.")
		end

		output_ED_merit_order_storages = DataFrame(unit=Symbol[],fuel=String[],capacity=Float64[],mc=Float64[])

		try
			for s in storages.unit
					mc = 0
					fuel = storages.fuel[s]
					capacity = storages.power[s]
					push!(output_ED_merit_order_storages, [s fuel capacity mc])
			end

			CSV.write(joinpath(outputpath, session*"__"*timeperiod_str, "ED_merit_order_storages.csv"), output_ED_merit_order_storages)
		catch e
			println("The marginal costs for renewables are not available.")
		end

	println("Export complete.")
	println()

### Initial run of first dayslice
	println("Initial run. Calculating first dayslice: " * string(dayslicer(timeperiod)[1]))
	println()
	println()

	ED_mod, ED_P_opt, ED_P_R_opt, ED_P_R_opt_dist, ED_P_S_opt, ED_D_S_opt,
		ED_L_S_opt, ED_price = EconomicDispatch80(dayslicer(timeperiod)[1], nodes, pp, res, storages)

	CM_mod, CM_ΔP_up_opt, CM_ΔP_dn_opt, CM_ΔP_R_up_opt, CM_ΔP_R_dn_opt,
		CM_Θ_opt, CM_P_flow_opt, CM_price, CM_P_gen_lost_opt, CM_P_load_lost_opt, CM_P_S_up_opt = CongestionManagementPHS80(dayslicer(timeperiod)[1],
		nodes, lines, pp, res, storages, ED_price, ED_P_opt, ED_P_R_opt_dist,
		ED_P_S_opt, ED_D_S_opt, ED_L_S_opt)

	next!(prog)

		C_V_RD_N = DataFrame(time=Int64[], node=Int64[], CostPP=Float64[], CostRes=Float64[], CostStor=Float64[], CostLostLoad=Float64[], CostLostGen=Float64[], CostTotal=Float64[],
			VolumePP=Float64[], VolumeRes=Float64[], VolumeStor=Float64[], VolumeLostLoad=Float64[], VolumeLostGen=Float64[], VolumeTotal=Float64[])
			for t in dayslicer(timeperiod)[1]
				for n in N
					if isempty(Node2PP[n])
						costpp = 0
					else
						costpp = sum((pp.mc[g][1]*CM_ΔP_up_opt[t, g] + (ED_price[t] - pp.mc[g][]) * CM_ΔP_dn_opt[t, g]) for g in Node2PP[n])
					end
					if isempty(Node2Res[n])
						costres = 0
					else
						costres = sum(((ED_price[t]) * CM_ΔP_R_dn_opt[t, res]) for res in Node2Res[n])
					end
					if isempty(Node2Stor[n])
						coststor = 0
					else
						coststor = sum(((ED_price[t]/storages.efficiency[stor]) * CM_P_S_up_opt[t, stor]) for stor in Node2Stor[n])
					end

					costlostload = 1000*CM_P_load_lost_opt[t, n]
					costlostgen = 1000*CM_P_gen_lost_opt[t, n] # ÜBERALL NULL!
					totalcost = costpp + costres + coststor + costlostload + costlostgen

					if isempty(Node2PP[n])
						volumepp = 0
					else
						volumepp = sum((CM_ΔP_up_opt[t, g] - CM_ΔP_dn_opt[t, g]) for g in Node2PP[n])
					end
					if isempty(Node2Res[n])
						volumeres = 0
					else
						volumeres = sum(-(CM_ΔP_R_dn_opt[t, res]) for res in Node2Res[n])
					end
					if isempty(Node2Stor[n])
						volumestor = 0
					else
						volumestor = sum((CM_P_S_up_opt[t, stor]) for stor in Node2Stor[n])
					end

					volumelostload = CM_P_load_lost_opt[t, n]
					volumelostgen = CM_P_gen_lost_opt[t, n]
					totalvolume = volumepp + volumeres + volumestor + volumelostload + volumelostgen

					push!(C_V_RD_N, [t n costpp costres coststor costlostload costlostgen totalcost volumepp volumeres volumestor volumelostload volumelostgen totalvolume])
				end
			end
			CSV.write(joinpath(outputpath, session*"__"*timeperiod_str, "C_V_RD_N__"*timeperiod_str*".csv"), C_V_RD_N)

	# Save ED Data
		# Dispatchable power plants
		output_ED_P_opt = DataFrame(time=Int64[],unit=Symbol[],fuel=String[],output=Float64[],node=Int64[])

		try
			for t in dayslicer(timeperiod)[1]
				for g in pp.unit
					val = ED_P_opt[t, g]
					fuel =  pp.fuel[g]
					push!(output_ED_P_opt, [t g fuel val pp.node[g]])
				end
			end
			CSV.write(joinpath(outputpath, session*"__"*timeperiod_str, "ED_P.csv"), output_ED_P_opt)
		catch e
			println("ED_P_opt does not exist.")
		end

		# RES generation
		output_ED_P_R_opt = DataFrame(time=Int64[],unit=Symbol[],fuel=Symbol[],output=Float64[],node=Int64[])
		try
			for t in dayslicer(timeperiod)[1]
				for r in res.unit
					val = ED_P_R_opt[t, r]
					fuel =  res.fuel[r]
					push!(output_ED_P_R_opt, [t r fuel val res.node[r]])
				end
			end
			CSV.write(joinpath(outputpath, session*"__"*timeperiod_str, "ED_P_R.csv"), output_ED_P_R_opt)
		catch e
			println("ED_P_R_opt does not exist.")
		end

		# Maximum available RES infeed from input data
		output_MaxRES = DataFrame(time=Int64[],unit=Symbol[],fuel=Symbol[],output=Float64[],node=Int64[])
		try
			for t in dayslicer(timeperiod)[1]
				for r in res.unit
					output = res.infeed[r][t]
					fuel =  res.fuel[r]
					push!(output_MaxRES, [t r fuel output res.node[r]])
				end
			end
			CSV.write(joinpath(outputpath, session*"__"*timeperiod_str, "MaxRES.csv"), output_MaxRES)
		catch e
			println("res.infeed does not exist.")
		end

		# Generation from storages
		output_ED_P_S_opt = DataFrame(time=Int64[],unit=Symbol[],fuel=String[],output=Float64[],node=Int64[])
		try
			for t in dayslicer(timeperiod)[1]
				for s in storages.unit
					val = ED_P_S_opt[t, s]
					fuel =  storages.fuel[s]
					push!(output_ED_P_S_opt, [t s fuel val storages.node[s]])
				end
			end
			CSV.write(joinpath(outputpath, session*"__"*timeperiod_str, "ED_P_S.csv"), output_ED_P_S_opt)
		catch e
			println("ED_P_S_opt does not exist.")
		end

		# Electricity demand from storages
		output_ED_D_S_opt = DataFrame(time=Int64[],unit=Symbol[],fuel=String[],output=Float64[],node=Int64[])
		try
			for t in dayslicer(timeperiod)[1]
				for s in storages.unit
					val = -1*ED_D_S_opt[t, s]
					fuel =  storages.fuel[s]
					push!(output_ED_D_S_opt, [t s fuel val storages.node[s]])
				end
			end
			CSV.write(joinpath(outputpath, session*"__"*timeperiod_str, "ED_D_S.csv"), output_ED_D_S_opt)
		catch e
			println("ED_D_S_opt does not exist.")
		end

		# Storage level
		output_ED_L_S_opt = DataFrame(time=Int64[],unit=Symbol[],fuel=String[],storage=Float64[],node=Int64[])
		try
			for t in dayslicer(timeperiod)[1]
				for s in storages.unit
					val = ED_L_S_opt[t, s]
					fuel =  storages.fuel[s]
					push!(output_ED_L_S_opt, [t s fuel val storages.node[s]])
				end
			end
			CSV.write(joinpath(outputpath, session*"__"*timeperiod_str, "ED_L_S.csv"), output_ED_L_S_opt)
		catch e
			println("ED_L_S_opt does not exist.")
		end

		# Load
		output_load = DataFrame(time=Int64[],load=Float64[])
		try
			for t in dayslicer(timeperiod)[1]
				load = nodes.systemload[t]
				push!(output_load, [t load])
			end
			CSV.write(joinpath(outputpath, session*"__"*timeperiod_str, "load.csv"), output_load)
		catch e
			println("nodes.systemload does not exist.")
		end

		# Prices
		output_ED_price = DataFrame(time=Int64[],price=Float64[])
		try
			for t in dayslicer(timeperiod)[1]
				price =  ED_price[t]
				push!(output_ED_price, [t price])
			end
			CSV.write(joinpath(outputpath, session*"__"*timeperiod_str, "ED_price.csv"), output_ED_price)
		catch e
			println("ED_price does not exist.")
		end

		# Must-run
		output_pmin = DataFrame(time=Int64[],unit=Symbol[],fuel=String[],output=Float64[],node=Int64[])
		try
			for t in dayslicer(timeperiod)[1]
				for g in pp.unit
					val = pp.pmin[g][t]
					fuel =  pp.fuel[g]
					push!(output_pmin, [t g fuel val pp.node[g]])
				end
			end
			CSV.write(joinpath(outputpath, session*"__"*timeperiod_str, "pmin.csv"), output_pmin)
		catch e
			println("pp.pmin does not exist.")
		end

		### Model parameters
		output_ED_info = DataFrame(timeperiod=String[],
										termination_status=String[],
										primal_status=String[],
										dual_status=String[],
										objective_value=Float64[],
										solve_time=Float64[])

		push!(output_ED_info, [string(dayslicer(timeperiod)[1]) string(termination_status(ED_mod)) string(primal_status(ED_mod)) string(dual_status(ED_mod)) objective_value(ED_mod) solve_time(ED_mod)])
		CSV.write(joinpath(outputpath, session*"__"*timeperiod_str, "ED_info.csv"), output_ED_info)

		# Nodal exchange
		output_exchange = DataFrame(time=Int64[],node=Int64[],output=Float64[])
		try
			for t in dayslicer(timeperiod)[1]
				for j in nodes.id
					push!(output_exchange, [t j nodes.exchange[j][t]])
				end
			end
			CSV.write(joinpath(outputpath, session*"__"*timeperiod_str, "exchange.csv"), output_exchange)
		catch e
			println("output_exchange does not exist.")
		end

###################
# CM ##############
###################
# Save CM Data
	# Dispatchable power plants
	output_CM_ΔP_up_opt = DataFrame(time=Int64[],unit=Symbol[],fuel=String[],output=Float64[],node=Int64[])
	try
		for t in dayslicer(timeperiod)[1]
			for g in pp.unit
				val = CM_ΔP_up_opt[t, g]
				fuel =  pp.fuel[g]
				push!(output_CM_ΔP_up_opt, [t g fuel val pp.node[g]])
			end
		end
		CSV.write(joinpath(outputpath, session*"__"*timeperiod_str, "CM_P_up.csv"), output_CM_ΔP_up_opt)
	catch e
		println("CM_ΔP_up_opt does not exist.")
	end

	output_CM_ΔP_dn_opt = DataFrame(time=Int64[],unit=Symbol[],fuel=String[],output=Float64[],node=Int64[])
	try
		for t in dayslicer(timeperiod)[1]
			for g in pp.unit
				val = -1*CM_ΔP_dn_opt[t, g]
				fuel =  pp.fuel[g]
				push!(output_CM_ΔP_dn_opt, [t g fuel val pp.node[g]])
			end
		end
		CSV.write(joinpath(outputpath, session*"__"*timeperiod_str, "CM_P_dn.csv"), output_CM_ΔP_dn_opt)
	catch e
		println("CM_ΔP_dn_opt does not exist.")
	end

	# RES generation
	output_CM_ΔP_R_up_opt = DataFrame(time=Int64[],unit=Symbol[],fuel=Symbol[],output=Float64[],node=Int64[])
	try
		for t in dayslicer(timeperiod)[1]
			for r in res.unit
				val = CM_ΔP_R_up_opt[t, r]
				fuel =  res.fuel[r]
				push!(output_CM_ΔP_R_up_opt, [t r fuel val res.node[r]])
			end
		end
		CSV.write(joinpath(outputpath, session*"__"*timeperiod_str, "CM_P_R_up.csv"), output_CM_ΔP_R_up_opt)
	catch e
		println("CM_ΔP_R_up_opt does not exist.")
	end

	output_CM_ΔP_R_dn_opt = DataFrame(time=Int64[],unit=Symbol[],fuel=Symbol[],output=Float64[],node=Int64[])
	try
		for t in dayslicer(timeperiod)[1]
			for r in res.unit
				val = -1*CM_ΔP_R_dn_opt[t, r]
				fuel =  res.fuel[r]
				push!(output_CM_ΔP_R_dn_opt, [t r fuel val res.node[r]])
			end
		end
		CSV.write(joinpath(outputpath, session*"__"*timeperiod_str, "CM_P_R_dn.csv"), output_CM_ΔP_R_dn_opt)
	catch e
		println("CM_ΔP_R_dn_opt does not exist.")
	end

	### Model parameters
	output_CM_info = DataFrame(timeperiod=String[],
									   termination_status=String[],
									   primal_status=String[],
									   dual_status=String[],
									   objective_value=Float64[],
									   solve_time=Float64[])

	push!(output_CM_info, [string(dayslicer(timeperiod)[1]) string(termination_status(CM_mod)) string(primal_status(CM_mod)) string(dual_status(CM_mod)) objective_value(CM_mod) solve_time(CM_mod)])
	CSV.write(joinpath(outputpath, session*"__"*timeperiod_str, "CM_info.csv"), output_CM_info)

	### Power flow results
	output_CM_P_flow_opt = DataFrame(time=Int64[],
									 line=Int64[],
									 from=Int64[],
									 to=Int64[],
									 pmax=Float64[],
									 pflow=Float64[])

	 try
 		for t in dayslicer(timeperiod)[1]
 			for l in lines.id
				from = parse(Int, string(lines.from[l]))
				to = parse(Int, string(lines.to[l]))
				pmax = lines.pmax[l]
				pflow = CM_P_flow_opt[t, l]
 				push!(output_CM_P_flow_opt, [t l from to pmax pflow])
 			end
 		end
 		CSV.write(joinpath(outputpath, session*"__"*timeperiod_str, "CM_P_flow.csv"), output_CM_P_flow_opt)
 	catch e
 		println("CM_P_flow_opt does not exist.")
 	end

	# Lost generation
	output_CM_P_gen_lost = DataFrame(time=Int64[],node=Int64[],output=Float64[])
	try
		for t in dayslicer(timeperiod)[1]
			for j in nodes.id
				output = CM_P_gen_lost_opt[t, j]
				push!(output_CM_P_gen_lost, [t, j, output])
			end
		end
		CSV.write(joinpath(outputpath, session*"__"*timeperiod_str, "CM_P_gen_lost.csv"), output_CM_P_gen_lost)
	catch e
		println("CM_P_gen_lost does not exist.")
	end

	# Lost load
	output_CM_P_load_lost = DataFrame(time=Int64[],node=Int64[],output=Float64[])
	try
		for t in dayslicer(timeperiod)[1]
			for j in nodes.id
				output = CM_P_load_lost_opt[t, j]
				push!(output_CM_P_load_lost, [t, j, output])
			end
		end
		CSV.write(joinpath(outputpath, session*"__"*timeperiod_str, "CM_P_load_lost.csv"), output_CM_P_load_lost)
	catch e
		println("CM_P_load_lost does not exist.")
	end

	output_CM_price = DataFrame(time=Int64[],node=Int64[],price=Float64[])
	 try
		 for t in dayslicer(timeperiod)[1]
			 for j in nodes.id
				 price =  CM_price[j, t]
				 push!(output_CM_price, [t j price])
			 end
		 end
		 CSV.write(joinpath(outputpath, session*"__"*timeperiod_str, "CM_price.csv"), output_CM_price)
	 catch e
		 println("CM_price does not exist.")
	 end

	# PHS output increase
	output_CM_P_S_up_opt = DataFrame(time=Int64[],unit=Symbol[],fuel=String[],output=Float64[],node=Int64[])
	try
		for t in dayslicer(timeperiod)[1]
			for s in storages.unit
				val = CM_P_S_up_opt[t, s]
				fuel =  storages.fuel[s]
				push!(output_CM_P_S_up_opt, [t s fuel val storages.node[s]])
			end
		end
		CSV.write(joinpath(outputpath, session*"__"*timeperiod_str, "CM_P_S_up.csv"), output_CM_P_S_up_opt)
	catch e
		println("CM_P_S_up_opt does not exist.")
	end

### Remaining periods: Initialising first hour with last hour of previous day
# 	and append data

	for hours in dayslicer(timeperiod)[2:end]
		hours_extended = [hours[1]-1; hours]

	println()
	println()
	println("Calculating economic dispatch: " * string(hours))

	global ED_mod, ED_P_opt, ED_P_R_opt, ED_P_R_opt_dist, ED_P_S_opt, ED_D_S_opt,
		ED_L_S_opt, ED_price = EconomicDispatch80(hours, nodes, pp, res, storages)

	    ###################
    # ED ##############
	    ###################

	    # Dispatchable power plants
	    output_ED_P_opt = DataFrame(time=Int64[],unit=Symbol[],fuel=String[],output=Float64[],node=Int64[])

	    try
	    	for t in hours
	    		for g in pp.unit
	    			val = ED_P_opt[t, g]
	    			fuel =  pp.fuel[g]
	    			push!(output_ED_P_opt, [t g fuel val pp.node[g]])
	    		end
	    	end
	    	CSV.write(joinpath(outputpath, session*"__"*timeperiod_str, "ED_P.csv"), output_ED_P_opt, append = true)
	    catch e
	    	println("ED_P_opt does not exist.")
	    end

	    # RES generation
	    output_ED_P_R_opt = DataFrame(time=Int64[],unit=Symbol[],fuel=Symbol[],output=Float64[],node=Int64[])
	    try
	    	for t in hours
	    		for r in res.unit
	    			val = ED_P_R_opt[t, r]
	    			fuel =  res.fuel[r]
	    			push!(output_ED_P_R_opt, [t r fuel val res.node[r]])
	    		end
	    	end
	    	CSV.write(joinpath(outputpath, session*"__"*timeperiod_str, "ED_P_R.csv"), output_ED_P_R_opt, append = true)
	    catch e
	    	println("ED_P_R_opt does not exist.")
	    end

	    # Maximum available RES infeed from input data
	    output_MaxRES = DataFrame(time=Int64[],unit=Symbol[],fuel=Symbol[],output=Float64[],node=Int64[])
	    try
	    	for t in hours
	    		for r in res.unit
	    			output = res.infeed[r][t]
	    			fuel =  res.fuel[r]
	    			push!(output_MaxRES, [t r fuel output res.node[r]])
	    		end
	    	end
	    	CSV.write(joinpath(outputpath, session*"__"*timeperiod_str, "MaxRES.csv"), output_MaxRES, append = true)
	    catch e
	    	println("res.infeed does not exist.")
	    end

	    # Generation from storages
	    output_ED_P_S_opt = DataFrame(time=Int64[],unit=Symbol[],fuel=String[],output=Float64[],node=Int64[])
	    try
	    	for t in hours
	    		for s in storages.unit
	    			val = ED_P_S_opt[t, s]
	    			fuel =  storages.fuel[s]
	    			push!(output_ED_P_S_opt, [t s fuel val storages.node[s]])
	    		end
	    	end
	    	CSV.write(joinpath(outputpath, session*"__"*timeperiod_str, "ED_P_S.csv"), output_ED_P_S_opt, append = true)
	    catch e
	    	println("ED_P_S_opt does not exist.")
	    end

	    # Electricity demand from storages
	    output_ED_D_S_opt = DataFrame(time=Int64[],unit=Symbol[],fuel=String[],output=Float64[],node=Int64[])
	    try
	    	for t in hours
	    		for s in storages.unit
	    			val = -1*ED_D_S_opt[t, s]
	    			fuel =  storages.fuel[s]
	    			push!(output_ED_D_S_opt, [t s fuel val storages.node[s]])
	    		end
	    	end
	    	CSV.write(joinpath(outputpath, session*"__"*timeperiod_str, "ED_D_S.csv"), output_ED_D_S_opt, append = true)
	    catch e
	    	println("ED_D_S_opt does not exist.")
	    end

	    # Storage level
	    output_ED_L_S_opt = DataFrame(time=Int64[],unit=Symbol[],fuel=String[],storage=Float64[],node=Int64[])
	    try
	    	for t in hours
	    		for s in storages.unit
	    			val = ED_L_S_opt[t, s]
	    			fuel =  storages.fuel[s]
	    			push!(output_ED_L_S_opt, [t s fuel val storages.node[s]])
	    		end
	    	end
	    	CSV.write(joinpath(outputpath, session*"__"*timeperiod_str, "ED_L_S.csv"), output_ED_L_S_opt, append = true)
	    catch e
	    	println("ED_L_S_opt does not exist.")
	    end

	    # Load
	    output_load = DataFrame(time=Int64[],load=Float64[])
	    try
	    	for t in hours
	    		load = nodes.systemload[t]
	    		push!(output_load, [t load])
	    	end
	    	CSV.write(joinpath(outputpath, session*"__"*timeperiod_str, "load.csv"), output_load, append = true)
	    catch e
	    	println("nodes.systemload does not exist.")
	    end

	    # Prices
	    output_ED_price = DataFrame(time=Int64[],price=Float64[])
	    try
	    	for t in hours
	    		price =  ED_price[t]
	    		push!(output_ED_price, [t price])
	    	end
	    	CSV.write(joinpath(outputpath, session*"__"*timeperiod_str, "ED_price.csv"), output_ED_price, append = true)
	    catch e
	    	println("ED_price does not exist.")
	    end

	    # Must-run
	    output_pmin = DataFrame(time=Int64[],unit=Symbol[],fuel=String[],output=Float64[],node=Int64[])
	    try
	    	for t in hours
	    		for g in pp.unit
	    			val = pp.pmin[g][t]
	    			fuel =  pp.fuel[g]
	    			push!(output_pmin, [t g fuel val pp.node[g]])
	    		end
	    	end
	    	CSV.write(joinpath(outputpath, session*"__"*timeperiod_str, "pmin.csv"), output_pmin, append = true)
	    catch e
	    	println("pp.pmin does not exist.")
	    end

		### Model parameters
		output_ED_info = DataFrame(timeperiod=String[],
										   termination_status=String[],
										   primal_status=String[],
										   dual_status=String[],
										   objective_value=Float64[],
										   solve_time=Float64[])

		push!(output_ED_info, [string(hours) string(termination_status(ED_mod)) string(primal_status(ED_mod)) string(dual_status(ED_mod)) objective_value(ED_mod) solve_time(ED_mod)])
		CSV.write(joinpath(outputpath, session*"__"*timeperiod_str, "ED_info.csv"), output_ED_info, append = true)

		###################
# CM ##############
		###################

		println()
		println()
		println("Calculating congestion management: " * string(hours))

			global CM_mod, CM_ΔP_up_opt, CM_ΔP_dn_opt, CM_ΔP_R_up_opt, CM_ΔP_R_dn_opt,
				CM_Θ_opt, CM_P_flow_opt, CM_price, CM_P_gen_lost_opt, CM_P_load_lost_opt, CM_P_S_up_opt = CongestionManagementPHS80(hours, nodes, lines, pp, res, storages, ED_price,
				ED_P_opt,  ED_P_R_opt_dist, ED_P_S_opt, ED_D_S_opt, ED_L_S_opt)

		C_V_RD_N = DataFrame(time=Int64[], node=Int64[], CostPP=Float64[], CostRes=Float64[], CostStor=Float64[], CostLostLoad=Float64[], CostLostGen=Float64[], CostTotal=Float64[],
			VolumePP=Float64[], VolumeRes=Float64[], VolumeStor=Float64[], VolumeLostLoad=Float64[], VolumeLostGen=Float64[], VolumeTotal=Float64[])
	    for t in hours
			for n in N
				if isempty(Node2PP[n])
					costpp = 0
				else
					costpp = sum((pp.mc[g][1]*CM_ΔP_up_opt[t, g] + (ED_price[t] - pp.mc[g][]) * CM_ΔP_dn_opt[t, g]) for g in Node2PP[n])
				end
				if isempty(Node2Res[n])
					costres = 0
				else
					costres = sum(((ED_price[t]) * CM_ΔP_R_dn_opt[t, res]) for res in Node2Res[n])
				end
				if isempty(Node2Stor[n])
					coststor = 0
				else
					coststor = sum(((ED_price[t]/storages.efficiency[stor]) * CM_P_S_up_opt[t, stor]) for stor in Node2Stor[n])
				end

				costlostload = 1000*CM_P_load_lost_opt[t, n]
				costlostgen = 1000*CM_P_gen_lost_opt[t, n] # ÜBERALL NULL!
				totalcost = costpp + costres + coststor + costlostload + costlostgen

				if isempty(Node2PP[n])
					volumepp = 0
				else
					volumepp = sum((CM_ΔP_up_opt[t, g] - CM_ΔP_dn_opt[t, g]) for g in Node2PP[n])
	    #            VolumePP = sum((pp.mc[g][1]*CM_ΔP_up_opt[t, g] + (ED_price[t] - pp.mc[g][]) * CM_ΔP_dn_opt[t, g]) for g in Node2PP[n])
				end
				if isempty(Node2Res[n])
					volumeres = 0
				else
					volumeres = sum(-(CM_ΔP_R_dn_opt[t, res]) for res in Node2Res[n])
				end
				if isempty(Node2Stor[n])
					volumestor = 0
				else
					volumestor = sum((CM_P_S_up_opt[t, stor]) for stor in Node2Stor[n])
				end

				volumelostload = CM_P_load_lost_opt[t, n]
				volumelostgen = CM_P_gen_lost_opt[t, n]
				totalvolume = volumepp + volumeres + volumestor + volumelostload + volumelostgen

				push!(C_V_RD_N, [t n costpp costres coststor costlostload costlostgen totalcost volumepp volumeres volumestor volumelostload volumelostgen totalvolume])
			end
		end
		CSV.write(joinpath(outputpath, session*"__"*timeperiod_str, "C_V_RD_N__"*timeperiod_str*".csv"), C_V_RD_N, append = true)

			# Dispatchable power plants
			output_CM_ΔP_up_opt = DataFrame(time=Int64[],unit=Symbol[],fuel=String[],output=Float64[],node=Int64[])
			try
				for t in hours
					for g in pp.unit
						val = CM_ΔP_up_opt[t, g]
						fuel =  pp.fuel[g]
						push!(output_CM_ΔP_up_opt, [t g fuel val pp.node[g]])
					end
				end
				CSV.write(joinpath(outputpath, session*"__"*timeperiod_str, "CM_P_up.csv"), output_CM_ΔP_up_opt, append = true)
			catch e
				println("CM_ΔP_up_opt does not exist.")
			end

			output_CM_ΔP_dn_opt = DataFrame(time=Int64[],unit=Symbol[],fuel=String[],output=Float64[],node=Int64[])
			try
				for t in hours
					for g in pp.unit
						val = -1*CM_ΔP_dn_opt[t, g]
						fuel =  pp.fuel[g]
						push!(output_CM_ΔP_dn_opt, [t g fuel val pp.node[g]])
					end
				end
				CSV.write(joinpath(outputpath, session*"__"*timeperiod_str, "CM_P_dn.csv"), output_CM_ΔP_dn_opt, append = true)
			catch e
				println("CM_ΔP_dn_opt does not exist.")
			end

			# RES generation
			output_CM_ΔP_R_up_opt = DataFrame(time=Int64[],unit=Symbol[],fuel=Symbol[],output=Float64[],node=Int64[])
			try
				for t in hours
					for r in res.unit
						val = CM_ΔP_R_up_opt[t, r]
						fuel =  res.fuel[r]
						push!(output_CM_ΔP_R_up_opt, [t r fuel val res.node[r]])
					end
				end
				CSV.write(joinpath(outputpath, session*"__"*timeperiod_str, "CM_P_R_up.csv"), output_CM_ΔP_R_up_opt, append = true)
			catch e
				println("CM_ΔP_R_up_opt does not exist.")
			end

			output_CM_ΔP_R_dn_opt = DataFrame(time=Int64[],unit=Symbol[],fuel=Symbol[],output=Float64[],node=Int64[])
			try
				for t in hours
					for r in res.unit
						val = -1*CM_ΔP_R_dn_opt[t, r]
						fuel =  res.fuel[r]
						push!(output_CM_ΔP_R_dn_opt, [t r fuel val res.node[r]])
					end
				end
				CSV.write(joinpath(outputpath, session*"__"*timeperiod_str, "CM_P_R_dn.csv"), output_CM_ΔP_R_dn_opt, append = true)
			catch e
				println("CM_ΔP_R_dn_opt does not exist.")
			end

			### Model parameters
			output_CM_info = DataFrame(timeperiod=String[],
											   termination_status=String[],
											   primal_status=String[],
											   dual_status=String[],
											   objective_value=Float64[],
											   solve_time=Float64[])

			push!(output_CM_info, [string(hours) string(termination_status(CM_mod)) string(primal_status(CM_mod)) string(dual_status(CM_mod)) objective_value(CM_mod) solve_time(CM_mod)])
			CSV.write(joinpath(outputpath, session*"__"*timeperiod_str, "CM_info.csv"), output_CM_info, append = true)

			### Power flow results
			output_CM_P_flow_opt = DataFrame(time=Int64[],
											 line=Int64[],
											 from=Int64[],
											 to=Int64[],
											 pmax=Float64[],
											 pflow=Float64[])

			try
			   for t in hours
				   for l in lines.id
					   from = parse(Int, string(lines.from[l]))
					   to = parse(Int, string(lines.to[l]))
					   pmax = lines.pmax[l]
					   pflow = CM_P_flow_opt[t, l]
					   push!(output_CM_P_flow_opt, [t l from to pmax pflow])
				   end
			   end
			   CSV.write(joinpath(outputpath, session*"__"*timeperiod_str, "CM_P_flow.csv"), output_CM_P_flow_opt, append = true)
		   catch e
			   println("CM_P_flow_opt does not exist.")
		   end

		   # Lost generation
		   output_CM_P_gen_lost = DataFrame(time=Int64[],node=Int64[],output=Float64[])
		   try
			   for t in hours
				   for j in nodes.id
					   output = CM_P_gen_lost_opt[t, j]
					   push!(output_CM_P_gen_lost, [t, j, output])
				   end
			   end
			   CSV.write(joinpath(outputpath, session*"__"*timeperiod_str, "CM_P_gen_lost.csv"), output_CM_P_gen_lost, append = true)
		   catch e
			   println("CM_P_gen_lost does not exist.")
		   end

		   # Lost load
		   output_CM_P_load_lost = DataFrame(time=Int64[],node=Int64[],output=Float64[])
		   try
			   for t in hours
				   for j in nodes.id
					   output = CM_P_load_lost_opt[t, j]
					   push!(output_CM_P_load_lost, [t, j, output])
				   end
			   end
			   CSV.write(joinpath(outputpath, session*"__"*timeperiod_str, "CM_P_load_lost.csv"), output_CM_P_load_lost, append = true)
		   catch e
			   println("CM_P_load_lost does not exist.")
		   end

		   # Noral Price
		   output_CM_price = DataFrame(time=Int64[],node=Int64[],price=Float64[])
	   	    try
	   	    	for t in hours
					for j in nodes.id
		   	    		price =  CM_price[j, t]
		   	    		push!(output_CM_price, [t j price])
					end
	   	    	end
	   	    	CSV.write(joinpath(outputpath, session*"__"*timeperiod_str, "CM_price.csv"), output_CM_price, append = true)
	   	    catch e
	   	    	println("CM_price does not exist.")
	   	    end

			output_CM_P_S_up_opt = DataFrame(time=Int64[],unit=Symbol[],fuel=String[],output=Float64[],node=Int64[])
			try
				for t in hours
					for s in storages.unit
						val = CM_P_S_up_opt[t, s]
						fuel =  storages.fuel[s]
						push!(output_CM_P_S_up_opt, [t s fuel val storages.node[s]])
					end
				end
				CSV.write(joinpath(outputpath, session*"__"*timeperiod_str, "CM_P_S_up.csv"), output_CM_P_S_up_opt, append = true)
			catch e
				println("CM_P_S_up_opt does not exist.")
			end

			# Nodal exchange
			output_exchange = DataFrame(time=Int64[],node=Int64[],output=Float64[])
			try
				for t in hours
					for j in nodes.id
						push!(output_exchange, [t j nodes.exchange[j][t]])
					end
				end
				CSV.write(joinpath(outputpath, session*"__"*timeperiod_str, "exchange.csv"), output_exchange, append = true)
			catch e
				println("output_exchange does not exist.")
			end

	next!(prog)

end

println()
println("Calculation complete.")


function APMFile()
    return APMFile(())
end

function add_maneuver(obj::APMFile, arg0::APMFile_APMManeuver)
    return jcall(obj, "addManeuver", void, (APMFile_APMManeuver,), arg0)
end

function get_attitude_quaternion_direction(obj::APMFile)
    return jcall(obj, "getAttitudeQuaternionDirection", JString, ())
end

function get_epoch(obj::APMFile)
    return jcall(obj, "getEpoch", AbsoluteDate, ())
end

function get_epoch_comment(obj::APMFile)
    return jcall(obj, "getEpochComment", List, ())
end

function get_euler_comment(obj::APMFile)
    return jcall(obj, "getEulerComment", List, ())
end

function get_euler_direction(obj::APMFile)
    return jcall(obj, "getEulerDirection", JString, ())
end

function get_euler_frame_a_string(obj::APMFile)
    return jcall(obj, "getEulerFrameAString", JString, ())
end

function get_euler_frame_b_string(obj::APMFile)
    return jcall(obj, "getEulerFrameBString", JString, ())
end

function get_euler_rot_seq(obj::APMFile)
    return jcall(obj, "getEulerRotSeq", JString, ())
end

function get_has_maneuver(obj::APMFile)
    return jcall(obj, "getHasManeuver", jboolean, ())
end

function get_i11(obj::APMFile)
    return jcall(obj, "getI11", jdouble, ())
end

function get_i12(obj::APMFile)
    return jcall(obj, "getI12", jdouble, ())
end

function get_i13(obj::APMFile)
    return jcall(obj, "getI13", jdouble, ())
end

function get_i22(obj::APMFile)
    return jcall(obj, "getI22", jdouble, ())
end

function get_i23(obj::APMFile)
    return jcall(obj, "getI23", jdouble, ())
end

function get_i33(obj::APMFile)
    return jcall(obj, "getI33", jdouble, ())
end

function get_inertia_ref_frame_string(obj::APMFile)
    return jcall(obj, "getInertiaRefFrameString", JString, ())
end

function get_maneuver(obj::APMFile, arg0::jint)
    return jcall(obj, "getManeuver", APMFile_APMManeuver, (jint,), arg0)
end

function get_maneuvers(obj::APMFile)
    return jcall(obj, "getManeuvers", List, ())
end

function get_meta_data(obj::APMFile)
    return jcall(obj, "getMetaData", ADMMetaData, ())
end

function get_meta_data_comment(obj::APMFile)
    return jcall(obj, "getMetaDataComment", List, ())
end

function get_nb_maneuvers(obj::APMFile)
    return jcall(obj, "getNbManeuvers", jint, ())
end

function get_nutation(obj::APMFile)
    return jcall(obj, "getNutation", jdouble, ())
end

function get_nutation_period(obj::APMFile)
    return jcall(obj, "getNutationPeriod", jdouble, ())
end

function get_nutation_phase(obj::APMFile)
    return jcall(obj, "getNutationPhase", jdouble, ())
end

function get_quaternion(obj::APMFile)
    return jcall(obj, "getQuaternion", Quaternion, ())
end

function get_quaternion_dot(obj::APMFile)
    return jcall(obj, "getQuaternionDot", Quaternion, ())
end

function get_quaternion_frame_a_string(obj::APMFile)
    return jcall(obj, "getQuaternionFrameAString", JString, ())
end

function get_quaternion_frame_b_string(obj::APMFile)
    return jcall(obj, "getQuaternionFrameBString", JString, ())
end

function get_rate_frame_string(obj::APMFile)
    return jcall(obj, "getRateFrameString", JString, ())
end

function get_rotation_angles(obj::APMFile)
    return jcall(obj, "getRotationAngles", Vector3D, ())
end

function get_rotation_rates(obj::APMFile)
    return jcall(obj, "getRotationRates", Vector3D, ())
end

function get_spacecraft_comment(obj::APMFile)
    return jcall(obj, "getSpacecraftComment", List, ())
end

function get_spin_alpha(obj::APMFile)
    return jcall(obj, "getSpinAlpha", jdouble, ())
end

function get_spin_angle(obj::APMFile)
    return jcall(obj, "getSpinAngle", jdouble, ())
end

function get_spin_angle_vel(obj::APMFile)
    return jcall(obj, "getSpinAngleVel", jdouble, ())
end

function get_spin_comment(obj::APMFile)
    return jcall(obj, "getSpinComment", List, ())
end

function get_spin_delta(obj::APMFile)
    return jcall(obj, "getSpinDelta", jdouble, ())
end

function get_spin_direction(obj::APMFile)
    return jcall(obj, "getSpinDirection", JString, ())
end

function get_spin_frame_a_string(obj::APMFile)
    return jcall(obj, "getSpinFrameAString", JString, ())
end

function get_spin_frame_b_string(obj::APMFile)
    return jcall(obj, "getSpinFrameBString", JString, ())
end

function set_attitude_quaternion_direction(obj::APMFile, arg0::JString)
    return jcall(obj, "setAttitudeQuaternionDirection", void, (JString,), arg0)
end

function set_epoch(obj::APMFile, arg0::AbsoluteDate)
    return jcall(obj, "setEpoch", void, (AbsoluteDate,), arg0)
end

function set_epoch_comment(obj::APMFile, arg0::List)
    return jcall(obj, "setEpochComment", void, (List,), arg0)
end

function set_euler_comment(obj::APMFile, arg0::List)
    return jcall(obj, "setEulerComment", void, (List,), arg0)
end

function set_euler_direction(obj::APMFile, arg0::JString)
    return jcall(obj, "setEulerDirection", void, (JString,), arg0)
end

function set_euler_frame_a_string(obj::APMFile, arg0::JString)
    return jcall(obj, "setEulerFrameAString", void, (JString,), arg0)
end

function set_euler_frame_b_string(obj::APMFile, arg0::JString)
    return jcall(obj, "setEulerFrameBString", void, (JString,), arg0)
end

function set_euler_rot_seq(obj::APMFile, arg0::JString)
    return jcall(obj, "setEulerRotSeq", void, (JString,), arg0)
end

function set_i11(obj::APMFile, arg0::jdouble)
    return jcall(obj, "setI11", void, (jdouble,), arg0)
end

function set_i12(obj::APMFile, arg0::jdouble)
    return jcall(obj, "setI12", void, (jdouble,), arg0)
end

function set_i13(obj::APMFile, arg0::jdouble)
    return jcall(obj, "setI13", void, (jdouble,), arg0)
end

function set_i22(obj::APMFile, arg0::jdouble)
    return jcall(obj, "setI22", void, (jdouble,), arg0)
end

function set_i23(obj::APMFile, arg0::jdouble)
    return jcall(obj, "setI23", void, (jdouble,), arg0)
end

function set_i33(obj::APMFile, arg0::jdouble)
    return jcall(obj, "setI33", void, (jdouble,), arg0)
end

function set_inertia_ref_frame_string(obj::APMFile, arg0::JString)
    return jcall(obj, "setInertiaRefFrameString", void, (JString,), arg0)
end

function set_nutation(obj::APMFile, arg0::jdouble)
    return jcall(obj, "setNutation", void, (jdouble,), arg0)
end

function set_nutation_period(obj::APMFile, arg0::jdouble)
    return jcall(obj, "setNutationPeriod", void, (jdouble,), arg0)
end

function set_nutation_phase(obj::APMFile, arg0::jdouble)
    return jcall(obj, "setNutationPhase", void, (jdouble,), arg0)
end

function set_quaternion(obj::APMFile, arg0::Quaternion)
    return jcall(obj, "setQuaternion", void, (Quaternion,), arg0)
end

function set_quaternion_dot(obj::APMFile, arg0::Quaternion)
    return jcall(obj, "setQuaternionDot", void, (Quaternion,), arg0)
end

function set_quaternion_frame_a_string(obj::APMFile, arg0::JString)
    return jcall(obj, "setQuaternionFrameAString", void, (JString,), arg0)
end

function set_quaternion_frame_b_string(obj::APMFile, arg0::JString)
    return jcall(obj, "setQuaternionFrameBString", void, (JString,), arg0)
end

function set_rate_frame_string(obj::APMFile, arg0::JString)
    return jcall(obj, "setRateFrameString", void, (JString,), arg0)
end

function set_rotation_angles(obj::APMFile, arg0::Vector3D)
    return jcall(obj, "setRotationAngles", void, (Vector3D,), arg0)
end

function set_rotation_rates(obj::APMFile, arg0::Vector3D)
    return jcall(obj, "setRotationRates", void, (Vector3D,), arg0)
end

function set_spacecraft_comment(obj::APMFile, arg0::List)
    return jcall(obj, "setSpacecraftComment", void, (List,), arg0)
end

function set_spin_alpha(obj::APMFile, arg0::jdouble)
    return jcall(obj, "setSpinAlpha", void, (jdouble,), arg0)
end

function set_spin_angle(obj::APMFile, arg0::jdouble)
    return jcall(obj, "setSpinAngle", void, (jdouble,), arg0)
end

function set_spin_angle_vel(obj::APMFile, arg0::jdouble)
    return jcall(obj, "setSpinAngleVel", void, (jdouble,), arg0)
end

function set_spin_comment(obj::APMFile, arg0::List)
    return jcall(obj, "setSpinComment", void, (List,), arg0)
end

function set_spin_delta(obj::APMFile, arg0::jdouble)
    return jcall(obj, "setSpinDelta", void, (jdouble,), arg0)
end

function set_spin_direction(obj::APMFile, arg0::JString)
    return jcall(obj, "setSpinDirection", void, (JString,), arg0)
end

function set_spin_frame_a_string(obj::APMFile, arg0::JString)
    return jcall(obj, "setSpinFrameAString", void, (JString,), arg0)
end

function set_spin_frame_b_string(obj::APMFile, arg0::JString)
    return jcall(obj, "setSpinFrameBString", void, (JString,), arg0)
end


"""
    compute_frames!(elements::Vector{UA}; parent=nothing)
        where UA<:Union{AbstractObject,AbstractAction}

Set elem.frames.frames to the computed frames for each elem in elements.
"""
function compute_frames!(
    elements::Vector{UA};
    parent = nothing,
    parent_counter = 0,
) where {UA<:Union{AbstractObject,AbstractAction}}
    available_subframes = typemin(Int):typemax(Int)
    if parent !== nothing
        last_frames = get_frames(parent)
        available_subframes = 1:length(get_frames(parent))
    else
        last_frames = nothing
    end
    is_first = true
    counter = 1
    for elem in elements
        if last_frames === nothing && get_frames(elem) === nothing
            throw(ArgumentError("Frames need to be defined explicitly in the initial
                Object/Background or Action."))
        end
        if get_frames(elem) === nothing
            set_frames!(parent, elem, last_frames; is_first = is_first)
        end
        last_frames = get_frames(elem)
        if !(get_frames(elem) ⊆ available_subframes)
            @warn("Action defined outside the frame range of the parent object.
            Action #$counter for Object #$parent_counter is defined for frames
            $(get_frames(elem)) but Object #$parent_counter exists only for $(available_subframes).
            (Info: Background is counted as Object #1)")
        end
        is_first = false
        counter += 1
    end
end

"""
    get_current_setting()

Return the current setting of the current object
"""
function get_current_setting()
    object = CURRENT_OBJECT[1]
    return object.current_setting
end

"""
    get_interpolation(frames::UnitRange, frame)

Return a value between 0 and 1 which represents the relative `frame` inside `frames`.
"""
function get_interpolation(frames::UnitRange, frame)
    frame == last(frames) && return 1.0
    t = (frame - first(frames)) / (length(frames) - 1)
    # makes sense to only allow 0 ≤ t ≤ 1
    t = min(1.0, t)
end

"""
    get_interpolation(action::AbstractAction, frame)

Return the value of the `action.anim` Animation based on the relative frame given by
`get_interpolation(get_frames(action), frame)`
"""
function get_interpolation(action::AbstractAction, frame)
    t = get_interpolation(get_frames(action), frame)
    if !(action.anim.frames[end].t ≈ 1)
        @warn "Animations should be defined from 0.0 to 1.0"
    end
    return interpolation_to_transition_val(at(action.anim, t), action.transition)
end

"""
    interpolation_to_transition_val(interpolation_val, Transition)

Returns the transition value for the given `interpolation_val`.
If the interpolation value is already of the correct form it just gets returned.
Otherwise the Transition function like `get_position` is called and the interpolated value
is calculated.
"""
interpolation_to_transition_val(t, ::Nothing) = t
interpolation_to_transition_val(t::Point, trans::Translation) = t
interpolation_to_transition_val(t::Float64, trans::Rotation) = t
interpolation_to_transition_val(t::Scale, trans::Scaling) = t

function interpolation_to_transition_val(t, trans::Translation)
    from = get_position(trans.from)
    to = get_position(trans.to)
    return from + t * (to - from)
end

function interpolation_to_transition_val(t, trans::Scaling)
    from = get_scale(trans.from)
    to = get_scale(trans.to)
    return from + t * (to - from)
end

function isapprox_discrete(val; atol = 1e-4)
    return isapprox(val, round(val); atol = atol)
end

function polywh(polygon::Vector{Vector{Point}})
    T = typeof(polygon[1][1].x)
    min_x = typemax(T)
    min_y = typemax(T)
    max_x = typemin(T)
    max_y = typemin(T)
    for poly in polygon
        for p in poly
            min_x = min(min_x, p.x)
            min_y = min(min_y, p.y)
            max_x = max(max_x, p.x)
            max_y = max(max_y, p.y)
        end
    end
    return max_x - min_x, max_y - min_y
end

function get_polypoint_at(points, t; pdist = polydistances(points))
    if t ≈ 0
        return points[1]
    end
    ind, surplus = nearestindex(pdist, t * pdist[end])

    nextind = mod1(ind + 1, length(points))
    overshootpoint = between(
        points[ind],
        points[nextind],
        surplus / distance(points[ind], points[nextind]),
    )
    return overshootpoint
end


# test functions for the 5taxon networks
# functions used in tests_5taxon.jl
# Claudia September 2014
# changed to new reparametrizations: bad diamondI,II and bad triangle I,II
# Claudia November 2014

# Case C bad triangle II
function testCaseC(net::HybridNetwork)
    n=searchHybridNode(net);
    node = n[1];
    node.k != 3 ? error("k diff than 3") : nothing
    node.isVeryBadTriangle ? nothing : error("does not know it is very bad triangle")
    node.isExtBadTriangle ? error("thinks it is extremely bad triangle") : nothing
    net.hasVeryBadTriangle ? nothing : error("net does not know it has very bad triangle")
    net.numBad == 0 ? nothing : error("net.numBad should be 0")
    net.numHybrids != 1 ? error("should have 1 hybrid, but net.numHybrids is $(net.numHybrids): $([n.number for n in net.hybrid])") : nothing
end

# Case F bad diamond
function testCaseF(net::HybridNetwork)
    net.visited = [e.istIdentifiable for e in net.edge];
    n = searchHybridNode(net);
    node = n[1];
    node.k != 4 ? error("k diff than 4") : nothing
    edge5 = getIndexEdge(5,net);
    edge9 = getIndexEdge(9,net);
    edge11 = getIndexEdge(11,net);
    edge12 = getIndexEdge(12,net);
    edge15 = getIndexEdge(15,net);
    edge14 = getIndexEdge(14,net);
    node1 = getIndexNode(1,net);
    node5 = getIndexNode(5,net);
    node11 = getIndexNode(11,net);
    node12 = getIndexNode(12,net);
    (net.edge[edge5].inCycle != node.number || net.edge[edge11].inCycle != node.number || net.edge[edge12].inCycle != node.number || net.edge[edge15].inCycle != node.number ) ? error("edges not correctly in cycle") : nothing
    (net.node[node1].inCycle  != node.number || net.node[node5].inCycle  != node.number || net.node[node11].inCycle  != node.number || net.node[node12].inCycle  != node.number) ? error("nodes 1,5,11,12 not correctly in cycle") : nothing
    !node.isBadDiamondI ? error("does not know it is bad diamondI") : nothing
    node.isBadDiamondII ? error("thinks it is bad diamond II") : nothing
    net.edge[edge14].containRoot ? error("edge can contain root") : nothing
    (!net.edge[edge11].hybrid || !net.edge[edge11].isMajor) ? error("edge 11 is not hybrid or major") : nothing
    net.node[node12].gammaz != net.edge[edge15].gamma*net.edge[edge12].z ? error("node 12 gammaz not correctly calculated") : nothing
    net.node[1].gammaz != net.edge[edge11].gamma*net.edge[edge5].z ? error("node 11 gammaz not correctly calculated") : nothing
    !net.edge[edge9].istIdentifiable ? error("edge9 not identifiable") : nothing
    net.visited[edge9] = false;
    !all([!id for id in net.visited]) ? error("edges not identifiable as identifiable") : nothing
    net.numHybrids != 1 ? error("should have 1 hybrid, but net.numHybrids is $(net.numHybrids): $([n.number for n in net.hybrid])") : nothing
    net.numBad == 1 ? nothing : error("net.numBad should be 1")
end

# Case G
function testCaseG(net::HybridNetwork)
    net.visited = [e.istIdentifiable for e in net.edge];
    n = searchHybridNode(net);
    node = n[1];
    node.k != 4 ? error("k diff than 4") : nothing
    edge9 = getIndexEdge(9,net);
    edge12 = getIndexEdge(12,net);
    edge15 = getIndexEdge(15,net);
    edge5 = getIndexEdge(5,net);
    edge13 = getIndexEdge(13,net);
    edge14 = getIndexEdge(14,net);
    node9 = getIndexNode(9,net);
    node5 = getIndexNode(5,net);
    node11 = getIndexNode(11,net);
    node12 = getIndexNode(12,net);
    (net.edge[edge9].inCycle != node.number || net.edge[edge12].inCycle != node.number || net.edge[edge13].inCycle != node.number || net.edge[edge15].inCycle != node.number ) ? error("edges not correctly in cycle") : nothing
    (net.node[node9].inCycle  != node.number || net.node[node5].inCycle  != node.number || net.node[node11].inCycle  != node.number || net.node[node12].inCycle  != node.number) ? error("nodes not correctly in cycle") : nothing
    (node.isBadDiamondI || node.isBadDiamondII) ? error("thinks it is bad diamond") : nothing
    net.edge[edge14].containRoot ? error("14 can contain root") : nothing
    (!net.edge[edge12].hybrid || !net.edge[edge12].isMajor) ? error("edge 12 is not hybrid or major") : nothing
    net.node[node12].gammaz != -1 ? error("node 12 gammaz should be -1") : nothing
    (!net.edge[edge9].istIdentifiable || !net.edge[edge5].istIdentifiable || !net.edge[edge13].istIdentifiable) ? error("edge9,5,13not identifiable") : nothing
    net.visited[edge9] = false;
    net.visited[edge5] = false;
    net.visited[edge13] = false;
    !all([!id for id in net.visited]) ? error("edges not identifiable as identifiable") : nothing
    net.numHybrids != 1 ? error("should have 1 hybrid, but net.numHybrids is $(net.numHybrids): $([n.number for n in net.hybrid])") : nothing
end

# Case H
function testCaseH(net::HybridNetwork)
    net.visited = [e.istIdentifiable for e in net.edge];
    n = searchHybridNode(net);
    node = n[1];
    node.k != 4 ? error("k diff than 4") : nothing
    edge9 = getIndexEdge(9,net);
    edge14 = getIndexEdge(14,net);
    edge15 = getIndexEdge(15,net);
    edge13 = getIndexEdge(13,net);
    edge5 = getIndexEdge(5,net);
    edge8 = getIndexEdge(8,net);
    node9 = getIndexNode(9,net);
    node5 = getIndexNode(5,net);
    node11 = getIndexNode(11,net);
    node12 = getIndexNode(12,net);
    (net.edge[edge9].inCycle != node.number || net.edge[edge5].inCycle != node.number || net.edge[edge14].inCycle != node.number || net.edge[edge15].inCycle != node.number ) ? error("edges not correctly in cycle") : nothing
    (net.node[node9].inCycle  != node.number || net.node[node5].inCycle  != node.number || net.node[node11].inCycle  != node.number || net.node[node12].inCycle  != node.number) ? error("nodes 9,5,11,12 not correctly in cycle") : nothing
    (node.isBadDiamondI || node.isBadDiamondII )? error("thinks it is bad diamond") : nothing
    net.edge[edge8].containRoot ? error("8 can contain root") : nothing
    (!net.edge[edge14].hybrid || !net.edge[edge14].isMajor) ? error("edge 14 is not hybrid or major") : nothing
    net.node[node12].gammaz != -1 ? error("node 12 gammaz should be -1") : nothing
    (!net.edge[edge9].istIdentifiable || !net.edge[edge5].istIdentifiable || !net.edge[edge13].istIdentifiable) ? error("edge9,5,13not identifiable") : nothing
    net.visited[edge9] = false;
    net.visited[edge5] = false;
    net.visited[edge13] = false;
    !all([!id for id in net.visited]) ? error("edges not identifiable as identifiable") : nothing
    net.numHybrids != 1 ? error("should have 1 hybrid, but net.numHybrids is $(net.numHybrids): $([n.number for n in net.hybrid])") : nothing
end

# Case J
function testCaseJ(net::HybridNetwork)
    net.visited = [e.istIdentifiable for e in net.edge];
    n = searchHybridNode(net);
    node = n[1];
    node.k != 5 ? error("k diff than 5") : nothing
    edge9 = getIndexEdge(9,net);
    edge10 = getIndexEdge(10,net);
    edge5 = getIndexEdge(5,net);
    edge15 = getIndexEdge(15,net);
    edge6 = getIndexEdge(6,net);
    edge14 = getIndexEdge(14,net);
    node9 = getIndexNode(9,net);
    node1 = getIndexNode(1,net);
    node5 = getIndexNode(5,net);
    node11 = getIndexNode(11,net);
    node12 = getIndexNode(12,net);
    (net.edge[edge9].inCycle != node.number || net.edge[edge10].inCycle != node.number || net.edge[edge6].inCycle != node.number || net.edge[edge15].inCycle != node.number || net.edge[edge5].inCycle != node.number) ? error("edges not correctly in cycle") : nothing
    (net.node[node9].inCycle  != node.number || net.node[node5].inCycle  != node.number || net.node[node11].inCycle  != node.number || net.node[node12].inCycle  != node.number || net.node[node1].inCycle) != node.number ? error("nodes 9,5,11,12,1 not correctly in cycle") : nothing
    net.edge[edge14].containRoot ? error("14 can contain root") : nothing
    (!net.edge[edge6].hybrid || !net.edge[edge6].isMajor) ? error("edge 6 is not hybrid or major") : nothing
    net.node[node12].gammaz != -1 ? error("node 12 gammaz should be -1") : nothing
    (!net.edge[edge9].istIdentifiable || !net.edge[edge5].istIdentifiable || !net.edge[edge10].istIdentifiable) ? error("edge9,5,10not identifiable") : nothing
    net.visited[edge9] = false;
    net.visited[edge5] = false;
    net.visited[edge10] = false;
    !all([!id for id in net.visited]) ? error("edges not identifiable as identifiable") : nothing
    net.numHybrids != 1 ? error("should have 1 hybrid, but net.numHybrids is $(net.numHybrids): $([n.number for n in net.hybrid])") : nothing
end

# Case D bad triangle I
function testCaseD(net::HybridNetwork)
    net.visited = [e.istIdentifiable for e in net.edge];
    n = searchHybridNode(net);
    node = n[1];
    node.k != 3 ? error("k diff than 3") : nothing
    node.isVeryBadTriangle ? nothing : error("does not know it is very bad triangle")
    node.isExtBadTriangle ? error("thinks it is extremely bad triangle") : nothing
    net.hasVeryBadTriangle ? nothing : error("net does not know it has very bad triangle")
    net.numBad == 0 ? nothing : error("net.numBad should be 0")
    net.numHybrids != 1 ? error("should have 1 hybrid, but net.numHybrids is $(net.numHybrids): $([n.number for n in net.hybrid])") : nothing
end

# Case E bad triangle I
function testCaseE(net::HybridNetwork)
    net.visited = [e.istIdentifiable for e in net.edge];
    n = searchHybridNode(net);
    node = n[1];
    node.k != 3 ? error("k diff than 3") : nothing
    node.isVeryBadTriangle ? nothing : error("does not know it is very bad triangle")
    node.isExtBadTriangle ? error("thinks it is extremely bad triangle") : nothing
    net.hasVeryBadTriangle ? nothing : error("net does not know it has very bad triangle")
    net.numBad == 0 ? nothing : error("net.numBad should be 0")
    net.numHybrids != 1 ? error("should have 1 hybrid, but net.numHybrids is $(net.numHybrids): $([n.number for n in net.hybrid])") : nothing
end

# Case I bad diamond II
function testCaseI(net::HybridNetwork)
    n = searchHybridNode(net);
    node = n[1];
    node.isBadDiamondII ? nothing : error("does not know it is bad diamond II")
    !node.isBadDiamondI ? nothing : error("thinks it is bad diamond I")
    net.numHybrids != 1 ? error("should have 1 hybrid, but net.numHybrids is $(net.numHybrids): $([n.number for n in net.hybrid])") : nothing
    net.visited = [e.istIdentifiable for e in net.edge];
    edge14 = getIndexEdge(14,net);
    edge10 = getIndexEdge(10,net);
    edge5 = getIndexEdge(5,net);
    edge15 = getIndexEdge(15,net);
    edge9 = getIndexEdge(9,net);
    edge11 = getIndexEdge(11,net);
    edge8 = getIndexEdge(8,net);
    node1 = getIndexNode(1,net);
    node12 = getIndexNode(12,net);
    node5 = getIndexNode(5,net);
    node11 = getIndexNode(11,net);
    (net.edge[edge5].inCycle != node.number || net.edge[edge9].inCycle != node.number || net.edge[edge11].inCycle != node.number || net.edge[edge15].inCycle != node.number ) ? error("edges not correctly in cycle") : nothing
    (net.node[node1].inCycle  != node.number || net.node[node12].inCycle  != node.number || net.node[node5].inCycle  != node.number || net.node[node11].inCycle  != node.number) ? error("nodes 1,5,11,12 not correctly in cycle") : nothing
    (net.edge[edge14].containRoot || net.edge[edge10].containRoot || net.edge[edge8].containRoot) ? error("edges can contain root and shouldn't") : nothing
    (!net.edge[edge9].hybrid || !net.edge[edge9].isMajor) ? error("edge 4 is not hybrid or major") : nothing
    net.edge[edge14].length != 0 ? error("edges should have length 0") : nothing
    net.edge[edge14].istIdentifiable ? error("edge14 identifiable and should not") : nothing
    net.visited[edge9] = false;
    net.visited[edge5] = false;
    net.visited[edge11] = false;
    net.visited[edge15] = false;
    !all([!id for id in net.visited]) ? error("edges not identifiable as identifiable") : nothing
    net.numBad == 0 ? nothing : error("net.numBad should be 0")
end

# tree example
# warning: if added hybridization (bad diamond/triangle) and then delete,
#          original edge lengths cannot be recovered
function testTree(net::HybridNetwork)
    !all([!e.hybrid for e in net.edge]) ? error("some edge is still hybrid") : nothing
    !all([!e.hybrid for e in net.node]) ? error("some node is still hybrid") : nothing
    !all([!e.hasHybEdge for e in net.node]) ? error("some node has hybrid edge") : nothing
    !all([e.isMajor for e in net.edge]) ? error("some edge is not major") : nothing
    !all([e.containRoot for e in net.edge]) ? error("some edge cannot contain root") : nothing
    edge9 = getIndexEdge(9,net);
    edge5 = getIndexEdge(5,net);
    (!net.edge[edge9].istIdentifiable || !net.edge[edge5].istIdentifiable) ? error("edge9,5 not identifiable") : nothing
    net.visited = [e.istIdentifiable for e in net.edge];
    net.visited[edge9] = false;
    net.visited[edge5] = false;
    !all([!id for id in net.visited]) ? error("edges not identifiable as identifiable") : nothing
    isempty(net.hybrid) ? nothing : error("something in net.hybrid")
    net.numHybrids != 0 ? error("should have 0 hybrid, but net.numHybrids is $(net.numHybrids)") : nothing
    #edge11 = getIndexEdge(11,net);
    #edge12 = getIndexEdge(12,net);
    #net.edge[edge11].length != 1.5 ? error("edge length for 11 is wrong") : nothing
    #net.edge[edge12].length != 0.2 ? error("edge length for 12 is wrong") : nothing
end


using Random
using DPMMSubClusters
include("run_experiments.jl")

dir = "/home/vlad/projects/DPMMSubClusters.jl/test_dir/"
SEED = 1234

Random.seed!(SEED)

# model = "pointNet"
model = "setTransformer"

REPEATS=3
ITERS=100
BURNOUT=10
# REPEATS=1
# ITERS=10
# BURNOUT=2
ALPHA=100

imagenet_random = repeat_experiment(:gauss2d,REPEATS, "none", ALPHA, BURNOUT, ITERS)
imagenet_kmeans = repeat_experiment(:gauss2d,REPEATS, "kmeans", ALPHA, BURNOUT, ITERS)
imagenet_splitnet = repeat_experiment(:gauss2d,REPEATS, "splitnet_2d", ALPHA,BURNOUT, ITERS)

# imagenet_random = repeat_experiment(:imagenet_swav_d128,REPEATS, "none", ALPHA, BURNOUT, ITERS)
# imagenet_kmeans = repeat_experiment(:imagenet_swav_d128,REPEATS, "kmeans", ALPHA, BURNOUT, ITERS)
# imagenet_splitnet = repeat_experiment(:imagenet_swav_d128,REPEATS, "splitnet-gpu", ALPHA,BURNOUT, ITERS)

all_results = Dict(
    "vanilla" => imagenet_random,
    "kmeans" => imagenet_kmeans,
    "splitnet" => imagenet_splitnet,
)

title = "ImageNet-Swav-D=128--Burnout-" * string(BURNOUT) * "_iters_"* string(ITERS)* "_alpha_"* string(ALPHA)

f1= plot_nmi_for_iters(all_results, title)
# savefig("./compare_methods_gifs/result_nmi_"* string(title) *".png")

f2 = plot_ll_for_iters(all_results, title)

f3 = plot_K_for_iters(all_results, title)

# f3 = plot(f1, f2, layout = (2, 1), legend = true, size=(600, 800))
f4 = plot(f1, f2, f3, layout = (3, 1), size=(600, 1200))

savefig("../compare_methods_gifs/result_"* string(title) *".png")


connector_comp_name(i::Int) = Symbol("ConnectorComp$i")

# Return the datatype to use for instance variables/parameters
function _instance_datatype(md::ModelDef, def::DatumDef)
    dtype = def.datatype == Number ? number_type(md) : def.datatype
    dims = dimensions(def)
    num_dims = dim_count(def)

    if num_dims == 0
        T = ScalarModelParameter{dtype}

    elseif dims[1] != :time
        T = Array{dtype, num_dims}

    else
        if isuniform(md)
            first, stepsize = first_and_step(md)
            T = TimestepArray{FixedTimestep{first, stepsize}, Union{dtype, Missing}, num_dims}
        else
            times = time_labels(md)
            T = TimestepArray{VariableTimestep{(times...,)}, Union{dtype, Missing}, num_dims}
        end
    end

    # @info "_instance_datatype returning $T"
    return T
end

# Create the Ref or Array that will hold the value(s) for a Parameter or Variable
function _instantiate_datum(md::ModelDef, def::DatumDef)
    dtype = _instance_datatype(md, def)
    dims = dimensions(def)
    num_dims = length(dims)

    # Scalar datum
    if num_dims == 0
        value = dtype(0)

    # Array datum, with :time dimension
    elseif dims[1] == :time

        if num_dims == 1
            value = dtype(dim_count(md, :time))
        else
            counts = dim_counts(md, Vector{Symbol}(dims))
            value = dtype <: AbstractArray ? dtype(undef, counts...) : dtype(counts...)
        end

    # Array datum, without :time dimension
    else
        # TBD: Handle unnamed indices properly
        counts = dim_counts(md, Vector{Symbol}(dims))
        value = dtype <: AbstractArray ? dtype(undef, counts...) : dtype(counts...)
    end

    return value
end

"""
    _instantiate_component_vars(md::ModelDef, comp_def::ComponentDef)

Instantiate a component `comp_def` in the model `md` and its variables (but not its parameters).
Return the resulting ComponentInstance.
"""
function _instantiate_component_vars(md::ModelDef, comp_def::ComponentDef)
    comp_name = name(comp_def)
    var_defs = variables(comp_def)

    names  = ([name(vdef) for vdef in var_defs]...,)
    types  = Tuple{[_instance_datatype(md, vdef) for vdef in var_defs]...}
    values = [_instantiate_datum(md, def) for def in var_defs]

    return ComponentInstanceVariables(names, types, values)
end

# Save a reference to the model's dimension dictionary to make it
# available in calls to run_timestep.
function save_dim_dict_reference(mi::ModelInstance)
    dim_dict = dim_value_dict(mi.md)

    for ci in values(mi.components)
        ci.dim_dict = dim_dict
    end

    return nothing
end

function build(m::Model)
    # Reference a copy in the ModelInstance to avoid changes underfoot
    m.mi = build(copy(m.md))
    return nothing
end

function build(md::ModelDef)

    add_connector_comps(md)

    # check if all parameters are set
    not_set = unconnected_params(md)
    if ! isempty(not_set)
        params = join(not_set, " ")
        msg = "Cannot build model; the following parameters are not set: $params"
        error(msg)
    end

    var_dict = Dict{Symbol, Any}()                 # collect all var defs and
    par_dict = Dict{Symbol, Dict{Symbol, Any}}()   # store par values as we go

    comp_defs = compdefs(md)
    for comp_def in comp_defs
        comp_name = name(comp_def)
        var_dict[comp_name] = _instantiate_component_vars(md, comp_def)
        par_dict[comp_name] = Dict()  # param value keyed by param name
    end

    # Iterate over connections to create parameters, referencing storage in vars
    for ipc in internal_param_conns(md)
        comp_name = ipc.src_comp_name

        vars = var_dict[comp_name]
        var_value_obj = get_property_obj(vars, ipc.src_var_name)

        par_values = par_dict[ipc.dst_comp_name]
        par_values[ipc.dst_par_name] = var_value_obj
    end

    for ext in external_param_conns(md)
        comp_name = ext.comp_name
        param = external_param(md, ext.external_param)
        par_values = par_dict[comp_name]
        par_values[ext.param_name] = param isa ScalarModelParameter ? param : value(param)
    end

    # Make the external parameter connections for the hidden ConnectorComps.
    # Connect each :input2 to its associated backup value.
    for (i, backup) in enumerate(md.backups)
        comp_name = connector_comp_name(i)
        param = external_param(md, backup)

        par_values = par_dict[comp_name]
        par_values[:input2] = param isa ScalarModelParameter ? param : value(param)
    end

    mi = ModelInstance(md)

    # instantiate parameters
    for comp_def in comp_defs
        comp_name = name(comp_def)

        vars = var_dict[comp_name]

        par_values = par_dict[comp_name]
        pnames = Tuple(parameter_names(comp_def))
        pvals  = [par_values[pname] for pname in pnames]
        ptypes = Tuple{map(typeof, pvals)...}
        pars = ComponentInstanceParameters(pnames, ptypes, pvals)

        first = first_period(md, comp_def)
        last = last_period(md, comp_def)

        ci = ComponentInstance{typeof(vars), typeof(pars)}(comp_def, vars, pars, first, last, comp_name)
        add_comp!(mi, ci)
    end

    save_dim_dict_reference(mi)
    return mi
end

"""
    create_marginal_model(base::Model, delta::Float64=1.0)

Create a `MarginalModel` where `base` is the baseline model and `delta` is the
difference used to create the `marginal` model.  Return the resulting `MarginaModel`
which shares the internal `ModelDef` between the `base` and `marginal`.
"""
function create_marginal_model(base::Model, delta::Float64=1.0)
    # Make sure the base has a ModelInstance before we copy since this
    # copies the ModelDef to avoid being affected by later changes.
    if base.mi === nothing
        build(base)
    end

    # Create a marginal model, which shares the internal ModelDef between base and marginal
    mm = MarginalModel(base, delta)
end

function Base.run(mm::MarginalModel, ntimesteps::Int=typemax(Int))
    run(mm.base, ntimesteps=ntimesteps)
    run(mm.marginal, ntimesteps=ntimesteps)
end


module SSCF

export SSCFFile

using Dates
using Colors
using Mmap

"""SSCF Header data structure"""
struct SSCFHeader
    version :: Float32
    channels :: Integer
    samples :: Integer
    dt :: Float32

    function SSCFHeader(f::IOStream)
	magic = read(f, Float32)

	if magic != 1003.
	    error("Not a valid SSCF file. Wrong magic number.")
	end

	version = read(f, Float32)
	channels = read(f, Float32)
	samples = read(f, Float32)
	dt = read(f, Float32)

	return new(version, channels, samples, dt)
    end
end

"""Channel Information Record"""
struct ChannelInfo
    title :: String
    units :: String
    instrument :: String
    # aux1 :: String
    # aux2 :: String
    transform :: Int32
    color :: RGB
    # aux :: Vector{Float32}

    function ChannelInfo(f::IOStream)
	io = IOBuffer(read(f, 24), read=true, write=false, maxsize=24)
	title = strip(read(io, String))

	io = IOBuffer(read(f, 16), read=true, write=false, maxsize=16)
	units = strip(read(io, String))

	io = IOBuffer(read(f, 32), read=true, write=false, maxsize=32)
	instrument = strip(read(io, String))

	# skip unused aux fields
	skip(f, 32 + 32)

	transform = read(f, Int32)
	cc = read(f,Int32)
	color = RGB(
	    cc % 256 / 255.,
	    ((cc // 256) % 256) / 255.,
	    ((cc // (256*256)) % 256) / 255.
	)

	# skip aux floats
	skip(f, 31*sizeof(Float32))

	return new(title, units, instrument, transform, color)
    end
end

"""Marker Table"""
struct MarkerTable
    samples :: Vector{Int32}
    code :: Vector{Char}

    function MarkerTable(raw::Array{Int32,1})
	samples = zeros(Int32, length(raw))
	code = Vector{Char}(undef, length(raw))

	for i in 1:length(raw)
            code[i], samples[i] = divrem(raw[i],1E6)
	end

	return new(samples, code)
    end

end

"""SSCF Footer Information"""
struct SSCFFooter
    timestamp :: DateTime
    remarks :: String
    markers :: MarkerTable
    channelinfo :: Vector{ChannelInfo}
    setup_path :: String
    setup_file :: String

    function SSCFFooter(f::IOStream, header::SSCFHeader)
	# skip auxiliary records (unused)
	skip(f, 10 * header.channels * sizeof(Float32))

	# date time
	day = read(f, Float32)
	month = read(f, Float32)
	year = read(f, Float32)
	hour = read(f, Float32)
	minute = read(f, Float32)
	second = read(f, Float32)

	stamp = DateTime(year, month, day, hour, minute, second)

	# remarks (part 1)
	io = IOBuffer(read(f, 240), read=true, write=false, maxsize=240)
	remarks = read(io, String)

	# markers
	marker_count = read(f, Int32)
	marker_data = Vector{Int32}(undef, marker_count)
	read!(f, marker_data)
	markers = MarkerTable(marker_data)

	# note markers
	marker_count = read(f, Int32)
	notemarkers = read(f, marker_count * 256)
	# TODO decode

	# check channel number
	channel_count = read(f, Int32)
	if channel_count != header.channels
	    error("Invalid SSCF file. Cannel count mismatch between header and footer.")
	end

	# channel info
	channelinfo = [ChannelInfo(f) for i in 1:header.channels]

	# user vars
	mdata = read(f, 50*sizeof(Float32))
	asize = read(f, Int32)
	aliases = read(f, asize)

	# setup info
	asize = read(f, Int32)
	if asize == 0
	    setup_path = ""
	    setup_file = ""
	elseif asize == 192
	    io = IOBuffer(read(f, 64), read=true, write=false, maxsize=64)
	    setup_file = read(io, String)
	    io = IOBuffer(read(f, 128), read=true, write=false, maxsize=128)
	    setup_path = read(io, String)
	else
	    error("Invalid SSCF file. Incorrect setup info size.")
	end

	# remarks (part 2)
	asize = read(f, Int32)
	io = IOBuffer(read(f, asize), read=true, write=false, maxsize=asize)
	remarks = remarks * read(io, String)

	return new(stamp, remarks, markers, channelinfo, setup_path, setup_file)
    end
end

"""Main SSCF File data structure"""
struct SSCFFile
    path :: String
    data :: Matrix{Float32}
    header :: SSCFHeader
    footer :: SSCFFooter

    """Open file descriptor and read file contents

    Example Usage:
    >> sscf = open("test.exp") |> SSCFFile
    """
    function SSCFFile(f::IOStream)
	header = SSCFHeader(f)

	# mem map
	data_size = header.channels * header.samples * sizeof(Float32)
	data = Mmap.mmap(f, Matrix{Float32}, (header.channels, header.samples))

	# skip to footer
	skip(f, data_size)
	footer = SSCFFooter(f, header)

	return new(f.name, data, header, footer)
    end

    """Open file given by filename

    Example Usage:
    >> sscf = SSCFFile("test.exp")
    """
    SSCFFile(filename::String)= open(filename) |> SSCFFile
end

"""Direct access for some header/footer data as properties

.samples - number of samples in file
.channels - number of data channels
.starttime - time of first sample
.endtime - time of last sample
.interval - sample interval rounded to msec precision
.times - vector of sample times
.markers - the marker table (type MarkerTable)
.remarks - remarks string
"""
function Base.getproperty(f::SSCFFile, v::Symbol)
	if v == :samples
    	    return f.header.samples
        elseif v == :times
            return f.starttime:f.interval:f.endtime
        elseif v == :starttime
            return f.endtime - (f.samples-1) * f.interval
        elseif v == :endtime
            return f.footer.timestamp
        elseif v == :interval
            return Millisecond(f.header.dt * 1000)
	elseif v == :channels
	    return f.header.channels
	elseif v == :markers
	    return f.footer.markers
	elseif v == :remarks
	    return f.footer.remarks
	else
	    return getfield(f, v)
	end
end

"""Index based access to data channels by name

    Example Usage:
    >> sscf = open("test.exp") |> SSCFFile
    >> o2_a = sscf["O2_A"]
"""
function Base.getindex(collection::SSCFFile, key::String)
    index = findfirst([x.title == key for x in collection.footer.channelinfo])
    return collection.data[index,:]
end

end


# This file is a part of BAT.jl, licensed under the MIT License (MIT).

mutable struct DirectSamplingState{
    Q<:AbstractProposalDist,
    S<:DensitySample
} <: AbstractMCMCState
    pdist::Q
    current_sample::S
    proposed_sample::S
    proposal_accepted::Bool
    current_nreject::Int64
    nsamples::Int64
    nsteps::Int64
end

function DirectSamplingState(
    pdist::AbstractProposalDist,
    current_sample::DensitySample
)
    proposed_sample = DensitySample(
        similar(current_sample.params),
        convert(typeof(current_sample.log_value), NaN),
        zero(current_sample.weight)
    )

    DirectSamplingState(
        pdist,
        current_sample,
        proposed_sample,
        false,
        zero(Int64),
        zero(Int64),
        one(Int64)
    )
end

nparams(state::DirectSamplingState) = nparams(state.pdist)

nsteps(state::DirectSamplingState) = state.nsteps

nsamples(state::DirectSamplingState) = state.nsamples

eff_acceptance_ratio(state::DirectSamplingState) = nsamples(state) / nsteps(state)

function next_cycle!(state::DirectSamplingState)
    state.current_sample.weight = one(state.current_sample.weight)
    state.nsamples = zero(Int64)
    state.nsteps = zero(Int64)
    state
end

density_sample_type(state::DirectSamplingState{Q,S}) where {Q,S} = S

_accrej_current_sample(state::DirectSamplingState) =
    ifelse(state.proposal_accepted, state.current_sample, state.proposed_sample)

function nsamples_available(chain::MCMCIterator{<:MCMCAlgorithm{DirectSamplingState}}, nonzero_weights::Bool = false)
    if nonzero_weights
        (_accrej_current_sample(chain.state).weight > 0) ? 1 : 0
    else
        1
    end
end

function get_samples!(appendable, chain::MCMCIterator{<:MCMCAlgorithm{DirectSamplingState}}, nonzero_weights::Bool)::typeof(appendable)
    sample = _accrej_current_sample(chain.state)
    if (!nonzero_weights || sample.weight > 0)
        push!(appendable, sample)
    end
    appendable
end

function get_sample_ids!(appendable, chain::MCMCIterator{<:MCMCAlgorithm{DirectSamplingState}}, nonzero_weights::Bool)::typeof(appendable)
    state = chain.state
    sample = _accrej_current_sample(state)
    if (!nonzero_weights || sample.weight > 0)
        sampletype = state.proposal_accepted ? 2 : 1
        sampleid = MCMCSampleID(chain.id, chain.cycle, state.nsteps, sampletype)
        push!(appendable, sampleid)
    end
    appendable
end

function MCMCIterator(
    algorithm::MCMCAlgorithm{DirectSamplingState},
    likelihood::AbstractDensity,
    prior::AbstractDensity,
    id::Int64,
    rng::AbstractRNG,
    initial_params::AbstractVector{P} = Vector{P}(),
    exec_context::ExecContext = ExecContext(),
) where {P<:Real}
    target = likelihood * prior

    cycle = zero(Int)
    reset_rng_counters!(rng, id, cycle, 0)

    params_vec = Vector{P}(nparams(target))
    if isempty(initial_params)
        rand_initial_params!(rng, algorithm, prior, params_vec)
    else
        params_vec .= initial_params
    end

    !(params_vec in param_bounds(target)) && throw(ArgumentError("Parameter(s) out of bounds"))

    m = length(params_vec)

    log_value = density_logval(target, params_vec, exec_context)
    L = typeof(log_value)
    W = sample_weight_type(typeof(algorithm))

    current_sample = DensitySample(
        params_vec,
        log_value,
        one(W)
    )

    proposed_sample = DensitySample(
        similar(current_sample.params),
        convert(typeof(current_sample.log_value), NaN),
        zero(W)
    )

    state = DirectSamplingState(
        algorithm,
        target,
        current_sample
    )

    chain = MCMCIterator(
        algorithm,
        target,
        state,
        rng,
        id,
        cycle,
        false,
        false
    )

    chain
end

exec_capabilities(mcmc_step!, callback::AbstractMCMCCallback, chain::MCMCIterator{<:MCMCAlgorithm{DirectSamplingState}}) =
    exec_capabilities(density_logval, chain.target, chain.state.proposed_sample.params)

function mcmc_step!(
    callback::AbstractMCMCCallback,
    chain::MCMCIterator{<:MCMCAlgorithm{DirectSamplingState}},
    exec_context::ExecContext,
    ll::LogLevel
)
    state = chain.state
    algorithm = chain.algorithm

    if !mcmc_compatible(algorithm, chain.state.pdist, param_bounds(chain.target))
        error("Implementation of algorithm $algorithm does not support current parameter bounds with current proposal distribution")
    end

    current_sample = state.current_sample
    proposed_sample = state.proposed_sample

    state.nsteps += 1
    reset_rng_counters!(chain)

    mcmc_propose_accept_reject!(callback, chain, exec_context)

    if state.proposal_accepted
        copy!(current_sample, proposed_sample)
        state.current_nreject = 0
        state.proposal_accepted = false
    end

    chain
end

struct DirectSampling <: MCMCAlgorithm{DirectSamplingState} end
export DirectSampling

mcmc_compatible(::DirectSampling, ::AbstractProposalDist, ::AbstractParamBounds) = true

# ToDo: Specialized version of rand_initial_params for DirectSampling:
#
#     rand_initial_params!(rng::AbstractRNG, algorithm::DirectSampling, target::DensityFunction, x::StridedVecOrMat{<:Real}) = ...

AbstractMCMCTunerConfig(algorithm::DirectSampling) = NoOpTunerConfig()

sample_weight_type(::Type{DirectSampling}) = Int

function DirectSamplingState(
    algorithm::DirectSampling,
    target::BAT.DensityProduct{2,<:Tuple{MvDistDensity, ConstDensity}},
    current_sample::DensitySample{P,T,W}
) where {P,T,W}
    DirectSamplingState(
        GenericProposalDist(parent(target)[1].d),
        current_sample
    )
end

function mcmc_propose_accept_reject!(
    callback::AbstractMCMCCallback,
    chain::MCMCIterator{<:DirectSampling},
    exec_context::ExecContext
)
    state = chain.state
    target = chain.target

    proposed_sample = state.proposed_sample
    proposed_params = proposed_sample.params

    # Propose new parameters:
    rand!(chain.rng, state.pdist.s, proposed_params)

    # Accept iff in bounds:
    if proposed_params in param_bounds(target)
        proposed_sample.log_value = density_logval(target, proposed_params, exec_context)

        state.proposed_sample.weight = 1
        @assert state.current_sample.weight == 1
        state.proposal_accepted = true
        state.nsamples += 1
    else
        proposed_sample.log_value = -Inf

        state.current_nreject += 1
    end
    callback(1, chain)

    chain
end


struct Entry
    plottype::PlotFunc
    mappings::Arguments
    attributes::Dict{Symbol, Any}
end

Entry(plottype::PlotFunc=Any, mappings=arguments(); attributes...) =
    Entry(plottype, mappings, Dict{Symbol, Any}(attributes))

Entry(mappings::Arguments; attributes...) = Entry(Any, mappings; attributes...)

function separate!(entry::Entry)
    discrete, _ = separate!(entry.mappings)
    return discrete => entry
end

function recombine!(discrete, entry::Entry)
    return Entry(
        entry.plottype,
        recombine!(discrete, entry.mappings),
        entry.attributes
    )
end

struct Entries
    entries::Vector{Entry}
    scales::Arguments
    labels::Arguments
end

Entries() = Entries(Entry[], arguments(), arguments())

function compute_axes_grid(fig, e::Entries; axis=NamedTuple())

    rowcol = (:row, :col)

    layout_scale, scales... = map((:layout, rowcol...)) do sym
        return get(e.scales, sym, nothing)
    end

    grid_size = map(scales, (first, last)) do scale, f
        isnothing(scale) || return maximum(scale.plot)
        isnothing(layout_scale) || return maximum(f, layout_scale.plot)
        return 1
    end

    axes_grid = map(CartesianIndices(grid_size)) do c
        ax = Axis(fig[Tuple(c)...]; axis...)
        return AxisEntries(ax, Entry[], e.scales, e.labels)
    end

    for entry in e.entries
        rows, cols = map(rowcol, scales, (first, last)) do sym, scale, f
            v = get(entry.mappings, sym, nothing)
            layout_v = get(entry.mappings, :layout, nothing)
            # without layout info, plot on all axes
            # all values in `v` and `layout_v` are equal
            isnothing(v) || return rescale(v[1:1], scale)
            isnothing(layout_v) || return map(f, rescale(layout_v[1:1], layout_scale))
            return 1:f(grid_size)
        end
        for i in rows, j in cols
            ae = axes_grid[i, j]
            push!(ae.entries, entry)
        end
    end

    return axes_grid

end

function AbstractPlotting.plot(entries::Entries; axis=NamedTuple(), figure=NamedTuple())
    fig = Figure(; figure...)
    grid = plot!(fig, entries; axis)
    return FigureGrid(fig, grid)
end

function AbstractPlotting.plot!(fig, entries::Entries; axis=NamedTuple())
    axes_grid = compute_axes_grid(fig, entries; axis)
    foreach(plot!, axes_grid)
    return axes_grid
end

"""
    AxisEntries(axis::Union{Axis, Nothing}, entries::Vector{Entry}, labels, scales)

Define all ingredients to make plots on an axis.
Each scale can be either a `CategoricalScale` (for discrete collections), such as
`CategoricalScale(["a", "b"], ["red", "blue"])`, or a function,
such as `log10`. Other scales may be supported in the future.
"""
struct AxisEntries
    axis::Axis
    entries::Vector{Entry}
    scales::Arguments
    labels::Arguments
end

AbstractPlotting.Axis(ae::AxisEntries) = ae.axis
Entries(ae::AxisEntries) = Entries(ae.entries, ae.labels, ae.scales)

function prefix(i::Int, sym::Symbol)
    var = (:x, :y, :z)[i]
    return Symbol(var, sym)
end

function AbstractPlotting.plot!(ae::AxisEntries)
    axis, entries, labels, scales = ae.axis, ae.entries, ae.labels, ae.scales
    for entry in entries
        plottype, mappings, attributes = entry.plottype, entry.mappings, entry.attributes
        trace = map(unwrap∘rescale, mappings, scales)
        positional, named = trace.positional, trace.named
        merge!(named, attributes)
        for sym in [:col, :row, :layout]
            pop!(named, sym, nothing)
        end
        plot!(plottype, axis, positional...; named...)
    end
    # TODO: support log colorscale
    for i in 1:2
        label, scale = get(labels, i, nothing), get(scales, i, nothing)
        any(isnothing, (label, scale)) && continue
        axislabel, ticks, axisscale = prefix.(i, (:label, :ticks, :scale))
        if scale isa CategoricalScale
            u = map(string, scale.data)
            getproperty(axis, ticks)[] = (axes(u, 1), u)
        end
        getproperty(axis, axislabel)[] = string(label)
    end
    return axis
end

struct FigureGrid
    figure::Figure
    grid::Matrix{AxisEntries}
end

Base.show(io::IO, fg::FigureGrid) = show(io, fg.figure)
Base.show(io::IO, m::MIME, fg::FigureGrid) = show(io, m, fg.figure)
Base.show(io::IO, ::MIME"text/plain", fg::FigureGrid) = print(io, "FigureGrid()")

Base.showable(mime::MIME{M}, fg::FigureGrid) where {M} = showable(mime, fg.figure)

Base.display(fg::FigureGrid) = display(fg.figure)

function FileIO.save(filename::String, fg::FigureGrid; kwargs...)
    return FileIO.save(FileIO.query(filename), fg; kwargs...)
end

function FileIO.save(file::FileIO.Formatted, fg::FigureGrid; kwargs...)
    return FileIO.save(file, fg.figure; kwargs...)
end

to_tuple(fg) = (fg.figure, fg.grid)

Base.iterate(fg::FigureGrid) = iterate(to_tuple(fg))
Base.iterate(fg::FigureGrid, i) = iterate(to_tuple(fg), i)


#!/Applications/Julia-0.6.1.app/Contents/Resources/julia/bin/julia
# Nearest Neighbor classifier, neural network based, and employing Hamming distances
# This classifier works with any mix of categorical or continuous variables, any number of classes,
# and can handle missing values.

__precompile__()

module HamNNNNclassifier

export main, train, test, readdatafile, printdatafiledescription, hamclass, generatetraintesttable, printattributeranking, anneal, glass

using NullableArrays, DataFrames, CategoricalArrays, CSV, Query, StatsBase, ArgParse

include("HamNNNNclassifierUtilities.jl")
include("HamNNNNclassifierio.jl")
include("HamNNNNclassifierReports.jl")

global version = v"0.1.1"
# v0.1.1 discontinues using the DataTables package, which has been deprecated. Uses DataFrames instead.
# also, need to use NullableArrays specifically, in order to have dropnull.

# Terminology:
# instances: cases; records; examples
#     instances can be grouped into sets, eg training set, test set, validation set
#     corresponds to a table row in a tabular data base
# Dataset
#     consists of a number of instances (cases, or examples)
# features: variables; fields; attributes
#     corresponds to a column in a tabular data base. The feature name is the column header
#     one of the features will be the class feature or class variable (also called the target variable, or
# 	class attribute)
#     features can be combined in various ways, to create new features
# feature extraction
#     the process of finding those features which provide the best classification performance
# classes
#     the set of values that the class feature can take
# parameters
#     eg, the k in k-nearest-neighbors
#     often, the work involved in applying machine learning to a problem is to find appropriate or optimal values for the parameters used by a given ML methodology
# Hamming distance (may also be called "overlap metric"
# bins
#     the number of bins or slices to be applied for a given continuous feature
# binning
#     the process of converting a continuous feature into a discrete feature
# discrete (as applied to a feature or variable)
#     nominal or ordinal data
# continuous (as applied to a feature or variable)
#     real-valued
#     ordinal data with a range greater than a certain parameter may also be treated as continous data

# NEWER ORANGE FORMAT:
# Prefixed attributes contain a one- or two-lettered prefix, followed by '#'
# and the attribute name. The first letter of the prefix can be:
# 	'm' for meta-attributes
# 	'i' to ignore the attribute
# 	'c' to define the class attribute
#
# the second letter of the prefix can be:
# 	'D' for discrete
# 	'C' for continuous
# 	'S' for string
# 	'B' for basket
# if there are no prefixes for an attribute (ie just the attribute name)
# then the attribute will be treated as discrete, unless the actual values
# are numbers, in which case it will be treated as continuous.
#
# OLDER ORANGE FORMAT:
# the information about variable type, etc is contained in two rows, following the row with
# attribute names:
# in the second line:
# 	'd' or 'discrete' or a list of values: denotes a discrete attribute
# 	'c' or 'continuous': denotes a continuous attribute
# 	'string' denotes a string variable, which we ignore
# 	'basket': these are continuous-valued meta attributes; ignore
# 	it may also contain a string of values separated by spaces. Use these
# 	as the values for a discrete attribute.
# the third line contains optional flags:
# 	'i' or 'ignore'
# 	'c' or 'class': there can only be one class attribute. If none is found,
# 	 use the last attribute as the class attribute.
# 	'm' or 'meta': meta attribute, eg weighting information; ignore
# 	'-dc' followed by a value: indicates how a don't care is represented.

# # Design Principles
# To work with the MLBase Julia package, it will be helpful to have functions which can be used
# as parameters for MLBase functions (see the MLBase documentation), eg for cross_validate,
# estfun –
# The estimation function, which takes a vector of training indices as input and returns a learned model, as:
# model = estfun(train_inds)
# evalfun –
# The evaluation function, which takes a model and a vector of testing indices as input and returns a score that indicates the goodness of the model, as
# score = evalfun(model, test_inds)

function hamclass()
	println("hamclass...")
end

function train()
	println("training...")
end

function test()
	println("testing...")
end

function parse_commandline()
	s = ArgParseSettings()

	@add_arg_table s begin
		"--opt1"
			help = "an option with an argument"
		"--opt2", "-o"
			help = "another option with an argument"
			arg_type = Int
			default = 0
		"--flag1"
			help = "an option without argument, i.e. a flag"
			action = :store_true
		"arg1"
			help = "a positional argument"
			required = true
	end

	return parse_args(s)
end

# function main()
# 	parsed_args = parse_commandline()
# 	println("Parsed args:")
# 	for (arg,val) in parsed_args
# 		println("  $arg  =>  $val")
# 	end
# end

paramDict = Dict

dataFile="/Users/henryolders/hammingnn/explore/test.tab"

function main()
	printdatafiledescription(dataFile)
	printattributeranking(dataFile)
	# train_x, train_codes, train_y = generatetraintesttable(dataFile)
	# @show train_x train_codes train_y
	train()
	test()
end

function anneal()
	printdatafiledescription("/Users/henryolders/hammingnn/explore/anneal.tab")
	printattributeranking("/Users/henryolders/hammingnn/explore/anneal.tab")
end

function glass()
	printdatafiledescription("/Users/henryolders/hammingnn/explore/glass.tab")
	printattributeranking("/Users/henryolders/hammingnn/explore/glass.tab")
end

# uncomment the line below, to run this module from the command line.
main()

end # module


InputFolder = './Images/Kidney/';
OutputFolder = './Results/Images/Kidney/';
Fill = 1;
Dilate = 5;
Lbl = 1;

@iA = '*_C0000*.tif';		% Image filter
@iS = '*_C0001*.tif';		% Image annotations filter

@fxg_lBlockClassify[iA, iS] > [S];
params.BlckSize = 32;
params.Feat = 'mnstdbifs';
params.ClassifierType = 'RF';
params.ClassifierFile = './Classifiers/ClassifierKidney_mnstdbifsRF.mat';
params.ExportAnnotations = './Classifiers/Annotation_Kidney.tif';
/endf

/show iA > S;
/keep S > tif;


@testset "zygote_friendly_map" begin
    @testset "$name" for (name, f, x) in [
        ("Vector{Float64}", x -> sin(x) + cos(x) * exp(x), randn(100)),
        ("Fill{Float64}", x -> sin(x) + exp(x) + 5, Fill(randn(), 100)),
        ("Vector{Vector{Float64}}", sum, [randn(25) for _ in 1:33]),
        (
            "zip(Vector{Float64}, Fill{Float64})",
            x -> x[1] * x[2],
            zip(randn(5), Fill(1.0, 5)),
        ),
    ]
        @test TemporalGPs.zygote_friendly_map(f, x) ≈ map(f, x)
        adjoint_test(x -> TemporalGPs.zygote_friendly_map(f, x), (x, ))
    end
end